Re: inotify back end for tail -f on linux

[Giuseppe Scrivano]

Hi Jim,


Jim Meyering <address@hidden> writes:

> No test is too small or too "obvious" to add ;-)
>
> Adding this simple test would be great.
> Thanks!

I found another problem.  The inotify version can't be used when --pid
is specified.  There are other inotify events that I want to investigate
better if they can be used with --pid and in this case we can add them
later when the inotify back end is stable enough for me to break it
again :)  What do you think?

This is the updated version, I integrated your test-case in the
tests/tail-2/wait file and I added a new file (tests/tail-2/pid) with
additional tests for the --pid option.

Regards,
Giuseppe




>From f0824b8b79c1fb22e9a48cbf831a87433fc4d3e8 Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Tue, 2 Jun 2009 08:28:23 +0200
Subject: [PATCH] tail: Use inotify if it is available.

* NEWS: Document the new feature.
* configure.ac: Check if inotify is present.
* src/tail.c (tail_forever_inotify): New function.
(main): Use the inotify-based function, if possible.
* tests/Makefile.am: Added new tests for tail.
* tests/tail-2/pid: New file.
* tests/tail-2/wait: New file.
---
 NEWS              |    2 +
 configure.ac      |    2 +
 src/tail.c        |  245 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
 tests/Makefile.am |    2 +
 tests/tail-2/pid  |   81 ++++++++++++++++++
 tests/tail-2/wait |  114 +++++++++++++++++++++++++
 6 files changed, 445 insertions(+), 1 deletions(-)
 create mode 100644 tests/tail-2/pid
 create mode 100644 tests/tail-2/wait

diff --git a/NEWS b/NEWS
index 29b09a0..c5a2ef5 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,8 @@ GNU coreutils NEWS                                    -*- 
outline -*-
   sort accepts a new option, --human-numeric-sort (-h): sort numbers
   while honoring human readable suffixes like KiB and MB etc.
 
+  tail uses inotify if it is present.
+
 
 * Noteworthy changes in release 7.4 (2009-05-07) [stable]
 
diff --git a/configure.ac b/configure.ac
index 4eb640e..a63ac0c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -410,6 +410,8 @@ AM_GNU_GETTEXT_VERSION([0.15])
 # For a test of uniq: it uses the $LOCALE_FR envvar.
 gt_LOCALE_FR
 
+AC_CHECK_FUNCS([inotify_init], [AC_DEFINE([HAVE_INOTIFY], [1], [Check for 
libinotify])])
+
 AC_CONFIG_FILES(
   Makefile
   doc/Makefile
diff --git a/src/tail.c b/src/tail.c
index 9d416e1..6d5f870 100644
--- a/src/tail.c
+++ b/src/tail.c
@@ -46,6 +46,11 @@
 #include "xstrtol.h"
 #include "xstrtod.h"
 
+#ifdef HAVE_INOTIFY
+# include "hash.h"
+# include <sys/inotify.h>
+#endif
+
 /* The official name of this program (e.g., no `g' prefix).  */
 #define PROGRAM_NAME "tail"
 
@@ -125,6 +130,17 @@ struct File_spec
   /* The value of errno seen last time we checked this file.  */
   int errnum;
 
+#ifdef HAVE_INOTIFY
+  /* The watch descriptor used by inotify.  */
+  int wd;
+
+  /* The parent directory watch descriptor.  It is used only
+   * when Follow_name is used.  */
+  int parent_wd;
+
+  /* Offset in NAME of the basename part.  */
+  size_t basename_start;
+#endif
 };
 
 /* Keep trying to open a file even if it is inaccessible when tail starts
@@ -1117,6 +1133,226 @@ tail_forever (struct File_spec *f, int nfiles, double 
sleep_interval)
     }
 }
 
+#ifdef HAVE_INOTIFY
+
+static size_t
+wd_hasher (const void *entry, size_t tabsize)
+{
+  const struct File_spec *spec = entry;
+  return spec->wd % tabsize;
+}
+
+static bool
+wd_comparator (const void *e1, const void *e2)
+{
+  const struct File_spec *spec1 = e1;
+  const struct File_spec *spec2 = e2;
+  return spec1->wd == spec2->wd;
+}
+
+/* Tail NFILES files forever, or until killed.
+   Check modifications using the inotify events system.  */
+static void
+tail_forever_inotify (int wd, struct File_spec *f, int nfiles)
+{
+  unsigned int i;
+  unsigned int max_realloc = 3;
+  Hash_table *wd_table;
+
+  bool found_watchable = false;
+  size_t last;
+  size_t evlen = 0;
+  char *evbuff;
+  size_t evbuff_off = 0;
+  ssize_t len = 0;
+
+  wd_table = hash_initialize (nfiles, NULL, wd_hasher, wd_comparator, NULL);
+  if (! wd_table)
+    xalloc_die ();
+
+  for (i = 0; i < nfiles; i++)
+    {
+      if (!f[i].ignore)
+        {
+          size_t fnlen = strlen (f[i].name);
+          if (evlen < fnlen)
+            evlen = fnlen;
+
+          f[i].wd = 0;
+
+          if (follow_mode == Follow_name)
+            {
+              size_t dirlen = dir_len (f[i].name);
+              char prev = f[i].name[dirlen];
+              f[i].basename_start = last_component (f[i].name) - f[i].name;
+
+              f[i].name[dirlen] = '\0';
+
+               /* It's fine to add the same directory more than once.
+                  In that case the same watch descriptor is returned.  */
+              f[i].parent_wd = inotify_add_watch (wd, dirlen ? f[i].name : ".",
+                                                  IN_CREATE | IN_MOVED_TO);
+
+              f[i].name[dirlen] = prev;
+
+              if (f[i].parent_wd < 0)
+                {
+                  error (0, errno, _("cannot watch parent directory of %s"),
+                         quote (f[i].name));
+                  continue;
+                }
+            }
+
+          if (f[i].errnum != ENOENT)
+            {
+              int old_wd = f[i].wd;
+              f[i].wd = inotify_add_watch (wd, f[i].name,
+                                           (IN_MODIFY | IN_ATTRIB
+                                            | IN_DELETE_SELF | IN_MOVE_SELF));
+
+              if (last == old_wd)
+                last = f[i].wd;
+
+              if (f[i].wd < 0)
+                {
+                  error (0, errno, _("cannot watch %s"), quote (f[i].name));
+                  continue;
+                }
+
+              hash_insert (wd_table, &(f[i]));
+            }
+
+          if (follow_mode == Follow_name || f[i].wd)
+            found_watchable = true;
+        }
+    }
+
+  if (!found_watchable)
+    return;
+
+  last = f[nfiles - 1].wd;
+
+  evlen += sizeof (struct inotify_event) + 1;
+  evbuff = xmalloc (evlen);
+
+  while (1)
+    {
+      char const *name;
+      struct File_spec *fspec;
+      uintmax_t bytes_read;
+      struct stat stats;
+
+      struct inotify_event *ev;
+
+      if (len <= evbuff_off)
+        {
+          len = read (wd, evbuff, evlen);
+          evbuff_off = 0;
+
+          if (len <= 0 && errno == EINVAL && max_realloc--)
+            {
+              len = 0;
+              evlen *= 2;
+              evbuff = xrealloc (evbuff, evlen);
+              continue;
+            }
+
+          if (len <= 0 && errno == EINTR)
+            {
+              len = 0;
+              continue;
+            }
+
+          if (len <= 0)
+            error (EXIT_FAILURE, errno, _("error reading inotify event"));
+        }
+
+      ev = (struct inotify_event *) (evbuff + evbuff_off);
+      evbuff_off += sizeof (*ev) + ev->len;
+
+      if (ev->mask & (IN_CREATE | IN_MOVED_TO))
+        {
+          for (i = 0; i < nfiles; i++)
+            {
+              if (f[i].parent_wd == ev->wd &&
+                  STREQ (ev->name, f[i].name + f[i].basename_start))
+                break;
+            }
+
+          /* It is not a watched file.  */
+          if (i == nfiles)
+            continue;
+
+          f[i].wd = inotify_add_watch (wd, f[i].name,
+                                       IN_MODIFY | IN_ATTRIB | IN_DELETE_SELF);
+
+          if (f[i].wd < 0)
+            {
+              error (0, errno, _("cannot watch %s"), quote (f[i].name));
+              continue;
+            }
+
+          fspec = &(f[i]);
+          hash_insert (wd_table, fspec);
+
+          if (follow_mode == Follow_name)
+            recheck (&(f[i]), false);
+        }
+      else
+        {
+          struct File_spec key;
+          key.wd = ev->wd;
+          fspec = hash_lookup (wd_table, &key);
+        }
+
+      if (! fspec)
+        continue;
+
+      if (ev->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF))
+        {
+          if (ev->mask & (IN_DELETE_SELF | IN_MOVE_SELF))
+            {
+              inotify_rm_watch (wd, f[i].wd);
+              hash_delete (wd_table, &(f[i]));
+            }
+          if (follow_mode == Follow_name)
+            recheck (fspec, false);
+
+          continue;
+        }
+
+      name = pretty_name (fspec);
+
+      if (fstat (fspec->fd, &stats) != 0)
+        {
+          close_fd (fspec->fd, name);
+          fspec->fd = -1;
+          fspec->errnum = errno;
+          continue;
+        }
+
+      if (S_ISREG (fspec->mode) && stats.st_size < fspec->size)
+        {
+          error (0, 0, _("%s: file truncated"), name);
+          last = ev->wd;
+          xlseek (fspec->fd, stats.st_size, SEEK_SET, name);
+          fspec->size = stats.st_size;
+        }
+
+      if (ev->wd != last)
+        {
+          if (print_headers)
+            write_header (name);
+          last = ev->wd;
+        }
+
+      bytes_read = dump_remainder (name, fspec->fd, COPY_TO_EOF);
+      fspec->size += bytes_read;
+    }
+
+}
+#endif
+
 /* Output the last N_BYTES bytes of file FILENAME open for reading in FD.
    Return true if successful.  */
 
@@ -1691,7 +1927,14 @@ main (int argc, char **argv)
     ok &= tail_file (&F[i], n_units);
 
   if (forever)
-    tail_forever (F, n_files, sleep_interval);
+    {
+#ifdef HAVE_INOTIFY
+      int wd = inotify_init ();
+      if (wd > 0 && pid == 0)
+        tail_forever_inotify (wd, F, n_files);
+#endif
+      tail_forever (F, n_files, sleep_interval);
+    }
 
   if (have_read_stdin && close (STDIN_FILENO) < 0)
     error (EXIT_FAILURE, errno, "-");
diff --git a/tests/Makefile.am b/tests/Makefile.am
index a0ed986..c108356 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -419,6 +419,8 @@ TESTS =                                             \
   tail-2/big-4gb                               \
   tail-2/proc-ksyms                            \
   tail-2/start-middle                          \
+  tail-2/pid                                   \
+  tail-2/wait                                  \
   touch/dangling-symlink                       \
   touch/dir-1                                  \
   touch/fail-diag                              \
diff --git a/tests/tail-2/pid b/tests/tail-2/pid
new file mode 100644
index 0000000..ed7bc7e
--- /dev/null
+++ b/tests/tail-2/pid
@@ -0,0 +1,81 @@
+#!/bin/sh
+# Test the --pid option of tail.
+
+# Copyright (C) 2003, 2006-2009 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+if test "$VERBOSE" = yes; then
+  set -x
+  tail --version
+fi
+
+. $srcdir/test-lib.sh
+
+sleep 2 &
+pid=$!
+sleep .5
+grep '^State:[  ]*[S]' /proc/$pid/status > /dev/null 2>&1 ||
+  skip_test_ "/proc/$pid/status: missing or 'different'"
+kill $pid
+
+if [ ! -e here ]; then
+  touch here || framework_failure
+fi
+
+fail=0
+
+
+# Use tail itself to create a background process.
+
+tail -f here &
+bg_pid=$!
+
+tail -f here --pid=$bg_pid &
+
+pid=$!
+
+sleep 2
+
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift # Remove the leading `_'.
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *) echo $0: process dead 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+sleep 2
+
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    *) ;;
+    S*) echo $0: process still active 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+
+kill $bg_pid
+rm here
+
+Exit $fail
diff --git a/tests/tail-2/wait b/tests/tail-2/wait
new file mode 100644
index 0000000..ce8c535
--- /dev/null
+++ b/tests/tail-2/wait
@@ -0,0 +1,114 @@
+#!/bin/sh
+# Make sure that `tail -f' returns immediately if a file doesn't exist
+# while `tail -F' waits for it to appear.
+
+# Copyright (C) 2003, 2006-2009 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+if test "$VERBOSE" = yes; then
+  set -x
+  tail --version
+fi
+
+. $srcdir/test-lib.sh
+
+sleep 2 &
+pid=$!
+sleep .5
+grep '^State:[  ]*[S]' /proc/$pid/status > /dev/null 2>&1 ||
+  skip_test_ "/proc/$pid/status: missing or 'different'"
+kill $pid
+
+if [ ! -e here ]; then
+  touch here || framework_failure
+fi
+
+if [ -e not_here ]; then
+  rm -f not_here || framework_failure
+fi
+
+fail=0
+
+rm -f  tail.err
+
+(tail -f not_here 2> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift # Remove the leading `_'.
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    *) ;;
+    S*) echo $0: process still active 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+(tail -f here 2>> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *)  echo $0: process died 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+# The `tail -F' behaviour on both files must not change.
+
+(tail -F here 2>> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *) echo $0: process died 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+(tail -F not_here 2>> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *) echo $0: process died 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+test  "$(wc -w < tail.err) eq 0"  || fail=1
+
+rm -f here
+rm -f tail.err
+
+
+Exit $fail
-- 
1.6.3.1