[PATCH] sort: use posix_fadvise to announce access patterns on files opened for reading

[Joey Degges]

Hello,

I added calls to the new function xfadvise wherever a file is opened for
reading. From what I could tell files are only opened for reading from
within stream_open and open_temp so that's where I added xfadvise calls.

xfadvise calls posix_fadvise with the flags NOREUSE, SEQUENTIAL, and
WILLNEED.

I tested this patch with ~500M non-cached files from both flash and disk
drives. To ensure the files were not cached I remounted the device before
each run. I found that the variance in the disk drive measurements was too
high to draw any conclusions (+/- 7seconds). On the flash drives the
variance was much lower and I saw a consistent speed up of 1 second.

Can you offer any suggestions/comments?

Thanks,
Joey



>From 34a93042301f0838d6086a4ca1aa251c89cf1ba0 Mon Sep 17 00:00:00 2001
From: Joey Degges <address@hidden>
Date: Thu, 25 Feb 2010 22:54:50 -0800
Subject: [PATCH] sort: use posix_fadvise to announce access patterns on
files opened for reading

---
 configure.ac |    3 +++
 src/sort.c   |   32 +++++++++++++++++++++++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index b07a52b..c07fbd4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -344,6 +344,9 @@ if test "$elf_sys" = "yes" && \
   gl_ADD_PROG([optional_pkglib_progs], [libstdbuf.so])
 fi

+# Check for fcntl.h/posix_fadvise
+AC_CHECK_HEADERS(fcntl.h, [AC_CHECK_FUNCS(posix_fadvise)])
+
 ############################################################################
 mk="$srcdir/src/Makefile.am"
 # Extract all literal names from the definition of $(EXTRA_PROGRAMS)
diff --git a/src/sort.c b/src/sort.c
index 481fdb8..ec63e62 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -47,6 +47,10 @@
 #include "xnanosleep.h"
 #include "xstrtol.h"

+#if HAVE_FCNTL_H
+# include <fcntl.h>
+#endif
+
 #if HAVE_SYS_RESOURCE_H
 # include <sys/resource.h>
 #endif
@@ -794,6 +798,19 @@ create_temp_file (int *pfd, bool survive_fd_exhaustion)
   return node;
 }

+/* Announce the access patterns NOREUSE, SEQUENTIAL, and WILLNEED on the
file
+   descriptor FD. Ignore any errors -- this is only advisory.  */
+
+static void
+xfadvise (int fd)
+{
+#if HAVE_POSIX_FADVISE
+  posix_fadvise (fd, 0, 0, POSIX_FADV_NOREUSE);
+  posix_fadvise (fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+  posix_fadvise (fd, 0, 0, POSIX_FADV_WILLNEED);
+#endif
+}
+
 /* Return a stream for FILE, opened with mode HOW.  A null FILE means
    standard output; HOW should be "w".  When opening for input, "-"
    means standard input.  To avoid confusion, do not return file
@@ -805,10 +822,18 @@ stream_open (const char *file, const char *how)
 {
   if (!file)
     return stdout;
-  if (STREQ (file, "-") && *how == 'r')
+  if (*how == 'r')
     {
-      have_read_stdin = true;
-      return stdin;
+      FILE *fp;
+      if (STREQ (file, "-"))
+        {
+          have_read_stdin = true;
+          fp =  stdin;
+        }
+      else
+        fp = fopen (file, how);
+      xfadvise (fileno (fp));
+      return fp;
     }
   return fopen (file, how);
 }
@@ -1044,6 +1069,7 @@ open_temp (const char *name, pid_t pid)
       break;
     }

+  xfadvise (fileno (fp));
   return fp;
 }

-- 
1.6.6.1