[Nmh-workers] [PATCH] New optimization for addir

[Eric Gillespie]

Removing that incorrect nlink optimization caused folder -all
-fast -recur to go from 0.021s to 2.929s on my hierarchy of
706710 messages in 239 folders.  Still not too shabby, and
probably faster than this command was even with its broken
optimization back in the 80s :), but I think there's room
for improvement.

This works on at least BSD and (I think) Linux; I don't know
about the proprietary unixes.  I've tested that the non-optimized
case is still correct.  I've also tested this with symlinks to
files and directories to confirm correct behavior for those.
I really ought to write a test...

=== modified file 'ChangeLog'
--- ChangeLog   2008-08-05 19:14:15 +0000
+++ ChangeLog   2008-08-05 20:33:25 +0000
@@ -1,3 +1,16 @@
+2008-08-05  Eric Gillespie  <address@hidden>
+
+       * uip/folder.c: chdir(nmhdir) in main rather than in dodir, which
+       is called many times.  Drop the chdir in get_folder_info, which
+       served no purpose at all.  If struct stat has d_type (BSD), use
+       that to skip processing (strcmp, stat, and so on) of message
+       files, mostly undoing the slow-down from the last change.
+
+       * configure.in: Call CHECK_TYPE_STRUCT_DIRENT_D_TYPE.
+
+       * aclocal.m4: Add CHECK_TYPE_STRUCT_DIRENT_D_TYPE from
+       GNU coreutils.
+
 2008-08-04  Eric Gillespie  <address@hidden>
 
        * uip/folder.c: Simplify dodir/addir/addfold.  Dump hacky

=== modified file 'aclocal.m4'
--- aclocal.m4  2005-12-24 17:17:38 +0000
+++ aclocal.m4  2008-08-05 20:16:13 +0000
@@ -84,3 +84,55 @@ else
   :
 fi
 ])dnl
+
+dnl ----------------
+dnl CHECK FOR d_type
+dnl ----------------
+dnl
+dnl From Jim Meyering.
+dnl
+dnl Check whether struct dirent has a member named d_type.
+dnl
+
+# Copyright (C) 1997, 1999, 2000, 2001, 2002, 2003, 2004 Free Software
+# Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+AC_DEFUN([CHECK_TYPE_STRUCT_DIRENT_D_TYPE],
+  [AC_REQUIRE([AC_HEADER_DIRENT])dnl
+   AC_CACHE_CHECK([for d_type member in directory struct],
+                  jm_cv_struct_dirent_d_type,
+     [AC_TRY_LINK(dnl
+       [
+#include <sys/types.h>
+#ifdef HAVE_DIRENT_H
+# include <dirent.h>
+#else /* not HAVE_DIRENT_H */
+# define dirent direct
+# ifdef HAVE_SYS_NDIR_H
+#  include <sys/ndir.h>
+# endif /* HAVE_SYS_NDIR_H */
+# ifdef HAVE_SYS_DIR_H
+#  include <sys/dir.h>
+# endif /* HAVE_SYS_DIR_H */
+# ifdef HAVE_NDIR_H
+#  include <ndir.h>
+# endif /* HAVE_NDIR_H */
+#endif /* HAVE_DIRENT_H */
+       ],
+       [struct dirent dp; dp.d_type = 0;],
+
+       jm_cv_struct_dirent_d_type=yes,
+       jm_cv_struct_dirent_d_type=no)
+     ]
+   )
+   if test $jm_cv_struct_dirent_d_type = yes; then
+     AC_DEFINE(HAVE_STRUCT_DIRENT_D_TYPE, 1,
+       [Define if there is a member named d_type in the struct describing
+        directory headers.])
+   fi
+  ]
+)

=== modified file 'configure.in'
--- configure.in        2008-05-13 02:39:04 +0000
+++ configure.in        2008-08-05 20:16:38 +0000
@@ -950,6 +950,8 @@ if test "$nmh_cv_union_wait" = yes; then
 fi
 AC_MSG_RESULT($nmh_cv_union_wait)
 
+CHECK_TYPE_STRUCT_DIRENT_D_TYPE()
+
 dnl -------------
 dnl CHECK SIGNALS
 dnl -------------

=== modified file 'uip/folder.c'
--- uip/folder.c        2008-08-05 19:14:15 +0000
+++ uip/folder.c        2008-08-05 20:32:09 +0000
@@ -365,6 +365,9 @@ main (int argc, char **argv)
        /*
         * If no folder is given, do them all
         */
+       /* change directory to base of nmh directory for dodir */
+       if (chdir (nmhdir) == NOTOK)
+           adios (nmhdir, "unable to change directory to");
        if (!argfolder) {
            if (msg)
                admonish (NULL, "no folder given for message %s", msg);
@@ -422,10 +425,6 @@ dodir (char *dir)
 
     start = foldp;
 
-    /* change directory to base of nmh directory */
-    if (chdir (nmhdir) == NOTOK)
-       adios (nmhdir, "unable to change directory to");
-
     addir (dir);
 
     for (i = start; i < foldp; i++) {
@@ -441,7 +440,6 @@ static int
 get_folder_info (char *fold, char *msg)
 {
     int        i, retval = 1;
-    char *mailfile;
     struct msgs *mp = NULL;
 
     i = total_folders++;
@@ -463,42 +461,36 @@ get_folder_info (char *fold, char *msg)
     fi[i].others = 0;
     fi[i].error  = 0;
 
-    mailfile = m_maildir (fold);
-
-    if (!chdir (mailfile)) {
-       if ((ftotal > 0) || !fshort || msg || fpack) {
-           /*
-            * create message structure and get folder info
-            */
-           if (!(mp = folder_read (fold))) {
-               admonish (NULL, "unable to read folder %s", fold);
-               return 0;
-           }
-
-           /* set the current message */
-           if (msg && !sfold (mp, msg))
-               retval = 0;
-
-           if (fpack) {
-               if (folder_pack (&mp, fverb) == -1)
-                   done (1);
-               seq_save (mp);          /* synchronize the sequences */
-               context_save ();        /* save the context file     */
-           }
+    if ((ftotal > 0) || !fshort || msg || fpack) {
+       /*
+        * create message structure and get folder info
+        */
+       if (!(mp = folder_read (fold))) {
+           admonish (NULL, "unable to read folder %s", fold);
+           return 0;
+       }
 
-           /* record info for this folder */
-           if ((ftotal > 0) || !fshort) {
-               fi[i].nummsg = mp->nummsg;
-               fi[i].curmsg = mp->curmsg;
-               fi[i].lowmsg = mp->lowmsg;
-               fi[i].hghmsg = mp->hghmsg;
-               fi[i].others = other_files (mp);
-           }
+       /* set the current message */
+       if (msg && !sfold (mp, msg))
+           retval = 0;
+
+       if (fpack) {
+           if (folder_pack (&mp, fverb) == -1)
+               done (1);
+           seq_save (mp);              /* synchronize the sequences */
+           context_save ();    /* save the context file     */
+       }
 
-           folder_free (mp); /* free folder/message structure */
+       /* record info for this folder */
+       if ((ftotal > 0) || !fshort) {
+           fi[i].nummsg = mp->nummsg;
+           fi[i].curmsg = mp->curmsg;
+           fi[i].lowmsg = mp->lowmsg;
+           fi[i].hghmsg = mp->hghmsg;
+           fi[i].others = other_files (mp);
        }
-    } else {
-       fi[i].error = 1;
+
+       folder_free (mp); /* free folder/message structure */
     }
 
     if (frecurse && (fshort || fi[i].others) && (fi[i].error == 0))
@@ -669,6 +661,7 @@ addir (char *name)
     struct stat st;
     struct dirent *dp;
     DIR * dd;
+    int child_is_folder;
 
     if (!(dd = opendir (name))) {
        admonish (name, "unable to read directory ");
@@ -682,11 +675,26 @@ addir (char *name)
     }
 
     while ((dp = readdir (dd))) {
+       /* If the system supports it, try to skip processing of children we
+        * know are not directories or symlinks. */
+       child_is_folder = -1;
+#if defined(HAVE_STRUCT_DIRENT_D_TYPE)
+       if (dp->d_type == DT_DIR) {
+           child_is_folder = 1;
+       } else if (dp->d_type != DT_LNK && dp->d_type != DT_UNKNOWN) {
+           continue;
+       }
+#endif
        if (!strcmp (dp->d_name, ".") || !strcmp (dp->d_name, "..")) {
            continue;
        }
        child = concat (prefix, dp->d_name, (void *)NULL);
-       if (stat (child, &st) != -1 && S_ISDIR(st.st_mode)) {
+       /* If we have no d_type or d_type is DT_LNK or DT_UNKNOWN, stat the
+        * child to see what it is. */
+       if (child_is_folder == -1) {
+           child_is_folder = (stat (child, &st) != -1 && S_ISDIR(st.st_mode));
+       }
+       if (child_is_folder) {
            /* addfold saves child in the list, don't free it */
            addfold (child);
        } else {