Re: [Bug-cpio] [PATCH] Add option "--reproducible" for reproducible archives

[Sergey Poznyakoff]

Hello,

I pushed the attached patch which implements the --reproducible option.

Regards,
Sergey

>From 3945f9db44c935608caa5f084fd7f67ae59ee9e1 Mon Sep 17 00:00:00 2001
From: Sergey Poznyakoff <address@hidden>
Date: Mon, 1 Dec 2014 15:02:38 +0200
Subject: [PATCH] New options to create device and inode-independent archives.

* src/util.c (inode_val): New member trans_inode
(find_inode_val): New function.
(find_inode_file): Rewrite using the above.
(add_inode): Initialize the trans_inode member
depending on the value of renumber_inodes_option.
(get_inode_and_dev): New function.
(stat_to_cpio): Use get_inode_and_dev.
(arf_stores_inode_p): New function.
* src/extern.h (renumber_inodes_option)
(ignore_devno_option): New externs.
* src/global.c (renumber_inodes_option)
(ignore_devno_option): New variables.
* src/main.c: Add new options.

* NEWS: Document changes.
* doc/cpio.1: Document new options.
* doc/cpio.texi: Likewise.
---
 NEWS          | 13 ++++++++-
 doc/cpio.1    | 15 +++++++++--
 doc/cpio.texi |  9 +++++++
 src/extern.h  |  8 ++++--
 src/global.c  |  4 +++
 src/main.c    | 34 ++++++++++++++++++++++--
 src/util.c    | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
 7 files changed, 150 insertions(+), 17 deletions(-)

diff --git a/NEWS b/NEWS
index a7d2a57..dbd6ce6 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-GNU cpio NEWS -- history of user-visible changes. 2014-01-30
+GNU cpio NEWS -- history of user-visible changes. 2014-12-01
 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2009,
 2010, 2014 Free Software Foundation, Inc.
 See the end of file for copying conditions.
@@ -9,7 +9,18 @@ Version 2.11.90 - Git

 * Improved documentation.
 * Manpages are installed by make install.
+* New options for copy-out mode:

+** --ignore-devno
+Store 0 in the device number fields, instead of the actual device
+number.
+
+** --renumber-inodes
+Renumber inodes when storing them in the archive.
+
+** --device-independent or --reproducible
+Create reproducible archives.  This is equivalent to
+--ignore-devno --renumber-inodes.
 
 Version 2.11 - Sergey Poznyakoff, 2010-03-10

diff --git a/doc/cpio.1 b/doc/cpio.1
index d48d2df..c71ba6c 100644
--- a/doc/cpio.1
+++ b/doc/cpio.1
@@ -13,7 +13,7 @@
 .\"
 .\" You should have received a copy of the GNU General Public License
 .\" along with GNU cpio.  If not, see <http://www.gnu.org/licenses/>.
-.TH CPIO 1 "January 30, 2014" "CPIO" "GNU CPIO"
+.TH CPIO 1 "December 1, 2014" "CPIO" "GNU CPIO"
 .SH NAME
 cpio \- copy files to and from archives
 .SH SYNOPSIS
@@ -29,7 +29,7 @@ cpio \- copy files to and from archives
 [\fB\-\-block\-size=\fIblocks\fR] [\fB\-\-dereference\fR]
 [\fB\-\-io\-size=\fIBYTES\fR] [\fB\-\-quiet\fR]
 [\fB\-\-force\-local\fR] [\fB\-\-rsh\-command=\fICOMMAND\fR]
-< \fIname-list\fR [\fB>\fR \fIarchive\fR]
+\fB<\fR \fIname-list\fR [\fB>\fR \fIarchive\fR]

 .B cpio
 {\fB\-i\fR|\fB\-\-extract\fR} [\fB\-bcdfmnrtsuvBSV\fR] [\fB\-C\fR \fIBYTES\fR]
@@ -262,6 +262,14 @@ in the archive, without actually extracting the files.
 .BR \-A ", " \-\-append
 Append to an existing archive.
 .TP
+.BR \-\-device\-independent ", " \-\-reproducible
+Create reproducible archives.  This is equivalent to
+.BR "\-\-ignore\-devno \-\-renumber\-inodes" .
+.TP
+.B \-\-ignore\-devno
+Store 0 in the device number field of each archive member, instead of
+the actual device number.
+.TP
 \fB\-O\fR address@hidden:\fR]\fIARCHIVE-NAME\fR
 Use \fIARCHIVE-NAME\fR instead of standard output. Optional \fIUSER\fR and
 \fIHOST\fR specify the user and host names in case of a remote
@@ -269,6 +277,9 @@ archive.

 The output archive name can be specified wither using this option, or
 using \fB\-F\fR (\fB\-\-file\fR), but not both.
+.TP
+.B \-\-renumber\-inodes
+Renumber inodes when storing them in the archive.
 .SS Operation modifiers valid only in copy-pass mode
 .TP
 .BR \-l ", " \-\-link
diff --git a/doc/cpio.texi b/doc/cpio.texi
index c1cf11b..dee3c13 100644
--- a/doc/cpio.texi
+++ b/doc/cpio.texi
@@ -461,6 +461,10 @@ Set the I/O block size to the given @var{number} of bytes.
 @item -d
 @itemx --make-directories
 Create leading directories where needed.
address@hidden --device-independent
address@hidden --reproducible
+Create reproducible archives.  This is equivalent to
address@hidden --renumber-inodes}.
 @item -D @var{dir}
 @itemx address@hidden
 Change to directory @var{dir}
@@ -483,6 +487,9 @@ Treat the archive file as local, even if its name contains 
colons.
 @itemx address@hidden
 Use given archive format.  @xref{format}, for a list of available
 formats.
address@hidden --ignore-devno
+Store 0 in the device number field of each archive member, instead of
+the actual device number.
 @item -l
 @itemx --link
 Link files instead of copying them, when possible.
@@ -512,6 +519,8 @@ Use @var{command} instead of @command{rsh} to access remote 
archives.
 @item -r
 @itemx --rename
 Interactively rename files
address@hidden --renumber-inodes
+Renumber inodes when storing them in the archive.
 @item -R
 @itemx address@hidden:address@hidden
 Set the ownership of all files created to the specified @var{user}
diff --git a/src/extern.h b/src/extern.h
index 92117cd..da16794 100644
--- a/src/extern.h
+++ b/src/extern.h
@@ -56,6 +56,8 @@ extern int only_verify_crc_flag;
 extern int no_abs_paths_flag;
 extern unsigned int warn_option;
 extern mode_t newdir_umask;
+extern int renumber_inodes_option;
+extern int ignore_devno_option;

 /* Values for warn_option */
 #define CPIO_WARN_NONE     0
@@ -171,8 +173,8 @@ void create_all_directories (char *name);
 void prepare_append (int out_file_des);
 char *find_inode_file (ino_t node_num,
                       unsigned long major_num, unsigned long minor_num);
-void add_inode (ino_t node_num, char *file_name,
-               unsigned long major_num, unsigned long minor_num);
+struct inode_val *add_inode (ino_t node_num, char *file_name,
+                            unsigned long major_num, unsigned long minor_num);
 int open_archive (char *file);
 void tape_offline (int tape_des);
 void get_next_reel (int tape_des);
@@ -218,3 +220,5 @@ void delay_set_stat (char const *file_name, struct stat *st,
 int repair_delayed_set_stat (struct cpio_file_stat *file_hdr);
 void apply_delayed_set_stat (void);

+int arf_stores_inode_p (enum archive_format arf);
+
diff --git a/src/global.c b/src/global.c
index c699f6e..0449193 100644
--- a/src/global.c
+++ b/src/global.c
@@ -195,3 +195,7 @@ int (*xstat) ();
 void (*copy_function) () = 0;

 char *change_directory_option;
+
+int renumber_inodes_option;
+int ignore_devno_option;
+
diff --git a/src/main.c b/src/main.c
index e1f2c5c..ee9f64e 100644
--- a/src/main.c
+++ b/src/main.c
@@ -58,7 +58,10 @@ enum cpio_options {
   FORCE_LOCAL_OPTION,
   DEBUG_OPTION,
   BLOCK_SIZE_OPTION,
-  TO_STDOUT_OPTION
+  TO_STDOUT_OPTION,
+  RENUMBER_INODES_OPTION,
+  IGNORE_DEVNO_OPTION,
+  DEVICE_INDEPENDENT_OPTION
 };

 const char *program_authors[] =
@@ -190,6 +193,13 @@ static struct argp_option options[] = {
    N_("Append to an existing archive."), GRID+1 },
   {NULL, 'O', N_("address@hidden:]FILE-NAME"), 0,
    N_("Archive filename to use instead of standard output. Optional USER and 
HOST specify the user and host names in case of a remote archive"), GRID+1 },
+  {"renumber-inodes", RENUMBER_INODES_OPTION, NULL, 0,
+   N_("Renumber inodes") },
+  {"ignore-devno", IGNORE_DEVNO_OPTION, NULL, 0,
+   N_("Don't store device numbers") },
+  {"device-independent", DEVICE_INDEPENDENT_OPTION, NULL, 0,
+   N_("Create device-independent (reproducible) archives") },
+  {"reproducible", 0, NULL, OPTION_ALIAS },
 #undef GRID

   /* ********** */
@@ -441,7 +451,19 @@ crc newc odc bin ustar tar (all-caps also recognized)"), 
arg));
        USAGE_ERROR ((0, 0, _("Mode already defined")));
       copy_function = process_copy_pass;
       break;
-
+
+    case IGNORE_DEVNO_OPTION:
+      ignore_devno_option = 1;
+      break;
+
+    case RENUMBER_INODES_OPTION:
+      renumber_inodes_option = 1;
+      break;
+
+    case DEVICE_INDEPENDENT_OPTION:
+      ignore_devno_option = renumber_inodes_option = 1;
+      break;
+
     case RSH_COMMAND_OPTION:
       rsh_command_option = arg;
       break;
@@ -592,6 +614,8 @@ process_args (int argc, char *argv[])
       CHECK_USAGE (xstat != lstat, "--dereference", "--extract");
       CHECK_USAGE (append_flag, "--append", "--extract");
       CHECK_USAGE (output_archive_name, "-O", "--extract");
+      CHECK_USAGE (renumber_inodes_option, "--renumber-inodes", "--extract");
+      CHECK_USAGE (ignore_devno_option, "--ignore-devno", "--extract");
       if (to_stdout_option)
        {
          CHECK_USAGE (create_dir_flag, "--make-directories", "--to-stdout");
@@ -649,6 +673,9 @@ process_args (int argc, char *argv[])
        archive_format = arf_binary;
       if (output_archive_name)
        archive_name = output_archive_name;
+
+      if (!arf_stores_inode_p (archive_format))
+       renumber_inodes_option = ignore_devno_option = 0;
     }
   else
     {
@@ -675,6 +702,9 @@ process_args (int argc, char *argv[])
       CHECK_USAGE (no_abs_paths_flag, "--absolute-pathnames",
                   "--pass-through");
       CHECK_USAGE (to_stdout_option, "--to-stdout", "--pass-through");
+      CHECK_USAGE (renumber_inodes_option, "--renumber-inodes",
+                  "--pass-through");
+      CHECK_USAGE (ignore_devno_option, "--ignore-devno", "--pass-through");

       directory_name = argv[index];
     }
diff --git a/src/util.c b/src/util.c
index 18b3e42..6c483f8 100644
--- a/src/util.c
+++ b/src/util.c
@@ -692,6 +692,7 @@ struct inode_val
   ino_t inode;
   unsigned long major_num;
   unsigned long minor_num;
+  ino_t trans_inode;
   char *file_name;
 };

@@ -715,8 +716,8 @@ inode_val_compare (const void *val1, const void *val2)
          && ival1->minor_num == ival2->minor_num;
 }

-char *
-find_inode_file (ino_t node_num, unsigned long major_num,
+static struct inode_val *
+find_inode_val (ino_t node_num, unsigned long major_num,
                 unsigned long minor_num)
 {
   struct inode_val sample;
@@ -728,32 +729,78 @@ find_inode_file (ino_t node_num, unsigned long major_num,
   sample.inode = node_num;
   sample.major_num = major_num;
   sample.minor_num = minor_num;
-  ival = hash_lookup (hash_table, &sample);
+  return hash_lookup (hash_table, &sample);
+}
+
+char *
+find_inode_file (ino_t node_num, unsigned long major_num,
+                unsigned long minor_num)
+{
+  struct inode_val *ival = find_inode_val (node_num, major_num, minor_num);
   return ival ? ival->file_name : NULL;
 }

 /* Associate FILE_NAME with the inode NODE_NUM.  (Insert into hash table.)  */

-void
+static ino_t next_inode;
+
+struct inode_val *
 add_inode (ino_t node_num, char *file_name, unsigned long major_num,
           unsigned long minor_num)
 {
   struct inode_val *temp;
-  struct inode_val *e;
+  struct inode_val *e = NULL;

   /* Create new inode record.  */
   temp = (struct inode_val *) xmalloc (sizeof (struct inode_val));
   temp->inode = node_num;
   temp->major_num = major_num;
   temp->minor_num = minor_num;
-  temp->file_name = xstrdup (file_name);
+  temp->file_name = file_name ? xstrdup (file_name) : NULL;
+
+  if (renumber_inodes_option)
+    temp->trans_inode = next_inode++;
+  else
+    temp->trans_inode = temp->inode;

   if (!((hash_table
         || (hash_table = hash_initialize (0, 0, inode_val_hasher,
                                           inode_val_compare, 0)))
        && (e = hash_insert (hash_table, temp))))
     xalloc_die ();
-  /* FIXME: e is not used */
+  return e;
+}
+
+static ino_t
+get_inode_and_dev (struct cpio_file_stat *hdr, struct stat *st)
+{
+  if (renumber_inodes_option)
+    {
+      if (st->st_nlink > 1)
+       {
+         struct inode_val *ival = find_inode_val (st->st_ino,
+                                                  major (st->st_dev),
+                                                  minor (st->st_dev));
+         if (!ival)
+           ival = add_inode (st->st_ino, NULL,
+                             major (st->st_dev), minor (st->st_dev));
+         hdr->c_ino = ival->trans_inode;
+       }
+      else
+       hdr->c_ino = next_inode++;
+    }
+  else
+    hdr->c_ino = st->st_ino;
+  if (ignore_devno_option)
+    {
+      hdr->c_dev_maj = 0;
+      hdr->c_dev_min = 0;
+    }
+  else
+    {
+      hdr->c_dev_maj = major (st->st_dev);
+      hdr->c_dev_min = minor (st->st_dev);
+    }
 }

 
@@ -1211,9 +1258,8 @@ sparse_write (int fildes, char *buf, size_t nbytes, bool 
flush)
 void
 stat_to_cpio (struct cpio_file_stat *hdr, struct stat *st)
 {
-  hdr->c_dev_maj = major (st->st_dev);
-  hdr->c_dev_min = minor (st->st_dev);
-  hdr->c_ino = st->st_ino;
+  get_inode_and_dev (hdr, st);
+
   /* For POSIX systems that don't define the S_IF macros,
      we can't assume that S_ISfoo means the standard Unix
      S_IFfoo bit(s) are set.  So do it manually, with a
@@ -1620,6 +1666,7 @@ change_dir ()
                         (warn_option & CPIO_WARN_INTERDIR) ?
                         _("Creating directory `%s'") : NULL))
            exit (PAXEXIT_FAILURE);
+
          if (chdir (change_directory_option) == 0)
            return;
        }
@@ -1627,3 +1674,20 @@ change_dir ()
             _("cannot change to directory `%s'"), change_directory_option);
     }
 }
+
+/* Return true if the archive format ARF stores inode numbers */
+int
+arf_stores_inode_p (enum archive_format arf)
+{
+  switch (arf)
+    {
+    case arf_tar:
+    case arf_ustar:
+      return 0;
+
+    default:
+      break;
+    }
+  return 1;
+}
+
--
1.7.12.1