[bug-gettext] [PATCH] Incorporate PODIFF feature into gettext-tools

[Daiki Ueno]

Hi folks,

Since I'm too lazy to manually examine translation updates, I searched
for a tool which works like "diff".  I tried a few and liked PODIFF[1]
by Sergey Poznyakoff best.  Particularly because the tool can produce
familiar context/unified diff output using the diff program itself,
although the text positions in PO files do not provide much information.

So, I'd like to have this functionality included in gettext-tools.  I
don't know if this has ever been raised here, but as the implementation
was straightforward, I've created a proposed patch.  It can be typically
used like this:

$ msgdiff --no-location -D-u eo.po eo.po.new

Comments would be appreciated.

Footnotes: 
[1]  http://puszcza.gnu.org.ua/projects/podiff

>From 365a881723318df22ab2b453be33d34bb9458b45 Mon Sep 17 00:00:00 2001
From: Daiki Ueno <address@hidden>
Date: Sat, 1 Mar 2014 18:52:55 +0900
Subject: [PATCH] Add new program msgdiff

msgdiff is a utility to compute differences between two specified PO files,
based on the idea of PODIFF, by Sergey Poznyakoff:
http://puszcza.gnu.org.ua/projects/podiff

The msgdiff command first normalizes those files, by removing "fuzzy"
comments and sorting entries, and runs the "diff" command on them.
---
 gettext-tools/src/Makefile.am |   9 +-
 gettext-tools/src/msgdiff.c   | 453 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 461 insertions(+), 1 deletion(-)
 create mode 100644 gettext-tools/src/msgdiff.c

diff --git a/gettext-tools/src/Makefile.am b/gettext-tools/src/Makefile.am
index e6713af..2b12195 100644
--- a/gettext-tools/src/Makefile.am
+++ b/gettext-tools/src/Makefile.am
@@ -26,7 +26,8 @@ RM = rm -f
 
 bin_PROGRAMS = \
 msgcmp msgfmt msgmerge msgunfmt xgettext \
-msgattrib msgcat msgcomm msgconv msgen msgexec msgfilter msggrep msginit 
msguniq \
+msgattrib msgcat msgcomm msgconv msgdiff msgen msgexec msgfilter msggrep \
+msginit msguniq \
 recode-sr-latin
 
 noinst_PROGRAMS = hostname urlget
@@ -200,6 +201,11 @@ else
 msgconv_SOURCES = ../woe32dll/c++msgconv.cc
 endif
 if !WOE32DLL
+msgdiff_SOURCES = msgdiff.c
+else
+msgdiff_SOURCES = ../woe32dll/c++msgdiff.cc
+endif
+if !WOE32DLL
 msgen_SOURCES = msgen.c
 else
 msgen_SOURCES = ../woe32dll/c++msgen.cc
@@ -279,6 +285,7 @@ msgattrib_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ 
$(WOE32_LDADD)
 msgcat_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgcomm_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgconv_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
+msgdiff_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgen_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgexec_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
 msgfilter_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ $(WOE32_LDADD)
diff --git a/gettext-tools/src/msgdiff.c b/gettext-tools/src/msgdiff.c
new file mode 100644
index 0000000..7563341
--- /dev/null
+++ b/gettext-tools/src/msgdiff.c
@@ -0,0 +1,453 @@
+/* GNU gettext - internationalization aids
+   Copyright (C) 2014 Free Software Foundation, Inc.
+   Written by Daiki Ueno <address@hidden>, 2014
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <errno.h>
+#include <getopt.h>
+#include <limits.h>
+#include <locale.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "closeout.h"
+#include "dir-list.h"
+#include "str-list.h"
+#include "file-list.h"
+#include "error.h"
+#include "error-progname.h"
+#include "progname.h"
+#include "relocatable.h"
+#include "basename.h"
+#include "message.h"
+#include "read-catalog.h"
+#include "read-po.h"
+#include "read-properties.h"
+#include "read-stringtable.h"
+#include "write-catalog.h"
+#include "write-po.h"
+#include "write-properties.h"
+#include "write-stringtable.h"
+#include "color.h"
+#include "msgl-cat.h"
+#include "propername.h"
+#include "gettext.h"
+#include "xalloc.h"
+#include "error.h"
+#include "concat-filename.h"
+#include "clean-temp.h"
+#include "spawn-pipe.h"
+#include "wait-process.h"
+
+
+/* A convenience macro.  I don't like writing gettext() every time.  */
+#define _(str) gettext (str)
+
+
+/* Force output of PO file even if empty.  */
+static int force_po;
+
+/* Target encoding.  */
+static const char *to_code;
+
+/* Long options.  */
+static const struct option long_options[] =
+{
+  { "add-location", no_argument, &line_comment, 1 },
+  { "diff-option", required_argument, NULL, 'D' },
+  { "escape", no_argument, NULL, 'E' },
+  { "files-from", required_argument, NULL, 'f' },
+  { "force-po", no_argument, &force_po, 1 },
+  { "help", no_argument, NULL, 'h' },
+  { "indent", no_argument, NULL, 'i' },
+  { "no-escape", no_argument, NULL, 'e' },
+  { "no-location", no_argument, &line_comment, 0 },
+  { "no-wrap", no_argument, NULL, CHAR_MAX + 2 },
+  { "omit-header", no_argument, NULL, CHAR_MAX + 1 },
+  { "properties-input", no_argument, NULL, 'P' },
+  { "properties-output", no_argument, NULL, 'p' },
+  { "stringtable-input", no_argument, NULL, CHAR_MAX + 3 },
+  { "stringtable-output", no_argument, NULL, CHAR_MAX + 4 },
+  { "to-code", required_argument, NULL, 't' },
+  { "version", no_argument, NULL, 'V' },
+  { "width", required_argument, NULL, 'w', },
+  { NULL, 0, NULL, 0 }
+};
+
+
+/* Forward declaration of local functions.  */
+static void usage (int status)
+#if defined __GNUC__ && ((__GNUC__ == 2 && __GNUC_MINOR__ > 4) || __GNUC__ > 2)
+        __attribute__ ((noreturn))
+#endif
+;
+
+static void normalize_msgdomain_list (msgdomain_list_ty *mdlp);
+
+
+int
+main (int argc, char *argv[])
+{
+  int cnt;
+  int optchar;
+  bool do_help = false;
+  bool do_version = false;
+  char **diff_argv = NULL;
+  int diff_argv_count;
+  int diff_argv_max;
+  msgdomain_list_ty *result1;
+  msgdomain_list_ty *result2;
+  catalog_input_format_ty input_syntax = &input_format_po;
+  catalog_output_format_ty output_syntax = &output_format_po;
+  struct temp_dir *tmpdir;
+  char *output_file1 = NULL;
+  char *output_file2 = NULL;
+  pid_t child;
+  int exitstatus;
+  int fd[1];
+  FILE *fp;
+  int exitcode = EXIT_SUCCESS;
+
+  /* Set program name for messages.  */
+  set_program_name (argv[0]);
+  error_print_progname = maybe_print_progname;
+
+#ifdef HAVE_SETLOCALE
+  /* Set locale via LC_ALL.  */
+  setlocale (LC_ALL, "");
+#endif
+
+  /* Set the text message domain.  */
+  bindtextdomain (PACKAGE, relocate (LOCALEDIR));
+  bindtextdomain ("bison-runtime", relocate (BISON_LOCALEDIR));
+  textdomain (PACKAGE);
+
+  /* Ensure that write errors on stdout are detected.  */
+  atexit (close_stdout);
+
+  /* Set default values for variables.  */
+  diff_argv_max = 4;
+  diff_argv = XCALLOC (diff_argv_max, char *);
+  diff_argv_count = 0;
+
+  while ((optchar = getopt_long (argc, argv, "D:eEhinpPt:Vw:",
+                                 long_options, NULL)) != EOF)
+    switch (optchar)
+      {
+      case '\0':                /* Long option.  */
+        break;
+
+      case 'D':
+        /* DIFF_ARGV will be ["diff", *OPTIONS, "file1", "file2", NULL].  */
+        if (4 + diff_argv_count == diff_argv_max)
+          {
+            diff_argv_max = diff_argv_max * 2 + 10;
+            diff_argv = xrealloc (diff_argv, sizeof (char *) * diff_argv_max);
+          }
+        diff_argv[++diff_argv_count] = optarg;
+        break;
+
+      case 'e':
+        message_print_style_escape (false);
+        break;
+
+      case 'E':
+        message_print_style_escape (true);
+        break;
+
+      case 'h':
+        do_help = true;
+        break;
+
+      case 'i':
+        message_print_style_indent ();
+        break;
+
+      case 'n':
+        line_comment = 1;
+        break;
+
+      case 'p':
+        output_syntax = &output_format_properties;
+        break;
+
+      case 'P':
+        input_syntax = &input_format_properties;
+        break;
+
+      case 't':
+        to_code = optarg;
+        break;
+
+      case 'V':
+        do_version = true;
+        break;
+
+      case 'w':
+        {
+          int value;
+          char *endp;
+          value = strtol (optarg, &endp, 10);
+          if (endp != optarg)
+            message_page_width_set (value);
+        }
+        break;
+
+      case CHAR_MAX + 1:
+        omit_header = true;
+        break;
+
+      case CHAR_MAX + 2: /* --no-wrap */
+        message_page_width_ignore ();
+        break;
+
+      case CHAR_MAX + 3: /* --stringtable-input */
+        input_syntax = &input_format_stringtable;
+        break;
+
+      case CHAR_MAX + 4: /* --stringtable-output */
+        output_syntax = &output_format_stringtable;
+        break;
+
+      default:
+        usage (EXIT_FAILURE);
+        /* NOTREACHED */
+      }
+
+  /* Version information requested.  */
+  if (do_version)
+    {
+      printf ("%s (GNU %s) %s\n", basename (program_name), PACKAGE, VERSION);
+      /* xgettext: no-wrap */
+      printf (_("Copyright (C) %s Free Software Foundation, Inc.\n\
+License GPLv3+: GNU GPL version 3 or later 
<http://gnu.org/licenses/gpl.html>\n\
+This is free software: you are free to change and redistribute it.\n\
+There is NO WARRANTY, to the extent permitted by law.\n\
+"),
+              "2014");
+      printf (_("Written by %s.\n"), proper_name ("Daiki Ueno"));
+      exit (EXIT_SUCCESS);
+    }
+
+  /* Help is requested.  */
+  if (do_help)
+    usage (EXIT_SUCCESS);
+
+  /* Test whether we have two .po file names.  */
+  if (optind >= argc)
+    {
+      error (EXIT_SUCCESS, 0, _("no input files given"));
+      usage (EXIT_FAILURE);
+    }
+  if (optind + 2 != argc)
+    {
+      error (EXIT_SUCCESS, 0, _("exactly two input files required"));
+      usage (EXIT_FAILURE);
+    }
+  if (strcmp (argv[optind], "-") == 0
+      && strcmp (argv[optind + 1], "-") == 0)
+    {
+      error (EXIT_SUCCESS, 0, _("one input file must not be standard input"));
+      usage (EXIT_FAILURE);
+    }
+  /* FIXME: Support git diff arguments.  */
+
+  /* Read the PO files.  */
+  result1 = read_catalog_file (argv[optind], input_syntax);
+  result2 = read_catalog_file (argv[optind + 1], input_syntax);
+
+  normalize_msgdomain_list (result1);
+  normalize_msgdomain_list (result2);
+  msgdomain_list_sort_by_msgid (result1);
+  msgdomain_list_sort_by_msgid (result2);
+
+  /* Write the PO files into a temporary directory.  */
+  tmpdir = create_temp_dir ("msg", NULL, false);
+  if (tmpdir == NULL)
+    {
+      exitcode = EXIT_FAILURE;
+      goto quit1;
+    }
+
+  output_file1 =
+    xconcatenated_filename (tmpdir->dir_name, "file1", NULL);
+  register_temp_file (tmpdir, output_file1);
+  msgdomain_list_print (result1, output_file1, output_syntax, force_po, false);
+
+  output_file2 =
+    xconcatenated_filename (tmpdir->dir_name, "file2", NULL);
+  register_temp_file (tmpdir, output_file2);
+  msgdomain_list_print (result2, output_file2, output_syntax, force_po, false);
+
+  /* Call diff on the output files.  */
+  diff_argv[0] = "diff";
+  diff_argv[diff_argv_count + 1] = output_file1;
+  diff_argv[diff_argv_count + 2] = output_file2;
+  diff_argv[diff_argv_count + 3] = NULL;
+
+  child = create_pipe_in ("diff", "diff", diff_argv, DEV_NULL, false,
+                          true, true, fd);
+  fp = fdopen (fd[0], "r");
+  if (fp == NULL)
+    error (EXIT_FAILURE, errno, _("fdopen() failed"));
+
+  while (!feof (fp))
+    {
+      char buf[4096];
+      size_t count = fread (buf, 1, sizeof buf, fp);
+
+      if (count == 0)
+        {
+          if (ferror (fp))
+            error (EXIT_FAILURE, errno, _("\
+error while reading \"%s\""), optarg);
+          /* EOF reached.  */
+          break;
+        }
+
+      /* FIXME: apply styles?  */
+      fwrite (buf, 1, count, stdout);
+    }
+  fclose (fp);
+
+  /* Remove zombie process from process list, and retrieve exit status.  */
+  exitstatus = wait_subprocess (child, "diff", false, false, true, true, NULL);
+  if (exitstatus != 0)
+    error (EXIT_FAILURE, 0, _("%s subprocess failed with exit code %d"),
+           "diff", exitstatus);
+
+ quit2:
+  cleanup_temp_dir (tmpdir);
+ quit1:
+  free (diff_argv);
+  exit (exitcode);
+}
+
+
+/* Display usage information and exit.  */
+static void
+usage (int status)
+{
+  if (status != EXIT_SUCCESS)
+    fprintf (stderr, _("Try '%s --help' for more information.\n"),
+             program_name);
+  else
+    {
+      printf (_("\
+Usage: %s [OPTION] FILES...\n\
+"), program_name);
+      printf ("\n");
+      /* xgettext: no-wrap */
+      printf (_("\
+Find differences between the two specified PO files.\n\
+"));
+      printf ("\n");
+      printf (_("\
+Mandatory arguments to long options are mandatory for short options too.\n"));
+      printf ("\n");
+      printf (_("\
+Input file location:\n"));
+      printf (_("\
+  FILES                       input files\n"));
+      printf (_("\
+FILES are 'FILE1 FILE2'.\n\
+If a FILE is '-', read standard input.\n"));
+      printf ("\n");
+      printf (_("\
+Output file location:\n"));
+      printf (_("\
+  -o, --output-file=FILE      write output to specified file\n"));
+      printf (_("\
+The results are written to standard output if no output file is specified\n\
+or if it is -.\n"));
+      printf ("\n");
+      printf (_("\
+Input file syntax:\n"));
+      printf (_("\
+  -P, --properties-input      input files are in Java .properties syntax\n"));
+      printf (_("\
+      --stringtable-input     input files are in NeXTstep/GNUstep .strings\n\
+                              syntax\n"));
+      printf ("\n");
+      printf (_("\
+Output details:\n"));
+      printf (_("\
+  -e, --no-escape             do not use C escapes in output (default)\n"));
+      printf (_("\
+  -E, --escape                use C escapes in output, no extended chars\n"));
+      printf (_("\
+      --force-po              write PO file even if empty\n"));
+      printf (_("\
+  -i, --indent                write the .po file using indented style\n"));
+      printf (_("\
+      --no-location           do not write '#: filename:line' lines\n"));
+      printf (_("\
+  -n, --add-location          generate '#: filename:line' lines (default)\n"));
+      printf (_("\
+  -p, --properties-output     write out a Java .properties file\n"));
+      printf (_("\
+      --stringtable-output    write out a NeXTstep/GNUstep .strings file\n"));
+      printf (_("\
+  -w, --width=NUMBER          set output page width\n"));
+      printf (_("\
+      --no-wrap               do not break long message lines, longer than\n\
+                              the output page width, into several lines\n"));
+      printf (_("\
+      --omit-header           don't write header with 'msgid \"\"' entry\n"));
+      printf ("\n");
+      printf (_("\
+Informative output:\n"));
+      printf (_("\
+  -h, --help                  display this help and exit\n"));
+      printf (_("\
+  -V, --version               output version information and exit\n"));
+      printf ("\n");
+      /* TRANSLATORS: The placeholder indicates the bug-reporting address
+         for this package.  Please add _another line_ saying
+         "Report translation bugs to <...>\n" with the address for translation
+         bugs (typically your translation team's web or email address).  */
+      fputs (_("Report bugs to <address@hidden>.\n"),
+             stdout);
+    }
+
+  exit (status);
+}
+
+static void
+normalize_msgdomain_list (msgdomain_list_ty *mdlp)
+{
+  int i;
+
+  for (i = 0; i < mdlp->nitems; i++)
+    {
+      message_list_ty *mlp = mdlp->item[i]->messages;
+      int j;
+
+      for (j = 0; j < mlp->nitems; j++)
+        {
+          if (is_header (mlp->item[j]))
+            continue;
+
+          mlp->item[j]->is_fuzzy = false;
+          mlp->item[j]->obsolete = false;
+
+          /* FIXME: Remove uninterested comments.  */
+        }
+    }
+}
-- 
1.9.0.rc3