[gnuastro-commits] master a98b7afc: Arithmetic: new --arguments to allow very long argument lists

[Mohammad Akhlaghi]

branch: master
commit a98b7afcd22a69f7a2cc4b8c8005213afcbe2c9e
Author: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Commit: Mohammad Akhlaghi <mohammad@akhlaghi.org>

    Arithmetic: new --arguments to allow very long argument lists
    
    Until now, if the list of arguments given to Arithmetic was too long, the
    shell would abort with an "Argument list is too long" error. In the last
    few commits an '--argument' option was added to the Fits program. This can
    be useful for Arithmetic also.
    
    With this commit, this option has been added to Arithmetic
---
 NEWS                  |  6 ++++++
 bin/arithmetic/args.h | 13 +++++++++++++
 bin/arithmetic/main.h |  1 +
 bin/arithmetic/ui.c   | 41 ++++++++++++++++++++++++++---------------
 bin/arithmetic/ui.h   |  1 +
 bin/fits/args.h       |  2 +-
 doc/gnuastro.texi     | 30 ++++++++++++++++++++++++------
 7 files changed, 72 insertions(+), 22 deletions(-)

diff --git a/NEWS b/NEWS
index 82a5bf48..058fbf41 100644
--- a/NEWS
+++ b/NEWS
@@ -26,6 +26,12 @@ See the end of the file for license conditions.
     --outfitsnocommit: do not write the Git commit.
 
 *** Arithmetic
+  --arguments: takes the name of plain-text file that contains the list of
+    arguments to the program. This option is critical if you have very long
+    (thousands) of operands and operators (which happen in large pipelines
+    where the arguments are constructed automatically). Without this
+    option, the shell is going to abort with an "Argument list too long"
+    error message.
   - New operators:
     - rotate-coord: given a 2D point's coordinates, return the coordinates
       after it has been rotated by your requested angle around your
diff --git a/bin/arithmetic/args.h b/bin/arithmetic/args.h
index b9d79796..4702c912 100644
--- a/bin/arithmetic/args.h
+++ b/bin/arithmetic/args.h
@@ -83,6 +83,19 @@ struct argp_option program_options[] =
       GAL_OPTIONS_NOT_MANDATORY,
       GAL_OPTIONS_NOT_SET
     },
+    {
+      "arguments",
+      UI_KEY_ARGUMENTS,
+      "STR",
+      0,
+      "Plain-text file with command-line arguments.",
+      GAL_OPTIONS_GROUP_INPUT,
+      &p->arguments,
+      GAL_TYPE_STRING,
+      GAL_OPTIONS_RANGE_ANY,
+      GAL_OPTIONS_NOT_MANDATORY,
+      GAL_OPTIONS_NOT_SET
+    },
 
 
 
diff --git a/bin/arithmetic/main.h b/bin/arithmetic/main.h
index 5d3b1fd4..663a9cc0 100644
--- a/bin/arithmetic/main.h
+++ b/bin/arithmetic/main.h
@@ -73,6 +73,7 @@ struct arithmeticparams
   struct gal_arithmetic_set_params setprm; /* Parameters for 'set-'.    */
 
   /* Input: */
+  char          *arguments;  /* File containing arguments to be used.   */
   gal_list_str_t     *hdus;  /* List of all given HDU strings.          */
   gal_list_str_t   *tokens;  /* List of all arithmetic tokens.          */
   char            *wcsfile;  /* File to use for output's WCS.           */
diff --git a/bin/arithmetic/ui.c b/bin/arithmetic/ui.c
index 5801f28e..d58786b9 100644
--- a/bin/arithmetic/ui.c
+++ b/bin/arithmetic/ui.c
@@ -28,6 +28,7 @@ along with Gnuastro. If not, see 
<http://www.gnu.org/licenses/>.
 #include <stdio.h>
 #include <string.h>
 
+#include <gnuastro/txt.h>
 #include <gnuastro/wcs.h>
 #include <gnuastro/list.h>
 #include <gnuastro/fits.h>
@@ -261,21 +262,31 @@ ui_check_options_and_arguments(struct arithmeticparams *p)
   char *tofilename=NULL, *basename=NULL;
   struct gal_options_common_params *cp=&p->cp;
 
-  /* First, make sure that any tokens are actually given. */
-  if(p->tokens==NULL)
-    error(EXIT_FAILURE, 0, "no input tokens. Please specify a file name "
-          "or number (as operands) along with operator(s) as input. "
-          "Please run any of the following commands for more "
-          "information.\n\n"
-          "    $ astarithmetic --help           # Short info.\n"
-          "    $ info astarithmetic             # Full invocation "
-          "documentation.\n");
-
-  /* The input tokens are put in a lastin-firstout (simple) linked list, so
-     change them to the correct order so the order we pop a token is the
-     same order that the user input a value. Note that for the options this
-     was done in 'gal_options_read_config_set'. */
-  gal_list_str_reverse(&p->tokens);
+  /* First, make sure that tokens are actually given. */
+  if(p->tokens)
+    {
+      /* The input tokens are put in a lastin-firstout (simple) linked
+         list, so change them to the correct order so the order we pop a
+         token is the same order that the user input a value. Note that for
+         the options this was done in 'gal_options_read_config_set'. */
+      gal_list_str_reverse(&p->tokens);
+    }
+  else
+    {
+      if(p->arguments)
+        {
+          p->tokens=gal_txt_read_to_list(p->arguments);
+          free(p->arguments); p->arguments=NULL;
+        }
+      else
+        error(EXIT_FAILURE, 0, "no input tokens. Please specify a file "
+              "name or number (as operands) along with operator(s) as "
+              "input; or use the '--arguments' options. Please run any "
+              "of the following commands for more information.\n\n"
+              "    $ astarithmetic --help           # Short info.\n"
+              "    $ info astarithmetic             # Full invocation "
+              "documentation.\n");
+    }
 
   /* To allow adding extensions to existing files, let the 'keep' flag be
      the same as the 'dontdelete'. */
diff --git a/bin/arithmetic/ui.h b/bin/arithmetic/ui.h
index 42d7df4c..5658e886 100644
--- a/bin/arithmetic/ui.h
+++ b/bin/arithmetic/ui.h
@@ -51,6 +51,7 @@ enum option_keys_enum
   /* Only with long version (start with a value 1000, the rest will be set
      automatically). */
   UI_KEY_ENVSEED         = 1000,
+  UI_KEY_ARGUMENTS,
 };
 
 
diff --git a/bin/fits/args.h b/bin/fits/args.h
index ffefc9eb..9aac71be 100644
--- a/bin/fits/args.h
+++ b/bin/fits/args.h
@@ -37,7 +37,7 @@ struct argp_option program_options[] =
       UI_KEY_ARGUMENTS,
       "STR",
       0,
-      "plain-text file with list of input files.",
+      "Plain-text file with command-line arguments.",
       GAL_OPTIONS_GROUP_INPUT,
       &p->arguments,
       GAL_TYPE_STRING,
diff --git a/doc/gnuastro.texi b/doc/gnuastro.texi
index 722c8213..768e6083 100644
--- a/doc/gnuastro.texi
+++ b/doc/gnuastro.texi
@@ -16205,7 +16205,7 @@ $ astfits image-a.fits --keyvalue=NAXIS,NAXIS1 \
 @noindent
 @cindex Argument list too long
 @strong{Argument list too long:} if the list of input files are too long, the 
shell is going to complain with the @code{Argument list too long} error!
-To avoid this problem, you can put the list of files in a plain-text file and 
give that plain-text file to the Fits program through the @option{--infilelist} 
option discussed below.
+To avoid this problem, you can put the list of files in a plain-text file and 
give that plain-text file to the Fits program through the @option{--arguments} 
option discussed below.
 @end cartouche
 
 The output is internally stored (and finally printed) as a table (with one 
column per keyword).
@@ -16252,17 +16252,17 @@ For example, with the command below you can search 
all the FITS files in all the
 astfits $(find /TOP/DIR/ -name "*.fits") --keyvalue=NAXIS2
 @end example
 
-@item --infilelist=STR
+@item --arguments=STR
 A plain-text file containing the list of input files that will be used in 
@option{--keyvalue}.
 Each word (group of characters separated by SPACE or new-line) is assumed to 
be the name of the separate input file.
 This option is only relevant when no input files are given as arguments on the 
command-line: if any arguments are given, this option is ignored.
 
-This is necessary when the list of input files are very long; causing the 
shell to emits an @code{Argument list too long} error!
+This is necessary when the list of input files are very long; causing the 
shell to abort with an @code{Argument list too long} error.
 In such cases, you can put the list into a plain-text file and use this option 
like below:
 
 @example
 $ ls $(path)/*.fits > list.txt
-$ astfits --infilelist=list.txt --keyvalue=NAXIS1
+$ astfits --arguments=list.txt --keyvalue=NAXIS1
 @end example
 
 @item -O
@@ -24000,7 +24000,7 @@ $ astarithmetic 1 image.fits / --out=inverse.fits
 $ astarithmetic image.fits -1 x --out=negative.fits
 
 ## Subtract extension 4 from extension 1 (counting from zero):
-$ astarithmetic image.fits image.fits - --out=skysub.fits           \
+$ astarithmetic image.fits image.fits - --out=skysub.fits \
                 --hdu=1 --hdu=4
 
 ## Add two images, then divide them by 2 (2 is read as floating point):
@@ -24011,7 +24011,7 @@ $ astarithmetic image1.fits image2.fits + 2.0 / 
--out=average.fits
 $ astarithmetic image1.fits image2.fits average --out=average.fits
 
 ## Calculate the median of three images in three separate extensions:
-$ astarithmetic img1.fits img2.fits img3.fits median                \
+$ astarithmetic img1.fits img2.fits img3.fits median \
                 -h0 -h1 -h2 --out=median.fits
 @end example
 
@@ -24039,6 +24039,24 @@ Arithmetic just redefines the @option{--hdu} and 
@option{--dontdelete} options a
 
 @table @option
 
+@item --arguments=STR
+A plain-text file containing the command-line arguments that will be used by 
Arithmetic.
+This option is only relevant when no arguments are given on the command-line: 
if any arguments are given, this option is ignored.
+
+@cindex Argument list too long
+This is necessary when the set of of input files and operators (arguments; see 
@ref{Arguments and options}) are very long (thousands of long file names for 
example; usually generated within large pipelines).
+Such long arguments will cause the shell to abort with an @code{Argument list 
too long} error.
+In such cases, you can put the list into a plain-text file and use this option 
like below (assuming you want to stack all the files in a certain directory 
with the @code{mean} operator; see @ref{Stacking operators}):
+
+@example
+$ counter=0;
+$ for f in $(pwd)/*.fits; do \
+    echo $f; counter=$((counter+1)); \
+  done > arguments.txt; \
+  echo "$counter mean" >> arguments.txt
+$ astarithmetic --arguments=list.txt -g1
+@end example
+
 @item -h INT/STR
 @itemx --hdu INT/STR
 The header data unit of the input FITS images, see @ref{Input output options}.