[gnuastro-commits] master afbefe19: Make extension: new function to extract unique values of a keyword

[Mohammad Akhlaghi]

branch: master
commit afbefe196934362b26fea58c8eff85b2cced3473
Author: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Commit: Mohammad Akhlaghi <mohammad@akhlaghi.org>

    Make extension: new function to extract unique values of a keyword
    
    Until now, when you wanted to get all the unique values that are given to a
    FITS keyword in various files, it was only possible by calling 'astfits',
    and using Table's column arithmetic to extract the unique values. Within a
    shell script this is fine, but within a Makefile, this is too annoying and
    can slow down the process.
    
    With this commit, a new Make extension function of Gnuastro has been
    defined for the purpose above. It can greatly help in designing generic
    pipelines on data where you don't know the full set of possible keyword
    values a-priori.
---
 NEWS                            |  13 +++--
 doc/gnuastro.texi               |  42 ++++++++++++++
 lib/Makefile.am                 |   2 +-
 lib/fits.c                      | 118 ++++++++++++++++++++++++++++++++++++++++
 lib/gnuastro/fits.h             |   7 +++
 lib/{gnumake.c => makeplugin.c} | 100 ++++++++++++++++++----------------
 6 files changed, 230 insertions(+), 52 deletions(-)

diff --git a/NEWS b/NEWS
index cd4046f7..8d6b9a77 100644
--- a/NEWS
+++ b/NEWS
@@ -104,18 +104,21 @@ See the end of the file for license conditions.
 
   Library functions:
   - gal_box_border_rotate_around_center: width of box after rotation.
-  - gal_color_name_to_id: return the ID of a color from its name.
   - gal_color_id_to_name: return the name of a color from its ID.
-  - gal_color_in_rgb: return the fraction of red-green-blue in given color.
-  - gal_eps_shape_name_to_id: return the ID of a shape from its name.
+  - gal_color_in_rgb: return the fraction of red-green-blue in a color.
+  - gal_color_name_to_id: return the ID of a color from its name.
   - gal_eps_shape_id_to_name: return the name of a shape from its ID.
+  - gal_eps_shape_name_to_id: return the ID of a shape from its name.
+  - gal_fits_unique_keyvalues: extract all unique values to a certain
+        keyword in many files.
+  - gal_fits_with_keyvalue: select FITS image with a certain key value.
   - gal_list_data_select_by_name: select a dataset from a list by its name.
   - gal_list_str_cat: Concatenate (append) list to a space-separated string.
   - gal_list_str_extract: Extract space-separated tokens to a list.
-  - gal_units_mag_to_sb: surface brightness (SB) from magnitude and area.
-  - gal_units_sb_to_mag: magnitude from SB and area.
   - gal_units_counts_to_sb: SB from counts, zeropoint and area.
+  - gal_units_mag_to_sb: surface brightness (SB) from magnitude and area.
   - gal_units_sb_to_counts: counts from SB, zeropoint and area.
+  - gal_units_sb_to_mag: magnitude from SB and area.
 
 
 ** Removed features
diff --git a/doc/gnuastro.texi b/doc/gnuastro.texi
index d33c5044..9fca55dd 100644
--- a/doc/gnuastro.texi
+++ b/doc/gnuastro.texi
@@ -27950,6 +27950,15 @@ For instructions on how to load Gnuastro's Make 
functions, see @ref{Loading the
 
 The Make functions in Gnuastro have been recently added (in August 2022), and 
will be gradually incrasing, as we find the need for more specialized functions.
 
+@cartouche
+@noindent
+@strong{Difference between `@code{=}' or `@code{:=}' for variable definition} 
When you define a variable with @code{=}, its value is expanded only when used, 
not when defined.
+However, when you use @code{:=}, it is immediately expanded.
+If your value doesn't contain functions, there is no difference between the 
two.
+However, if you have functions in the value of a variable, it is better to 
expand the value immediately with @code{:=}.
+Otherwise, each time the value is used, the function is expaned (possibly may 
times) and this will reduce the speed of your pipeline.
+For more, see the 
@url{https://www.gnu.org/software/make/manual/html_node/Flavors.html, The two 
flavors of variables} in the GNU Make manual.
+@end cartouche
 
 @table @code
 @item $(ast-fits-with-keyvalue FITS_FILES, HDU, KEYNAME, KEYVALUES)
@@ -27970,6 +27979,31 @@ all:
        echo "Full:     $(words $(files)) files";
        echo "Selected: $(words $(selected)) files"
 @end verbatim
+
+@item $(ast-fits-unique-keyvalues FITS_FILES, HDU, KEYNAME)
+Will return the unique values given to the FITS keyword (@code{KEYNAME}) in 
the given HDU of all the input FITS files.
+For example, after the commands below, the @code{keyvalues} variable will 
contain the unique values given to the @code{FOO} keyword in HDU number 1 of 
all the FITS files in @file{/datasets/images/*.fits}.
+
+@example
+files := $(wildcard /datasets/images/*.fits)
+keyvalues := $(ast-fits-unique-keyvalues $(files), 1, FOO)
+@end example
+
+This is useful when you don't know the full range of values a-priori.
+For example, let's assume that you are looking at a night's observations with 
a telescope.
+Furthermore, the purpose of the FITS image is written in the @code{OBJECT} 
keyword of the image (which we can assume is in HDU number 1).
+This keyword can have the name of the various science (for example 
@code{NGC123} and @code{M31}) and calibration targets (for example @code{BIAS} 
and @code{FLAT}).
+The list of science targets is different from project to project, such that in 
one night, you can observe multiple projects.
+But the calibration frames have unique names.
+Knowing the calibration keyword values, you can use 
@code{ast-fits-unique-keyvalues} to extract the science keyword values of the 
night, and group them together in the next stages to make separate stacks of 
deep images for each science target.
+For next stages, you can select files based on their keyword values using the 
@code{ast-fits-with-keyvalue} function.
+
+@example
+calib = BIAS FLAT
+files := $(wildcard /datasets/images/*.fits)
+science := $(filter-out $(calib), \
+             $(ast-fits-unique-keyvalues $(files), 1, OBJECT))
+@end example
 @end table
 
 
@@ -31960,6 +31994,14 @@ of their FITS outputs (which is empty when the FITS 
file is created by
 Gnuastro's program and this library).
 @end deftypefun
 
+@deftypefun {gal_list_str_t *} gal_fits_with_keyvalue (gal_list_str_t *files, 
char *hdu, char *name, gal_list_str_t *values)
+Given a list of FITS file names (@code{files}), a certain HDU (@code{hdu}), a 
certain keyword name (@code{name}), and a list of acceptable values 
(@code{values}), return the subset of file names where the requested keyword 
name has one of the acceptable values.
+@end deftypefun
+
+@deftypefun {gal_list_str_t *} gal_fits_unique_keyvalues (gal_list_str_t 
*files, char *hdu, char *name)
+Given a list of FITS file names (@code{files}), a certain HDU (@code{hdu}), a 
certain keyword name (@code{name}), retun the list of unique values to that 
keyword name in all the files.
+@end deftypefun
+
 
 
 
diff --git a/lib/Makefile.am b/lib/Makefile.am
index f87b6c30..b8597d1a 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -71,7 +71,7 @@ libgnuastro_make_la_LDFLAGS = -version-info $(GAL_LT_VERSION)
 
 
 # Specify the library .c files
-libgnuastro_make_la_SOURCES = gnumake.c
+libgnuastro_make_la_SOURCES = makeplugin.c
 libgnuastro_la_SOURCES = \
   $(MAYBE_WCSDISTORTION) \
   arithmetic.c \
diff --git a/lib/fits.c b/lib/fits.c
index 41af504b..bab69df4 100644
--- a/lib/fits.c
+++ b/lib/fits.c
@@ -34,6 +34,7 @@ along with Gnuastro. If not, see 
<http://www.gnu.org/licenses/>.
 #include <gsl/gsl_version.h>
 
 #include <gnuastro/git.h>
+#include <gnuastro/txt.h>
 #include <gnuastro/wcs.h>
 #include <gnuastro/list.h>
 #include <gnuastro/fits.h>
@@ -2269,6 +2270,123 @@ gal_fits_key_write_config(gal_fits_list_key_t 
**keylist, char *title,
 
 
 
+/* From an input list of FITS files and a HDU, select those that have a
+   certain value(s) in a certain keyword.*/
+gal_list_str_t *
+gal_fits_with_keyvalue(gal_list_str_t *files, char *hdu, char *name,
+                       gal_list_str_t *values)
+{
+  int status=0;
+  fitsfile *fptr;
+  char keyvalue[FLEN_VALUE];
+  gal_list_str_t *f, *v, *out=NULL;
+
+  /* Go over the list of files and see if they have the requested
+     keyword(s). */
+  for(f=files; f!=NULL; f=f->next)
+    {
+      /* Open the file. */
+      fptr=gal_fits_hdu_open(f->v, hdu, READONLY, 0);
+
+      /* Only attempt to read the value if the requested HDU could be
+         opened ('fptr!=NULL'). */
+      if(fptr)
+        {
+          /* Check if the keyword actually exists. */
+          if( gal_fits_key_exists_fptr(fptr, name) )
+            {
+              /* Read the keyword. Note that we aren't checking for the
+                 'status' here. If for any reason CFITSIO couldn't read the
+                 value and status if non-zero, the next step won't consider
+                 the file any more. */
+              status=0;
+              fits_read_key(fptr, TSTRING, name, &keyvalue, NULL,
+                            &status);
+
+              /* If the value corresponds to any of the user's values for this
+                 keyword, add it to the list of output names. */
+              if(status==0)
+                for(v=values; v!=NULL; v=v->next)
+                  {
+                    if( strcmp(v->v, keyvalue)==0 )
+                      { gal_list_str_add(&out, f->v, 1); break; }
+                  }
+            }
+
+          /* Close the file. */
+          if( fits_close_file(fptr, &status) )
+            gal_fits_io_error(status, NULL);
+        }
+    }
+
+  /* Reverse the list to be in same order as input and return. */
+  gal_list_str_reverse(&out);
+  return out;
+}
+
+
+
+
+
+/* From an input list of FITS files and a HDU, select those that have a
+   certain value(s) in a certain keyword.*/
+gal_list_str_t *
+gal_fits_unique_keyvalues(gal_list_str_t *files, char *hdu, char *name)
+{
+  fitsfile *fptr;
+  int status=0, newvalue;
+  char *keyv, keyvalue[FLEN_VALUE];
+  gal_list_str_t *f, *v, *out=NULL;
+
+  /* Go over the list of files and see if they have the requested
+     keyword(s). */
+  for(f=files; f!=NULL; f=f->next)
+    {
+      /* Open the file. */
+      fptr=gal_fits_hdu_open(f->v, hdu, READONLY, 0);
+
+      /* Only attempt to read the value if the requested HDU could be
+         opened ('fptr!=NULL'). */
+      if(fptr)
+        {
+          /* Check if the keyword actually exists. */
+          if( gal_fits_key_exists_fptr(fptr, name) )
+            {
+              /* Read the keyword. Note that we aren't checking for the
+                 'status' here. If for any reason CFITSIO couldn't read the
+                 value and status if non-zero, the next step won't consider
+                 the file any more. */
+              status=0;
+              fits_read_key(fptr, TSTRING, name, &keyvalue, NULL,
+                            &status);
+
+              /* If the value is new, add it to the list. */
+              if(status==0)
+                {
+                  newvalue=1;
+                  keyv=gal_txt_trim_space(keyvalue);
+                  for(v=out; v!=NULL; v=v->next)
+                    { if( strcmp(v->v, keyv)==0 ) newvalue=0; }
+                  if(newvalue) gal_list_str_add(&out, keyv, 1);
+                }
+            }
+
+          /* Close the file. */
+          if( fits_close_file(fptr, &status) )
+            gal_fits_io_error(status, NULL);
+        }
+    }
+
+  /* Reverse the list to be in same order as input and return. */
+  gal_list_str_reverse(&out);
+  return out;
+}
+
+
+
+
+
+
 
 
 
diff --git a/lib/gnuastro/fits.h b/lib/gnuastro/fits.h
index 8e6da2e6..56e5340a 100644
--- a/lib/gnuastro/fits.h
+++ b/lib/gnuastro/fits.h
@@ -268,6 +268,13 @@ void
 gal_fits_key_write_config(gal_fits_list_key_t **keylist, char *title,
                           char *extname, char *filename, char *hdu);
 
+gal_list_str_t *
+gal_fits_with_keyvalue(gal_list_str_t *files, char *hdu, char *name,
+                       gal_list_str_t *values);
+
+gal_list_str_t *
+gal_fits_unique_keyvalues(gal_list_str_t *files, char *hdu, char *name);
+
 
 
 
diff --git a/lib/gnumake.c b/lib/makeplugin.c
similarity index 60%
rename from lib/gnumake.c
rename to lib/makeplugin.c
index 7d897589..217a6d1d 100644
--- a/lib/gnumake.c
+++ b/lib/makeplugin.c
@@ -47,8 +47,9 @@ int plugin_is_GPL_compatible=1;
 
 
 /* Names of the separate functions */
-#define GNUMAKE_FUNC_PREFIX "ast"
-char *with_keyvalue_name=GNUMAKE_FUNC_PREFIX"-fits-with-keyvalue";
+#define MAKEPLUGIN_FUNC_PREFIX "ast"
+static char 
*fits_with_keyvalue_name=MAKEPLUGIN_FUNC_PREFIX"-fits-with-keyvalue";
+static char 
*fits_unique_keyvalues_name=MAKEPLUGIN_FUNC_PREFIX"-fits-unique-keyvalues";
 
 
 
@@ -57,7 +58,7 @@ char 
*with_keyvalue_name=GNUMAKE_FUNC_PREFIX"-fits-with-keyvalue";
 /* Select the input files that have the requested value(s) in the requested
    keywords. */
 static int
-gnumake_goodinput(char **argv, size_t numargs, char *name)
+makeplugin_goodinput(char **argv, size_t numargs, char *name)
 {
   char *c;
   size_t i;
@@ -85,66 +86,61 @@ gnumake_goodinput(char **argv, size_t numargs, char *name)
 
 
 static char *
-gnumake_fits_with_keyvalue(const char *caller, unsigned int argc,
-                           char **argv)
+makeplugin_fits_with_keyvalue(const char *caller, unsigned int argc,
+                              char **argv)
 {
-  int status=0;
-  fitsfile *fptr;
-  char keyvalue[FLEN_VALUE];
   gal_list_str_t *outlist=NULL;
   char *name=gal_txt_trim_space(argv[2]);
+  gal_list_str_t *files=NULL, *values=NULL;
   char *out, *hdu=gal_txt_trim_space(argv[1]);
-  gal_list_str_t *f, *v, *files=NULL, *values=NULL;
 
   /* If any of the inputs are empty, then don't bother continuing. */
-  if( gnumake_goodinput(argv, 4, with_keyvalue_name)==0 )
+  if( makeplugin_goodinput(argv, 4, fits_with_keyvalue_name)==0 )
     return NULL;
 
   /* Extract the components in the arguments with possibly multiple
-     values.*/
+     values and find the output files.*/
   files=gal_list_str_extract(argv[0]);
   values=gal_list_str_extract(argv[3]);
+  outlist=gal_fits_with_keyvalue(files, hdu, name, values);
 
-  /* Go over the list of files and see if they have the requested
-     keyword(s). */
-  for(f=files; f!=NULL; f=f->next)
-    {
-      /* Open the file. */
-      fptr=gal_fits_hdu_open(f->v, hdu, READONLY, 0);
+  /* Write the output value. */
+  out=gal_list_str_cat(outlist);
 
-      /* Only attempt to read the value if the requested HDU could be
-         opened ('fptr!=NULL'). */
-      if(fptr)
-        {
-          /* Check if the keyword actually exists. */
-          if( gal_fits_key_exists_fptr(fptr, name) )
-            {
-              /* Read the keyword. */
-              if( fits_read_key(fptr, TSTRING, name, &keyvalue, NULL,
-                                &status) )
-                gal_fits_io_error(status, NULL);
-
-              /* If the value corresponds to any of the user's values for this
-                 keyword, add it to the list of output names. */
-              for(v=values; v!=NULL; v=v->next)
-                {
-                  if( strcmp(v->v, keyvalue)==0 )
-                    { gal_list_str_add(&outlist, f->v, 1); break; }
-                }
-            }
-
-          /* Close the file. */
-          if( fits_close_file(fptr, &status) )
-            gal_fits_io_error(status, NULL);
-        }
-    }
+  /* Clean up and return. */
+  gal_list_str_free(files, 1);
+  gal_list_str_free(values, 1);
+  gal_list_str_free(outlist, 1);
+  return out;
+}
+
+
+
+
+
+static char *
+makeplugin_fits_unique_keyvalues(const char *caller, unsigned int argc,
+                                 char **argv)
+{
+  gal_list_str_t *files=NULL;
+  gal_list_str_t *outlist=NULL;
+  char *name=gal_txt_trim_space(argv[2]);
+  char *out, *hdu=gal_txt_trim_space(argv[1]);
+
+  /* If any of the inputs are empty, then don't bother continuing. */
+  if( makeplugin_goodinput(argv, 3, fits_unique_keyvalues_name)==0 )
+    return NULL;
+
+  /* Extract the components in the arguments with possibly multiple
+     values and find the output files.*/
+  files=gal_list_str_extract(argv[0]);
+  outlist=gal_fits_unique_keyvalues(files, hdu, name);
 
   /* Write the output value. */
   out=gal_list_str_cat(outlist);
 
   /* Clean up and return. */
   gal_list_str_free(files, 1);
-  gal_list_str_free(values, 1);
   gal_list_str_free(outlist, 1);
   return out;
 }
@@ -152,17 +148,29 @@ gnumake_fits_with_keyvalue(const char *caller, unsigned 
int argc,
 
 
 
+
 /* Top-level function (that should have this name). */
 int
 libgnuastro_make_gmk_setup()
 {
   /* Select files, were a certain keyword has a certain value. It takes
-     for arguments:
+     four arguments:
        0. List of files.
        1. HDU (fixed in all files).
        2. Keyword name.
        3. Keyword value(s). */
-  gmk_add_function(with_keyvalue_name, gnumake_fits_with_keyvalue,
+  gmk_add_function(fits_with_keyvalue_name, makeplugin_fits_with_keyvalue,
                    4, 4, GMK_FUNC_DEFAULT);
+
+  /* Return the unique values given to a certain keyword in many FITS
+     files. It takes three arguments.
+       0. List of files.
+       1. HDU (fixed in all files).
+       2. Keyword name. */
+  gmk_add_function(fits_unique_keyvalues_name,
+                   makeplugin_fits_unique_keyvalues,
+                   3, 3, GMK_FUNC_DEFAULT);
+
+  /* Everything is good, return 1 (success). */
   return 1;
 }