[gnuastro-commits] master 7b33afa 043/125: FITS ascii and binary table I/O done with tests

[Mohammad Akhlaghi]

branch: master
commit 7b33afa150109ebaed0dc91e11a94c704a9b9536
Author: Mohammad Akhlaghi <address@hidden>
Commit: Mohammad Akhlaghi <address@hidden>

    FITS ascii and binary table I/O done with tests
    
    The FITS standard doesn't have any code to store `signed char', `unsigned
    short', or `unsigned int' types. So they suggest using special values for
    the `TSCALn' (to scale a column values) and `TZEROn' (to shift the column
    values) keywords to let CFITSIO know that we are dealing with one of these
    types instead of their unsigned/signed counterparts.
    
    While CFITSIO does do this internally, we also need to do this outside
    CFITSIO because the space/type to keep the values must already be
    allocated/known. So `gal_fits_table_info' now reads the `TZEROn' and
    `TSCALn' keywords also and the new function
    `fits_correct_bin_table_int_types' is in charge of checking these two
    values and correcting the type if it is necessary. With this correction
    (and those in the last commit), the example table in
    `tests/table/table.txt' which contains all types and blank values and etc
    can now be read as an ASCII or binary FITS file and converted back into a
    plain text file.
    
    The test table was also corrected to be usable with the default values as
    discussed in the comments above the table. Also, two tests were added to
    first read this table as an ASCII FITS table, then convert that back to a
    plain text table (two similar tests with Binary FITS tables alreay
    existed). All four of these tests now pass (on my system at least).
---
 lib/fits.c                       | 110 +++++++++++++++++++++++++++++++++++++--
 lib/gnuastro/table.h             |   2 +-
 tests/Makefile.am                |  33 ++++++------
 tests/table/fits-ascii-to-txt.sh |  51 ++++++++++++++++++
 tests/table/table.txt            |  39 ++++++++------
 tests/table/txt-to-fits-ascii.sh |  51 ++++++++++++++++++
 6 files changed, 250 insertions(+), 36 deletions(-)

diff --git a/lib/fits.c b/lib/fits.c
index 9088f87..693b410 100644
--- a/lib/fits.c
+++ b/lib/fits.c
@@ -1734,6 +1734,53 @@ set_display_format(char *tdisp, gal_data_t *data, char 
*filename, char *hdu,
 
 
 
+/* The FITS standard for binary tables (not ASCII tables) does not allow
+   unsigned types for short, int and long types, or signed char! So it has
+   `TSCALn' and `TZEROn' to scale the signed types to an unsigned type. It
+   does this internally, but since we need to define our data type and
+   allocate space for it before actually reading the array, it is necessary
+   to do this setting here.  */
+static void
+fits_correct_bin_table_int_types(gal_data_t *allcols, int tfields,
+                                 int *tscal, long long *tzero)
+{
+  size_t i;
+
+  for(i=0;i<tfields;++i)
+    {
+      /* If TSCALn is not 1, the reason for it isn't to use a different
+         signed/unsigned type, so don't change anything. */
+      if(tscal[i]!=1) continue;
+
+      /* For a check
+      printf("Column %zu initial type: %s (s: %d, z: %lld)\n", i+1,
+             gal_data_type_as_string(allcols[i].type, 1), tscal[i], tzero[i]);
+      */
+
+      /* Correct the type based on the initial read type and the value to
+         tzero. If tzero is any other value, then again, its not a type
+         conversion, so just ignore it. */
+      if(allcols[i].type==GAL_DATA_TYPE_UCHAR && tzero[i]==SCHAR_MIN)
+        allcols[i].type = GAL_DATA_TYPE_CHAR;
+
+      else if ( allcols[i].type==GAL_DATA_TYPE_SHORT
+                && tzero[i] == -(long long)SHRT_MIN )
+        allcols[i].type = GAL_DATA_TYPE_USHORT;
+
+      else if (allcols[i].type==GAL_DATA_TYPE_LONG
+               && tzero[i] ==  -(long long)INT_MIN)
+        allcols[i].type = GAL_DATA_TYPE_UINT;
+
+      /* For a check
+      printf("Column %zu corrected type: %s\n", i+1,
+             gal_data_type_as_string(allcols[i].type, 1));
+      */
+    }
+}
+
+
+
+
 
 /* See the descriptions of `gal_table_info'. */
 gal_data_t *
@@ -1745,8 +1792,9 @@ gal_fits_table_info(char *filename, char *hdu, size_t 
*numcols,
   char *tailptr;
   fitsfile *fptr;
   size_t i, index;
+  long long *tzero;
   gal_data_t *allcols;
-  int status=0, datatype;
+  int status=0, datatype, *tscal;
   char keyname[FLEN_KEYWORD]="XXXXXXXXXXXXX", value[FLEN_VALUE];
 
   /* Open the FITS file and get the basic information. */
@@ -1760,6 +1808,20 @@ gal_fits_table_info(char *filename, char *hdu, size_t 
*numcols,
   allcols=gal_data_calloc_dataarray(tfields);
 
 
+  /* See comments of `fits_correct_bin_table_int_types'. Here we are
+     allocating the space to keep these values. */
+  errno=0;
+  tscal=calloc(tfields, sizeof *tscal);
+  if(tscal==NULL)
+    error(EXIT_FAILURE, errno, "%zu bytes for tscal in "
+          "`gal_fits_table_info'", tfields*sizeof *tscal);
+  errno=0;
+  tzero=calloc(tfields, sizeof *tzero);
+  if(tzero==NULL)
+    error(EXIT_FAILURE, errno, "%zu bytes for tzero in "
+          "`gal_fits_table_info'", tfields*sizeof *tzero);
+
+
   /* Read all the keywords one by one and if they match, then put them in
      the correct value. Note that we are starting from keyword 9 because
      according to the FITS standard, the first 8 keys in a FITS table are
@@ -1805,7 +1867,7 @@ gal_fits_table_info(char *filename, char *hdu, size_t 
*numcols,
                   if(*tabletype==GAL_TABLE_TYPE_AFITS)
                     {
                       repeat=strtol(&value[2], &tailptr, 0);
-                      if(tailptr=='\0')
+                      if(*tailptr!='\0')
                         error(EXIT_FAILURE, 0, "%s (hdu: %s): the value to "
                               "keyword `%s' (`%s') is not in `Aw' format "
                               "(for strings) as required by the FITS "
@@ -1816,6 +1878,34 @@ gal_fits_table_info(char *filename, char *hdu, size_t 
*numcols,
             }
         }
 
+      /* COLUMN SCALE FACTOR. */
+      else if(strncmp(keyname, "TSCAL", 5)==0)
+        {
+          index = strtoul(&keyname[5], &tailptr, 10) - 1;
+          if(index<tfields)
+            {
+              tscal[index]=strtol(value, &tailptr, 0);
+              if(*tailptr!='\0')
+                error(EXIT_FAILURE, 0, "%s (hdu: %s): value to %s keyword "
+                      "(`%s') couldn't be read as a number", filename, hdu,
+                      keyname, value);
+            }
+        }
+
+      /* COLUMN ZERO VALUE (for signed/unsigned types). */
+      else if(strncmp(keyname, "TZERO", 5)==0)
+        {
+          index = strtoul(&keyname[5], &tailptr, 10) - 1;
+          if(index<tfields)
+            {
+              tzero[index]=strtoll(value, &tailptr, 0);
+              if(*tailptr!='\0')
+                error(EXIT_FAILURE, 0, "%s (hdu: %s): value to %s keyword "
+                      "(`%s') couldn't be read as a number", filename, hdu,
+                      keyname, value);
+            }
+        }
+
       /* COLUMN NAME. All strings in CFITSIO start and finish with single
          quotation marks, CFITSIO puts them in itsself, so if we don't
          remove them here, we might have duplicates later, its easier to
@@ -1884,8 +1974,15 @@ gal_fits_table_info(char *filename, char *hdu, size_t 
*numcols,
                                  keyname);
             }
         }
+
+      /* Column zero. */
     }
 
+  /* Correct integer types, then free the allocated arrays. */
+  fits_correct_bin_table_int_types(allcols, tfields, tscal, tzero);
+  free(tscal);
+  free(tzero);
+
   /* Close the FITS file and report an error if we had any. */
   fits_close_file(fptr, &status);
   gal_fits_io_error(status, NULL);
@@ -2094,6 +2191,7 @@ fits_write_tnull_tcomm(fitsfile *fptr, gal_data_t *col, 
int tabletype,
   int status=0;
   char *keyname, *bcomment;
 
+  /* Write the NULL value */
   switch(tabletype)
     {
     case GAL_TABLE_TYPE_AFITS:
@@ -2106,11 +2204,15 @@ fits_write_tnull_tcomm(fitsfile *fptr, gal_data_t *col, 
int tabletype,
       break;
 
     case GAL_TABLE_TYPE_BFITS:
+      /* FITS binary tables don't accept NULL values for floating point or
+         string columns. For floating point is must be NaN and for strings
+         it is a blank string. */
       if( col->type!=GAL_DATA_TYPE_FLOAT
-          && col->type!=GAL_DATA_TYPE_DOUBLE )
+          && col->type!=GAL_DATA_TYPE_DOUBLE
+          && col->type!=GAL_DATA_TYPE_STRING )
         {
-          asprintf(&keyname, "TNULL%zu", colnum);
           blank=gal_data_alloc_blank(col->type);
+          asprintf(&keyname, "TNULL%zu", colnum);
           fits_write_key(fptr, gal_fits_type_to_datatype(col->type),
                          keyname, blank, "blank value for this column",
                          &status);
diff --git a/lib/gnuastro/table.h b/lib/gnuastro/table.h
index 1df6e5a..6acc21a 100644
--- a/lib/gnuastro/table.h
+++ b/lib/gnuastro/table.h
@@ -59,7 +59,7 @@ __BEGIN_C_DECLS  /* From C++ preparations */
 #define GAL_TABLE_DEF_STR_WIDTH       6
 #define GAL_TABLE_DEF_INT_WIDTH       6
 #define GAL_TABLE_DEF_LINT_WIDTH      10
-#define GAL_TABLE_DEF_FLT_WIDTH       10
+#define GAL_TABLE_DEF_FLT_WIDTH       12
 #define GAL_TABLE_DEF_DBL_WIDTH       18
 
 #define GAL_TABLE_DEF_INT_PRECISION   0
diff --git a/tests/Makefile.am b/tests/Makefile.am
index e4b3113..b852146 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -65,21 +65,22 @@ multithread_SOURCES = lib/multithread.c
 
 
 # The actual test scripts that are run:
-TESTS = $(check_PROGRAMS) prepconf.sh mkprof/mosaic1.sh mkprof/mosaic2.sh    \
-  mkprof/mosaic3.sh mkprof/mosaic4.sh mkprof/radeccat.sh imgcrop/imgcat.sh   \
-  imgcrop/wcscat.sh imgcrop/xcyc.sh imgcrop/xcycnoblank.sh                   \
-  imgcrop/section.sh imgcrop/radec.sh imgcrop/imgpolygon.sh                  \
-  imgcrop/imgoutpolygon.sh imgcrop/wcspolygon.sh convertt/fitstotxt.sh       \
-  convertt/fitstojpeg.sh convertt/blankch.sh convertt/jpegtotxt.sh           \
-  convertt/fitstojpegcmyk.sh convertt/jpegtofits.sh convertt/fitstopdf.sh    \
-  convolve/spatial.sh convolve/frequency.sh imgwarp/imgwarp_scale.sh         \
-  imgwarp/homographic.sh mknoise/addnoise.sh mkprof/ellipticalmasks.sh       \
-  mkprof/inputascanvas.sh header/write.sh header/print.sh header/update.sh   \
-  header/delete.sh imgstat/basicstats.sh subtractsky/subtractsky.sh          \
-  noisechisel/noisechisel.sh mkcatalog/simple.sh mkcatalog/aperturephot.sh   \
-  arithmetic/snimage.sh arithmetic/onlynumbers.sh arithmetic/where.sh        \
-  arithmetic/or.sh cosmiccal/simpletest.sh table/txt-to-fits-binary.sh       \
-  table/fits-binary-to-txt.sh
+TESTS = $(check_PROGRAMS) prepconf.sh mkprof/mosaic1.sh mkprof/mosaic2.sh  \
+  mkprof/mosaic3.sh mkprof/mosaic4.sh mkprof/radeccat.sh imgcrop/imgcat.sh \
+  imgcrop/wcscat.sh imgcrop/xcyc.sh imgcrop/xcycnoblank.sh                 \
+  imgcrop/section.sh imgcrop/radec.sh imgcrop/imgpolygon.sh                \
+  imgcrop/imgoutpolygon.sh imgcrop/wcspolygon.sh convertt/fitstotxt.sh     \
+  convertt/fitstojpeg.sh convertt/blankch.sh convertt/jpegtotxt.sh         \
+  convertt/fitstojpegcmyk.sh convertt/jpegtofits.sh convertt/fitstopdf.sh  \
+  convolve/spatial.sh convolve/frequency.sh imgwarp/imgwarp_scale.sh       \
+  imgwarp/homographic.sh mknoise/addnoise.sh mkprof/ellipticalmasks.sh     \
+  mkprof/inputascanvas.sh header/write.sh header/print.sh header/update.sh \
+  header/delete.sh imgstat/basicstats.sh subtractsky/subtractsky.sh        \
+  noisechisel/noisechisel.sh mkcatalog/simple.sh mkcatalog/aperturephot.sh \
+  arithmetic/snimage.sh arithmetic/onlynumbers.sh arithmetic/where.sh      \
+  arithmetic/or.sh cosmiccal/simpletest.sh table/txt-to-fits-binary.sh     \
+  table/fits-binary-to-txt.sh table/txt-to-fits-ascii.sh                   \
+  table/fits-ascii-to-txt.sh
 
 
 
@@ -130,6 +131,8 @@ mkprof/mosaic4.sh: prepconf.sh.log
 mkprof/radeccat.sh: prepconf.sh.log
 table/txt-to-fits-binary.sh: prepconf.sh.log
 table/fits-binary-to-txt.sh: table/txt-to-fits-binary.sh.log
+table/txt-to-fits-ascii.sh: prepconf.sh.log
+table/fits-ascii-to-txt.sh: table/txt-to-fits-ascii.sh.log
 imgcrop/imgcat.sh: mkprof/mosaic1.sh.log
 imgcrop/wcscat.sh: mkprof/mosaic1.sh.log mkprof/mosaic2.sh.log     \
                    mkprof/mosaic3.sh.log mkprof/mosaic4.sh.log
diff --git a/tests/table/fits-ascii-to-txt.sh b/tests/table/fits-ascii-to-txt.sh
new file mode 100755
index 0000000..96df459
--- /dev/null
+++ b/tests/table/fits-ascii-to-txt.sh
@@ -0,0 +1,51 @@
+# Convert an ASCII table to a binary table
+#
+# See the Tests subsection of the manual for a complete explanation
+# (in the Installing gnuastro section).
+#
+# Original author:
+#     Mohammad Akhlaghi <address@hidden>
+# Contributing author(s):
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+
+
+
+
+
+# Preliminaries
+# =============
+#
+# Set the variabels (The executable is in the build tree). Do the
+# basic checks to see if the executable is made or if the defaults
+# file exists (basicchecks.sh is in the source tree).
+prog=table
+execname=../bin/$prog/ast$prog
+table=ascii-table.fits
+
+
+
+
+
+# Skip?
+# =====
+#
+# If the dependencies of the test don't exist, then skip it. There are two
+# types of dependencies:
+#
+#   - The executable was not made (for example due to a configure option),
+#
+#   - The input data was not made (for example the test that created the
+#     data file failed).
+if [ ! -f $execname ] || [ ! -f $img ]; then exit 77; fi
+
+
+
+
+
+# Actual test script
+# ==================
+$execname $table --output=from-ascii-table.txt -h1
diff --git a/tests/table/table.txt b/tests/table/table.txt
index 7538dd2..5c5f670 100644
--- a/tests/table/table.txt
+++ b/tests/table/table.txt
@@ -9,13 +9,13 @@
 #  - Columns with missing information
 #  - Some blank values different from the internal blank values.
 
-# Column 10: DOUBLE      [no units, d]  Column with double values
+# Column 10: DOUBLE      [no units, d, 255]  Column with double values
 
-# Column 1:  UCHAR_DATA  [no units, uc, 255]  Column with unsigned char values
-# Column 2:              [no-units, c, -5] Column with char values
+# Column 1:  UCHAR_DATA  [no units, uc, 5]  Column with unsigned char values
+# Column 2:  CHAR        [no-units, c] Column with char values
 # Column 3:  STRING data [, str21, no data]          Column with string values
 # Column 4:  USHORT data [,us] Column with unsigned short values
-# Column 5:              [no units, s]  Column with short values
+# Column 5:  SHORT       [no units, s]  Column with short values
 # Column 7:  LONG        [no units, l]  Column with long values
 # Column 8: LONGLONG    [no units, L]  Column with LONGLONG values
 # Column 9: FLOAT       [no units, f,-99]  Column with float values
@@ -23,17 +23,24 @@
 # Column 6:  UINT [,ui]
 
 
+# IMPORTANT NOTE FOR FITS ASCII tables: CFITSIO will give its error
+# 412 (data type conversion overflow) when the number cannot be
+# printed in the provided space (with full precision). So make sure
+# that the full-integer part of the number has less characters than
+# the `width - precision - 1' (when everything is default, you can get
+# these values from `lib/gnuastro/table.h').
 
 
-1    -1    The vertical lines     3455    -3455  1  -1   9     2.3e4   
8.39133434352355
-2    -2    are only for easy      3466    -3466  2  -2   10    2300    
893.3497e50
-3    -3    identification of the  3467    -3467  3  -3   12    2.4     
93487.3473234343
-4    -4    limits of this string  3468    -3468  4  -4   800   894.83  8.2e8
-5    -5    |column. Characters |  3469    -3469  5  -5   8923  99.232  
-7.3238473343e4
-6    -6    |beyond this will be|  20821  -20821  6  -6   9823  8874.2  
769387424234234
-7    -7    read as a number.      20822  -20822  7  -7   7232  1E10    
8.3343e-5
-8    -8    no data                20823  -20823  8  -8   921   -8.4e2  
934892734.82
-9    -9    address@hidden@    60983  -25722  9  -9   8127  -99.0   
3474924.489252
-10  -10    Characters%^&*()~      62182  -30100 10  -10  8287  7.3e-9  
-3467.3432e10
-11  -127   no data                65500  -32700 11  -11  999   8.73e2  
897234.34897
-255 -12    -- Last Line :-) --    65501  -32701 12  -12  8282  892.23  
89750298.3653
+
+1    -1    The vertical lines     3455  -3455  1  -1   9     1874.103872  
8.391334343995    1
+2    -2    are only for easy      3466      -3466  2  -2   10    123.93373    
893.3497e5        2
+3    -3    visual identification  3467 -3467  3  -3   12    -0.737648    nan   
            3
+4    -4    of the limits of this  3468  -3468  4  -4   800   78.983       
8.2328438e8       4
+5    -5    |string column.     |  3469       -3469  5  -5   8923  -99       
-7.32384e4        5
+6    -6    |characters beyond  |  20821  -20821  6  -6   9823  -99     nan     
          6
+7    -7    the last one will be   20822  -20822  7  -7   7232  9999   
8.3343e-5         7
+8    -8    read as a number.   20823  -20823  8  -8   921   2.3     
934892734.82      8
+9    -9    address@hidden@    60983  -25722  9  -9   8127  -99          
3474924.489252    9
+10  -10    Characters%^&*()~           62182  -30100 10  -10  8287  7.3e-4     
  -3467.3432e5      10
+11  -12 no data                65500  -32700 11  -11  999    8.73E2       nan  
             11
+255 -12             -- Last Line :-) --    65501  -32701 12  -12  8282  892.23 
      8975.3653      12
diff --git a/tests/table/txt-to-fits-ascii.sh b/tests/table/txt-to-fits-ascii.sh
new file mode 100755
index 0000000..8d69c8c
--- /dev/null
+++ b/tests/table/txt-to-fits-ascii.sh
@@ -0,0 +1,51 @@
+# Convert an ASCII table to a binary table
+#
+# See the Tests subsection of the manual for a complete explanation
+# (in the Installing gnuastro section).
+#
+# Original author:
+#     Mohammad Akhlaghi <address@hidden>
+# Contributing author(s):
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+
+
+
+
+
+# Preliminaries
+# =============
+#
+# Set the variabels (The executable is in the build tree). Do the
+# basic checks to see if the executable is made or if the defaults
+# file exists (basicchecks.sh is in the source tree).
+prog=table
+execname=../bin/$prog/ast$prog
+table=$topsrc/tests/$prog/table.txt
+
+
+
+
+
+# Skip?
+# =====
+#
+# If the dependencies of the test don't exist, then skip it. There are two
+# types of dependencies:
+#
+#   - The executable was not made (for example due to a configure option),
+#
+#   - The input data was not made (for example the test that created the
+#     data file failed).
+if [ ! -f $execname ] || [ ! -f $img ]; then exit 77; fi
+
+
+
+
+
+# Actual test script
+# ==================
+$execname $table --output=ascii-table.fits --tabletype=fits-ascii