[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[gnuastro-commits] master 596b88f: Stdin in ConvertType, Match, MakeProf
|
From: |
Mohammad Akhlaghi |
|
Subject: |
[gnuastro-commits] master 596b88f: Stdin in ConvertType, Match, MakeProfiles, Table, and book |
|
Date: |
Mon, 5 Nov 2018 08:20:42 -0500 (EST) |
branch: master
commit 596b88f44b6fbf3c49974a990d7c5be658bd8967
Author: Mohammad Akhlaghi <address@hidden>
Commit: Mohammad Akhlaghi <address@hidden>
Stdin in ConvertType, Match, MakeProfiles, Table, and book
The low-level structure to read from the standard input is now also
implemented in ConvertType, Match, MakeProfiles and Table. A "Standard
input" section has also been added to the book and it is referred to in the
programs that use it.
---
NEWS | 3 +
bin/convertt/ui.c | 37 +-
bin/match/main.h | 1 +
bin/match/match.c | 10 +-
bin/match/ui.c | 67 +-
bin/mkprof/ui.c | 23 +-
bin/table/ui.c | 15 +-
doc/gnuastro.texi | 2466 +++++++++++++++++++++------------------
lib/gnuastro-internal/options.h | 3 +
lib/options.c | 34 +-
10 files changed, 1444 insertions(+), 1215 deletions(-)
diff --git a/NEWS b/NEWS
index f097ddb..1d59fe6 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,9 @@ GNU Astronomy Utilities NEWS -*-
outline -*-
- Input files and parameters written as keywords in the first extension
of output when it is FITS. This is only relevant for some programs,
(for example not the Fits or Table programs).
+ - Standard input (for example from pipes) is now available to feed input
+ to all programs that accept plain text input (ConvertType, Match,
+ MakeProfiles, Statistics, Table).
Arithmetic:
--onedasimage: write output as an image if it has one dimension, not table.
diff --git a/bin/convertt/ui.c b/bin/convertt/ui.c
index 97137ae..47b2194 100644
--- a/bin/convertt/ui.c
+++ b/bin/convertt/ui.c
@@ -146,7 +146,6 @@ ui_initialize_options(struct converttparams *p,
case GAL_OPTIONS_KEY_SEARCHIN:
case GAL_OPTIONS_KEY_IGNORECASE:
case GAL_OPTIONS_KEY_TABLEFORMAT:
- case GAL_OPTIONS_KEY_STDINTIMEOUT:
cp->coptions[i].flags=OPTION_HIDDEN;
break;
}
@@ -272,14 +271,11 @@ ui_read_check_only_options(struct converttparams *p)
static void
ui_check_options_and_arguments(struct converttparams *p)
{
- /* Check if there was any inputs. */
- if(p->inputnames==NULL)
- error(EXIT_FAILURE, 0, "no input files given");
-
- /* Reverse the `inputnames' linked list, note that the `hdu' linked list
- was reversed during option parsing.*/
+ /* Reverse the `inputnames' linked list if it was given (recall that we
+ also accept input from the standard input). Note that the `hdu' linked
+ list was reversed during option parsing, so we don't need to do it
+ here any more. */
gal_list_str_reverse(&p->inputnames);
-
}
@@ -397,11 +393,27 @@ ui_make_channels_ll(struct converttparams *p)
{
char *hdu=NULL;
gal_data_t *data;
- gal_list_str_t *name;
size_t dsize=0, dirnum;
+ gal_list_str_t *name, *lines;
- /* Go through the input files and add the channel(s). */
+ /* Initialize the counting of channels. */
p->numch=0;
+
+ /* If any standard input is provided, we want to process that first. Note
+ that since other input arguments are also allowed (as other channels),
+ we'll need to process the standard input independently first, then go
+ onto the possible list of other files.*/
+ lines=gal_txt_stdin_read(p->cp.stdintimeout);
+ if(lines)
+ {
+ data=gal_txt_image_read(NULL, lines, p->cp.minmapsize);
+ gal_list_data_add(&p->chll, data);
+ gal_list_str_free(lines, 1);
+ ++p->numch;
+ }
+
+
+ /* Go through the input files and add the channel(s). */
for(name=p->inputnames; name!=NULL; name=name->next)
{
/* Check if p->numch has not exceeded 4. */
@@ -499,6 +511,9 @@ ui_make_channels_ll(struct converttparams *p)
}
}
+ /* If there weren't any channels, abort with an error. */
+ if(p->numch==0)
+ error(EXIT_FAILURE, 0, gal_options_stdin_error(p->cp.stdintimeout, 0));
/* Reverse the list of channels into the input order. */
gal_list_data_reverse(&p->chll);
@@ -617,7 +632,7 @@ void
ui_add_dot_use_automatic_output(struct converttparams *p)
{
gal_list_str_t *stll;
- char *tmp, *firstname="output.txt", *suffix=p->cp.output;
+ char *tmp, *firstname="converttype.txt", *suffix=p->cp.output;
/* Find the first non-blank file name in the input(s). */
for(stll=p->inputnames; stll!=NULL; stll=stll->next)
diff --git a/bin/match/main.h b/bin/match/main.h
index 87664a1..49ddf93 100644
--- a/bin/match/main.h
+++ b/bin/match/main.h
@@ -69,6 +69,7 @@ struct matchparams
char *logname; /* Name of log file. */
char *out1name; /* Name of first matched output. */
char *out2name; /* Name of second matched output. */
+ gal_list_str_t *stdinlines; /* Lines given by Standard input. */
/* Output: */
time_t rawtime; /* Starting time of the program. */
diff --git a/bin/match/match.c b/bin/match/match.c
index c1461f7..74d3ef4 100644
--- a/bin/match/match.c
+++ b/bin/match/match.c
@@ -66,8 +66,9 @@ match_catalog_read_write_all(struct matchparams *p, size_t
*permutation,
__func__, "numcolmatch");
/* Read the full table. */
- cat=gal_table_read(filename, hdu, NULL, cols, p->cp.searchin,
- p->cp.ignorecase, p->cp.minmapsize, *numcolmatch);
+ cat=gal_table_read(filename, hdu, filename ? NULL : p->stdinlines, cols,
+ p->cp.searchin, p->cp.ignorecase, p->cp.minmapsize,
+ *numcolmatch);
origsize=cat->size;
@@ -316,7 +317,10 @@ match(struct matchparams *p)
the output. */
if(gal_fits_name_is_fits(p->out1name))
{
- gal_fits_key_write_filename("input1", p->input1name, &p->cp.okeys, 1);
+ gal_fits_key_write_filename("input1", ( p->input1name
+ ? p->input1name
+ : "Standard input" ),
+ &p->cp.okeys, 1);
gal_fits_key_write_filename("input2", p->input2name, &p->cp.okeys, 1);
gal_fits_key_write_config(&p->cp.okeys, "Match configuration",
"MATCH-CONFIG", p->out1name, "0");
diff --git a/bin/match/ui.c b/bin/match/ui.c
index 1d2ad5b..c7a81e6 100644
--- a/bin/match/ui.c
+++ b/bin/match/ui.c
@@ -170,7 +170,7 @@ parse_opt(int key, char *arg, struct argp_state *state)
{
if(p->input2name)
argp_error(state, "only two arguments (input files) should be "
- "given");
+ "given, not any more");
else
p->input2name=arg;
}
@@ -228,21 +228,34 @@ ui_read_check_only_options(struct matchparams *p)
static void
ui_check_options_and_arguments(struct matchparams *p)
{
- /* Make sure two input file names were given and if they a FITS file,
- that a HDU is also given for each. */
+ /* The first input might come from the standard input, in that case, the
+ second input's name is stored in `input1name' and `input2name' will be
+ left to NULL. So we'll first correct for this. */
if(p->input1name)
{
- if( gal_fits_name_is_fits(p->input1name) && p->cp.hdu==NULL )
- error(EXIT_FAILURE, 0, "no HDU for first input. When the input is "
- "a FITS file, a HDU must also be specified, you can use the "
- "`--hdu' (`-h') option and give it the HDU number (starting "
- "from zero), extension name, or anything acceptable by "
- "CFITSIO");
+ if(p->input2name==NULL)
+ {
+ p->input2name=p->input1name;
+ p->input1name=NULL;
+ }
}
else
- error(EXIT_FAILURE, 0, "no input file is specified: two inputs are "
- "necessary");
-
+ error(EXIT_FAILURE, 0, "no inputs! two inputs are necessary. The first "
+ "can be from the standard input (e.g., a pipe)");
+
+
+ /* If the first input is a FITS file, make sure its HDU is also given. */
+ if( p->input1name
+ && gal_fits_name_is_fits(p->input1name)
+ && p->cp.hdu==NULL )
+ error(EXIT_FAILURE, 0, "no HDU for first input. When the input is "
+ "a FITS file, a HDU must also be specified, you can use the "
+ "`--hdu' (`-h') option and give it the HDU number (starting "
+ "from zero), extension name, or anything acceptable by "
+ "CFITSIO");
+
+ /* If the second input is a FITS file, make sure its HDU is also
+ given. */
if(p->input2name)
{
if( gal_fits_name_is_fits(p->input2name) && p->hdu2==NULL )
@@ -283,7 +296,7 @@ ui_set_mode(struct matchparams *p)
{
/* Check if we are in image or catalog mode. We will base the mode on the
first input, then check with the second. */
- if( gal_fits_name_is_fits(p->input1name) )
+ if( p->input1name && gal_fits_name_is_fits(p->input1name) )
p->mode = ( (gal_fits_hdu_format(p->input1name, p->cp.hdu) == IMAGE_HDU)
? MATCH_MODE_WCS
: MATCH_MODE_CATALOG );
@@ -404,9 +417,16 @@ ui_read_columns_to_double(struct matchparams *p, char
*filename, char *hdu,
"Please give more specific values to `--ccol1' (column "
"numberes are the only identifiers guaranteed to be unique).";
- /* Read the columns. */
- tout=gal_table_read(filename, hdu, NULL, cols, cp->searchin,
- cp->ignorecase, cp->minmapsize, NULL);
+ /* Read the columns. Note that the first input's name can be NULL (it the
+ user intended to use the standrad input). Also note that this function
+ is called more than one time, so if the Standard input is already read
+ once, we don't want to write a blank list over it (the Standard input
+ will be empty after being read). */
+ if(p->stdinlines==NULL)
+ p->stdinlines=gal_options_check_stdin(filename, p->cp.stdintimeout);
+ tout=gal_table_read(filename, hdu, filename ? NULL : p->stdinlines,
+ cols, cp->searchin, cp->ignorecase, cp->minmapsize,
+ NULL);
/* A small sanity check. */
if(gal_list_data_number(tout)!=numcols)
@@ -563,6 +583,7 @@ ui_preparations_out_name(struct matchparams *p)
{
/* To temporarily keep the original value. */
uint8_t keepinputdir_orig;
+ char *refname = p->input1name ? p->input1name : p->input2name;
/* Set the output file(s) name(s). */
if(p->logasoutput)
@@ -572,10 +593,10 @@ ui_preparations_out_name(struct matchparams *p)
else
{
if(p->cp.tableformat==GAL_TABLE_FORMAT_TXT)
- p->logname=gal_checkset_automatic_output(&p->cp, p->input1name,
+ p->logname=gal_checkset_automatic_output(&p->cp, refname,
"_matched.txt");
else
- p->logname=gal_checkset_automatic_output(&p->cp, p->input1name,
+ p->logname=gal_checkset_automatic_output(&p->cp, refname,
"_matched.fits");
}
@@ -595,7 +616,7 @@ ui_preparations_out_name(struct matchparams *p)
gal_checkset_allocate_copy(p->cp.output, &p->out1name);
else
p->out1name = gal_checkset_automatic_output(&p->cp,
- p->input1name, ( p->cp.tableformat==GAL_TABLE_FORMAT_TXT
+ refname, ( p->cp.tableformat==GAL_TABLE_FORMAT_TXT
? "_matched.txt" : "_matched.fits") );
gal_checkset_writable_remove(p->out1name, 0, p->cp.dontdelete);
}
@@ -632,8 +653,7 @@ ui_preparations_out_name(struct matchparams *p)
{
if(p->cp.tableformat==GAL_TABLE_FORMAT_TXT)
{
- p->out1name=gal_checkset_automatic_output(&p->cp,
- p->input1name,
+ p->out1name=gal_checkset_automatic_output(&p->cp, refname,
"_matched_1.txt");
p->out2name=gal_checkset_automatic_output(&p->cp,
p->input2name,
@@ -641,8 +661,7 @@ ui_preparations_out_name(struct matchparams *p)
}
else
{
- p->out1name=gal_checkset_automatic_output(&p->cp,
- p->input1name,
+ p->out1name=gal_checkset_automatic_output(&p->cp, refname,
"_matched.fits");
gal_checkset_allocate_copy(p->out1name, &p->out2name);
}
@@ -765,7 +784,6 @@ ui_read_check_inputs_setup(int argc, char *argv[], struct
matchparams *p)
keywords to write in output later. */
if(gal_fits_name_is_fits(p->out1name))
gal_options_as_fits_keywords(&p->cp);
-
}
@@ -800,6 +818,7 @@ ui_free_report(struct matchparams *p, struct timeval *t1)
free(p->cp.output);
gal_data_free(p->ccol1);
gal_data_free(p->ccol2);
+ gal_list_str_free(p->stdinlines, 1);
/* Print the final message.
if(!p->cp.quiet)
diff --git a/bin/mkprof/ui.c b/bin/mkprof/ui.c
index 159f21d..4239de6 100644
--- a/bin/mkprof/ui.c
+++ b/bin/mkprof/ui.c
@@ -202,7 +202,6 @@ ui_initialize_options(struct mkprofparams *p,
break;
case GAL_OPTIONS_KEY_TABLEFORMAT:
- case GAL_OPTIONS_KEY_STDINTIMEOUT:
cp->coptions[i].flags=OPTION_HIDDEN;
break;
@@ -530,10 +529,6 @@ ui_check_options_and_arguments(struct mkprofparams *p)
error(EXIT_FAILURE, 0, "no `hdu' specified for the input FITS "
"table '%s', to ", p->catname);
}
- else
- error(EXIT_FAILURE, 0, "no input catalog provided. To build "
- "profiles, you need to give a catalog/table containing "
- "the information of the profiles");
}
@@ -555,7 +550,10 @@ ui_check_options_and_arguments(struct mkprofparams *p)
{
gal_checkset_allocate_copy(p->cp.output, &p->outdir);
gal_checkset_check_dir_write_add_slash(&p->outdir);
- tmpname=gal_checkset_automatic_output(&p->cp, p->catname, ".fits");
+ tmpname=gal_checkset_automatic_output(&p->cp,
+ ( p->catname
+ ? p->catname
+ : "makeprofiles" ), ".fits");
p->mergedimgname=gal_checkset_malloc_cat(p->outdir, tmpname);
free(tmpname);
}
@@ -597,8 +595,8 @@ ui_read_cols(struct mkprofparams *p)
int checkblank;
size_t i, counter=0;
char *colname=NULL, **strarr;
- gal_list_str_t *colstrs=NULL, *ccol;
gal_data_t *cols, *tmp, *corrtype=NULL;
+ gal_list_str_t *ccol, *lines, *colstrs=NULL;
/* The coordinate columns are a linked list of strings. */
ccol=p->ccol;
@@ -622,8 +620,17 @@ ui_read_cols(struct mkprofparams *p)
gal_list_str_reverse(&colstrs);
/* Read the desired columns from the file. */
- cols=gal_table_read(p->catname, p->cp.hdu, NULL, colstrs, p->cp.searchin,
+ lines=gal_options_check_stdin(p->catname, p->cp.stdintimeout);
+ cols=gal_table_read(p->catname, p->cp.hdu, lines, colstrs, p->cp.searchin,
p->cp.ignorecase, p->cp.minmapsize, NULL);
+ gal_list_str_free(lines, 1);
+
+ /* The name of the input catalog is only for informative steps from now
+ on (we won't be dealing with the actual file any more). So if the
+ standard input was used (therefore `catname==NULL', set it to
+ `stdin'). */
+ if(p->catname==NULL)
+ gal_checkset_allocate_copy("standard-input", &p->catname);
/* Set the number of objects. */
p->num=cols->size;
diff --git a/bin/table/ui.c b/bin/table/ui.c
index cf219f1..b94cc96 100644
--- a/bin/table/ui.c
+++ b/bin/table/ui.c
@@ -129,7 +129,6 @@ ui_initialize_options(struct tableparams *p,
/* Options to ignore. */
case GAL_OPTIONS_KEY_TYPE:
- case GAL_OPTIONS_KEY_STDINTIMEOUT:
cp->coptions[i].flags=OPTION_HIDDEN;
break;
}
@@ -243,8 +242,6 @@ ui_check_options_and_arguments(struct tableparams *p)
"zero), extension name, or anything acceptable by CFITSIO");
}
- else
- error(EXIT_FAILURE, 0, "no input file is specified");
}
@@ -275,11 +272,15 @@ ui_print_info_exit(struct tableparams *p)
char *tmp;
int tableformat;
gal_data_t *allcols;
+ gal_list_str_t *lines;
size_t i, numcols, numrows;
/* Read the table information for the number of columns and rows. */
- allcols=gal_table_info(p->filename, p->cp.hdu, NULL, &numcols,
+ lines=gal_options_check_stdin(p->filename, p->cp.stdintimeout);
+ allcols=gal_table_info(p->filename, p->cp.hdu, lines, &numcols,
&numrows, &tableformat);
+ if(p->filename==NULL) p->filename="Standard-input";
+ gal_list_str_free(lines, 1);
/* If there was no actual data in the file, then inform the user */
if(allcols==NULL)
@@ -361,6 +362,7 @@ ui_columns_prepare(struct tableparams *p)
static void
ui_preparations(struct tableparams *p)
{
+ gal_list_str_t *lines;
struct gal_options_common_params *cp=&p->cp;
/* If there were no columns specified or the user has asked for
@@ -372,9 +374,12 @@ ui_preparations(struct tableparams *p)
ui_columns_prepare(p);
/* Read in the table columns. */
- p->table=gal_table_read(p->filename, cp->hdu, NULL, p->columns,
+ lines=gal_options_check_stdin(p->filename, p->cp.stdintimeout);
+ p->table=gal_table_read(p->filename, cp->hdu, lines, p->columns,
cp->searchin, cp->ignorecase, cp->minmapsize,
NULL);
+ if(p->filename==NULL) p->filename="Standard-input";
+ gal_list_str_free(lines, 1);
/* If there was no actual data in the file, then inform the user and
abort. */
diff --git a/doc/gnuastro.texi b/doc/gnuastro.texi
index b235d05..acf71a4 100644
--- a/doc/gnuastro.texi
+++ b/doc/gnuastro.texi
@@ -291,18 +291,19 @@ Common program behavior
* Command-line:: How to use the command-line.
* Configuration files:: Values for unspecified variables.
+* Getting help:: Getting more information on the go.
* Multi-threaded operations:: How threads are managed in Gnuastro.
* Numeric data types:: Different types and how to specify them.
* Tables:: Recognized table formats.
* Tessellation:: Tile the dataset into non-overlapping bins.
-* Getting help:: Getting more information on the go.
* Automatic output:: About automatic output names.
* Output FITS files:: Common properties when outputs are FITS.
Command-line
-* Arguments and options:: Basics of options and arguments.
-* Common options:: Common options to all Gnuastro programs.
+* Arguments and options:: Different ways to specify inputs and
configuration.
+* Common options:: Options that are shared between all programs.
+* Standard input:: Using output of another program as input.
Arguments and options
@@ -322,6 +323,14 @@ Configuration files
* Current directory and User wide:: Local and user configuration files.
* System wide:: System wide configuration files.
+Getting help
+
+* --usage:: View option names and value formats.
+* --help:: List all options with description.
+* Man pages:: Man pages generated from --help.
+* Info:: View complete book in terminal.
+* help-gnuastro mailing list:: Contacting experienced users.
+
Multi-threaded operations
* A note on threads:: Caution and suggestion on using threads.
@@ -337,14 +346,6 @@ Recognized table formats
* Gnuastro text table format:: Reading plain text tables
-Getting help
-
-* --usage:: View option names and value formats.
-* --help:: List all options with description.
-* Man pages:: Man pages generated from --help.
-* Info:: View complete book in terminal.
-* help-gnuastro mailing list:: Contacting experienced users.
-
Data containers
* Fits:: View and manipulate extensions and keywords.
@@ -7130,28 +7131,60 @@ If your problem was not listed above, please file a bug
report
@chapter Common program behavior
All the programs in Gnuastro share a set of common behavior mainly to do
-with user interaction to facilitate their usage. The most basic is how you
-can configure each program to do what you want: define the input, change
-parameter/option values, or identify the output. All Gnuastro programs can
-also read your desired configuration from pre-defined or user-specified
-files so you don't have to specify all the (sometimes numerous) parameters
-on the command-line each time you run a program. These files define the
-``default'' program behavior in each directory, for each user, or on each
-system. In other cases, some programs can greatly benefit from the many
-threads available in modern CPUs, so here we'll also discuss how you can
-get the most out of your hardware. Among some other issues, we will also
-discuss how you can get immediate and distraction-free (without taking your
-hands off the keyboard!) help, or access to this whole book, on the
-command-line.
+with user interaction to facilitate their usage and development. This
+includes how to feed input datasets into the programs, how to configure
+them, specifying the outputs, numerical data types, treating columns of
+information in tables and etc. This chapter is devoted to describing this
+common behavior in all programs. Because the behaviors discussed here are
+common to several programs, they are not repeated in each program's
+description.
+
+In @ref{Command-line}, a very general description of running the programs
+on the command-line is discussed, like difference between arguments and
+options, as well as options that are common/shared between all
+programs. None of Gnuastro's programs keep any internal configuration value
+(values for their different operational steps), they read their
+configuration primarily from the command-line, then from specific files in
+directory, user, or system-wide settings. Using these configuration files
+can greatly help reproducible and robust usage of Gnuastro, see
address@hidden files} for more.
+
+It is not possible to always have the different options and configurations
+of each program on the top of your head. It is very natural to forget the
+options of a program, their current default values, or how it should be run
+and what it did. Gnuastro's programs have multiple ways to help you refresh
+your memory in multiple levels (just an option name, a short description,
+or fast access to the relevant section of the manual. See @ref{Getting
+help} for more for more on benefiting from this very convenient feature.
+
+Many of the programs use the multi-threaded character of modern CPUs, in
address@hidden operations} we'll discuss how you can configure this
+behavior, along with some tips on making best use of them. In @ref{Numeric
+data types}, we'll review the various types to store numbers in your
+datasets: setting the proper type for the usage address@hidden
+example if the values in your dataset can only be integers between 0 or
+65000, store them in a unsigned 16-bit type, not 64-bit floating point type
+(which is the default in most systems). It takes four times less space and
+is much faster to process.} can grealy improve the file size and also speed
+of reading, writing or processing them.
+
+We'll then look into the recognized table formats in @ref{Tables} and how
+large datasets are broken into tiles, or mesh grid in
address@hidden Finally, we'll take a look at the behavior regarding
+output files: @ref{Automatic output} discribes how the programs set a
+default name for their output when you don't give one explicitly (using
address@hidden). When the output is a FITS file, all the programs also
+store some very useful information in the header that is discussed in
address@hidden FITS files}.
@menu
* Command-line:: How to use the command-line.
* Configuration files:: Values for unspecified variables.
+* Getting help:: Getting more information on the go.
* Multi-threaded operations:: How threads are managed in Gnuastro.
* Numeric data types:: Different types and how to specify them.
* Tables:: Recognized table formats.
* Tessellation:: Tile the dataset into non-overlapping bins.
-* Getting help:: Getting more information on the go.
* Automatic output:: About automatic output names.
* Output FITS files:: Common properties when outputs are FITS.
@end menu
@@ -7159,42 +7192,40 @@ command-line.
@node Command-line, Configuration files, Common program behavior, Common
program behavior
@section Command-line
-All the programs in Gnuastro are customized through the standard GNU style
-command-line options. Thus, we'll start by defining this general style that
-is very common in many command-line tools on Unix-like operating
-systems. Finally, the options that are common to all the programs in
-Gnuastro are discussed.
-
address@hidden Metacharacters
address@hidden Token separation
address@hidden Command-line token separation
address@hidden Separating tokens on the command-line
-The command-line text that you type is passed onto the shell (or program
-managing the command-line) as a string of characters. See the ``Invoking
-ProgramName'' sections in this manual for some examples of commands with
-each program, for example @ref{Invoking asttable}. That string is then
-broken up into separate @emph{tokens} or @emph{words} by any
address@hidden (like space, tab, @command{|}, @command{>} or
address@hidden;}) that might exist in the text. To learn more, please see the
-GNU Bash manual, for the complete list of meta-characters and other GNU
-Bash definitions (GNU Bash is the most common shell program). Its ``Shell
-Operation'' section has a short summary of the steps the shell takes before
-passing the commands to the program you called.
+Gnuastro's programs are customized through the standard Unix-like
+command-line environment and GNU style command-line options. Both are very
+common in many Unix-like operating system programs. In @ref{Arguments and
+options} we'll start with the difference between arguments and options and
+elaborate on the GNU style of options. Afterwards, in @ref{Common options},
+we'll go into the detailed list of all the options that are common to all
+the programs in Gnuastro.
@menu
-* Arguments and options:: Basics of options and arguments.
-* Common options:: Common options to all Gnuastro programs.
+* Arguments and options:: Different ways to specify inputs and
configuration.
+* Common options:: Options that are shared between all programs.
+* Standard input:: Using output of another program as input.
@end menu
@node Arguments and options, Common options, Command-line, Command-line
@subsection Arguments and options
address@hidden Shell
@cindex Options to programs
@cindex Command-line options
@cindex Arguments to programs
@cindex Command-line arguments
-On the command-line, the first thing you usually enter is the name of the
-program you want to run. After that, you can specify two types of input:
+When you type a command on the command-line, it is passed onto the shell (a
+generic name for the program that manages the command-line) as a string of
+characters. As an example, see the ``Invoking ProgramName'' sections in
+this manual for some examples of commands with each program, like
address@hidden asttable}, @ref{Invoking astfits}, or @ref{Invoking
+aststatistics}.
+
+The shell then brokes up your string into separate @emph{tokens} or
address@hidden using any @emph{metacharacters} (like white-space, tab,
address@hidden|}, @command{>} or @command{;}) that are in the string. On the
+command-line, the first thing you usually enter is the name of the program
+you want to run. After that, you can specify two types of tokens:
@emph{arguments} and @emph{options}. In the GNU-style, arguments are those
tokens that are not preceded by any hyphens (@command{-}, see
@ref{Arguments}). Here is one example:
@@ -7203,29 +7234,32 @@ tokens that are not preceded by any hyphens
(@command{-}, see
$ astcrop --center=53.162551,-27.789676 -w10/3600 --mode=wcs udf.fits
@end example
-In this example, the argument is @file{udf.fits}. Arguments are most
-commonly the input file names containing your data. Options start with one
-or two hyphens, followed by an identifier for the option (the option's
-name) and its value (see @ref{Options}). Through options you tell the
-program how to interpret the data. In this example, we are running
address@hidden to crop a region of width 10 arc-seconds centered at the given
-RA and Dec from the input Hubble Ultra-Deep Field (UDF) FITS image. So
-options come with an identifier (the option name which is separate from
-their value).
+In the example above, we are running @ref{Crop} to crop a region of width
+10 arc-seconds centered at the given RA and Dec from the input Hubble
+Ultra-Deep Field (UDF) FITS image. Here, the argument is
address@hidden Arguments are most commonly the input file names
+containing your data. Options start with one or two hyphens, followed by an
+identifier for the option (the option's name, for example,
address@hidden, @option{-w}, @option{--mode} in the example above) and
+its value (anything after the option name, or the optional @key{=}
+character). Through options you can configure how the program runs
+(interprets the data you provided).
@vindex --help
@vindex --usage
@cindex Mandatory arguments
-Arguments can be both mandatory and optional and unlike options they don't
-have any identifiers (or help from you). Hence, their order might also
-matter (for example in @command{cp} which is used for copying one file to
-another location). The outputs of @option{--usage} and @option{--help}
+Arguments can be mandatory and optional and unlike options, they don't have
+any identifiers. Hence, when there multiple arguments, their order might
+also matter (for example in @command{cp} which is used for copying one file
+to another location). The outputs of @option{--usage} and @option{--help}
shows which arguments are optional and which are mandatory, see
address@hidden As their name suggests, @emph{options} on the command-line
-can be considered to be optional and most of the time, you don't have to
-worry about what order you specify them in. When the order does matter, or
-the option can be invoked multiple times, it is explicitly mentioned in the
-``Invoking ProgramName'' section of each program.
address@hidden
+
+As their name suggests, @emph{options} can be considered to be optional and
+most of the time, you don't have to worry about what order you specify them
+in. When the order does matter, or the option can be invoked multiple
+times, it is explicitly mentioned in the ``Invoking ProgramName'' section
+of each program (this is a very important aspect of an option).
@cindex Metacharacters on the command-line
In case your arguments or option values contain any of the shell's
@@ -7241,17 +7275,16 @@ For example, let's say you want to specify the header
data unit (HDU) of
your FITS file using a complex expression like address@hidden; images(exposure
> 100)}'. If you simply add these after the @option{--hdu} (@option{-h})
option, the programs in Gnuastro will read the value to the HDU option as
address@hidden' and run. Then, Bash will attempt to run a separate command
address@hidden(exposure > 100)}' and complain about a syntax error. This
-is because the semicolon (@command{;}) is an `end of command' character in
-the shell. To solve this problem you can simply put double quotes around
-the whole string you want to pass to @option{--hdu} as seen below:
address@hidden' and run. Then, the shell will attempt to run a separate
+command address@hidden(exposure > 100)}' and complain about a syntax
+error. This is because the semicolon (@command{;}) is an `end of command'
+character in the shell. To solve this problem you can simply put double
+quotes around the whole string you want to pass to @option{--hdu} as seen
+below:
@example
-$ astcrop --hdu="3; images(exposure > 100)" FITSimage.fits
+$ astcrop --hdu="3; images(exposure > 100)" image.fits
@end example
-Alternatively you can put a address@hidden' before every meta-character in
-this string, but try doing that, and probably you will agree that the
-double quotes are much more easier, elegant and readable.
+
@@ -7465,7 +7498,7 @@ others 1. You can assume by default that counting starts
from 1, if it
starts from 0 for a special option, it will be explicitly mentioned.
@end cartouche
address@hidden Common options, , Arguments and options, Command-line
address@hidden Common options, Standard input, Arguments and options,
Command-line
@subsection Common options
@cindex Options common to all programs
@@ -7497,11 +7530,12 @@ programs.
@cindex Standard input
@item --stdintimeout
Number of micro-seconds to wait for writing/typing in the @emph{first line}
-of standard input from the command-line. This is only relevant for programs
-that also accept input from the standard input, @emph{and} you want to
-manually write/type the contents on the terminal. When the standard input
-is already connected to a pipe (output of another program), there won't be
-any waiting (hence no timeout, thus making this option redundant).
+of standard input from the command-line (see @ref{Standard input}). This is
+only relevant for programs that also accept input from the standard input,
address@hidden you want to manually write/type the contents on the
+terminal. When the standard input is already connected to a pipe (output of
+another program), there won't be any waiting (hence no timeout, thus making
+this option redundant).
If the first line-break (for example with the @key{ENTER} key) is not
provided before the timeout, the program will abort with an error that no
@@ -7981,8 +8015,103 @@ others, this option will be ignored.
address@hidden Standard input, , Common options, Command-line
address@hidden Standard input
+
address@hidden Standard input
address@hidden Stream: standard input
+The most common way to feed the primary/first input dataset into a program
+is to give its filename as an argument (discussed in @ref{Arguments}). When
+you want to run a series of programs in sequence, this means that each will
+have to keep the output of each program in a separate file and re-type that
+file's name in the next command. This can be very slow and frustrating
+(mis-typing a file's name).
+
address@hidden Standard output stream
address@hidden Stream: standard output
+To solve the problem, the founders of Unix defined pipes to directly feed
+the output of one program (its ``Standard output'' stream) into the
+``standard input'' of a next program. This removes the need to make
+temporary files between separate processes and became one of the best
+demonstrations of the Unix-way, or Unix philosopy.
+
+Every program has three streams identifying where it reads/writes non-file
+inputs/outputs: @emph{Standard input}, @emph{Standard output}, and
address@hidden error}. When a program is called alone, all three are
+directed to the terminal that you are using. If it needs an input, it will
+prompt you for one and you can type it in. Or, it prints its results in the
+terminal for you to see.
+
+For example, say you have a FITS table/catalog containing the B and V band
+magnitudes (@code{MAG_B} and @code{MAG_V} columns) of a selection of
+galaxies along with many other columns. If you want to see only these two
+columns in your terminal, can use Gnuastro's @ref{Table} program like
+below:
+
address@hidden
+$ asttable cat.fits -cMAG_B,MAG_V
address@hidden example
+
+Through the Unix pipe mechanism, when the shell confronts the pipe
+character (@key{|}), it connects the standard output of the program before
+the pipe, to the standard input of the program after it. So it is literally
+a ``pipe'': everything that you would see printed by the first program on
+the command (without any pipe), is now passed to the second program (and
+not seen by you).
+
address@hidden AWK
address@hidden GNU AWK
+To continue the previous example, let's say you want to see the B-V
+color. To do this, you can pipe Table's output to AWK (a wonderful tool for
+processing things like plain text tables):
+
address@hidden
+$ asttable cat.fits -cMAG_B,MAG_V | awk '@{print address@hidden'
address@hidden example
+
+But understanding the distribution by visually seeing all the numbers under
+each other is not too useful! You can therefore feed this single column
+information into @ref{Statistics} to give you a general feeling of the
+distribution with the same command:
+
address@hidden
+$ asttable cat.fits -cMAG_B,MAG_V | awk '@{print address@hidden' |
aststatistics
address@hidden example
+
+Gnuastro's programs that accept input from standard input, only look into
+the Standard input stream if there is no first argument. In other words,
+arguments take precedence over Standard input. When no argument is
+provided, the programs check if the standard input stream is already full
+or not (output from another program is waiting to be used). If data is
+present in the standard input stream, it is used.
address@hidden Configuration files, Multi-threaded operations, Command-line,
Common program behavior
+When the standard input is empty, the program will wait
address@hidden micro-seconds for you to manually enter the first
+line (ending with a new-line character, or the @key{ENTER} key, see
address@hidden output options}). If it detects the first line in this time,
+there is no more time limit, and you can manually write/type all the lines
+for as long as it takes. To inform the program that Standard input has
+finished, press @key{CTRL-D} after a new line. If the program doesn't catch
+the first line before the time-out finishes, it will abort with an error
+saying that no input was provided.
+
address@hidden
address@hidden
address@hidden input in Standard input is discarded: } Be careful that when
+you manually fill the Standard input, the data will be discarded once the
+program finishes and reproducing the result will be impossible. Therefore
+this form of providing input is only good for temporary tests.
address@hidden cartouche
+
address@hidden
address@hidden
address@hidden input currently only for plain text: } Currently Standard
+input only works for plain text inputs like the example above. We will
+later allow FITS files into the programs through standard input also.
address@hidden cartouche
+
+
address@hidden Configuration files, Getting help, Command-line, Common program
behavior
@section Configuration files
@cindex @file{etc}
@@ -8234,1192 +8363,1200 @@ source by your self as in @ref{Quick start}.
address@hidden Multi-threaded operations, Numeric data types, Configuration
files, Common program behavior
address@hidden Multi-threaded operations
address@hidden Getting help, Multi-threaded operations, Configuration files,
Common program behavior
address@hidden Getting help
address@hidden nproc
address@hidden pthread
address@hidden CPU threads
address@hidden GNU Coreutils
address@hidden Using CPU threads
address@hidden CPU, using all threads
address@hidden Multi-threaded programs
address@hidden Using multiple CPU cores
address@hidden Simultaneous multithreading
-Some of the programs benefit significantly when you use all the threads
-your computer's CPU has to offer to your operating system. The number of
-threads available can be larger than the number of physical (hardware)
-cores in the CPU (also known as Simultaneous multithreading). For example,
-in Intel's CPUs (those that implement its Hyper-threading technology) the
-number of threads is usually double the number of physical cores in your
-CPU. On a GNU/Linux system, the number of threads available can be found
-with the command @command{$ nproc} command (part of GNU Coreutils).
address@hidden Help
address@hidden Book formats
address@hidden Remembering options
address@hidden Convenient book formats
+Probably the first time you read this book, it is either in the PDF
+or HTML formats. These two formats are very convenient for when you
+are not actually working, but when you are only reading. Later on,
+when you start to use the programs and you are deep in the middle of
+your work, some of the details will inevitably be forgotten. Going to
+find the PDF file (printed or digital) or the HTML webpage is a major
+distraction.
address@hidden --numthreads
address@hidden Number of threads available
address@hidden Available number of threads
address@hidden Internally stored option value
-Gnuastro's programs can find the number of threads available to your system
-internally at run-time (when you execute the program). However, if a value
-is given to the @option{--numthreads} option, the given number will be
-used, see @ref{Operating mode options} and @ref{Configuration files} for ways
to
-use this option. Thus @option{--numthreads} is the only common option in
-Gnuastro's programs with a value that doesn't have to be specified anywhere
-on the command-line or in the configuration files.
address@hidden Online help
address@hidden Command-line help
+GNU software have a very unique set of tools for aiding your memory on
+the command-line, where you are working, depending how much of it you
+need to remember. In the past, such command-line help was known as
+``online'' help, because they were literally provided to you `on'
+the command `line'. However, nowadays the word ``online'' refers to
+something on the internet, so that term will not be used. With this
+type of help, you can resume your exciting research without taking
+your hands off the keyboard.
address@hidden
-* A note on threads:: Caution and suggestion on using threads.
-* How to run simultaneous operations:: How to run things simultaneously.
address@hidden menu
address@hidden Installed help methods
+Another major advantage of such command-line based help routines is
+that they are installed with the software in your computer, therefore
+they are always in sync with the executable you are actually
+running. Three of them are actually part of the executable. You don't
+have to worry about the version of the book or program. If you rely
+on external help (a PDF in your personal print or digital archive or
+HTML from the official webpage) you have to check to see if their
+versions fit with your installed program.
address@hidden A note on threads, How to run simultaneous operations,
Multi-threaded operations, Multi-threaded operations
address@hidden A note on threads
+If you only need to remember the short or long names of the options,
address@hidden is advised. If it is what the options do, then
address@hidden is a great tool. Man pages are also provided for those
+who are use to this older system of documentation. This full book is
+also available to you on the command-line in Info format. If none of
+these seems to resolve the problems, there is a mailing list which
+enables you to get in touch with experienced Gnuastro users. In the
+subsections below each of these methods are reviewed.
address@hidden Using multiple threads
address@hidden Best use of CPU threads
address@hidden Efficient use of CPU threads
-Spinning off threads is not necessarily the most efficient way to run an
-application. Creating a new thread isn't a cheap operation for the
-operating system. It is most useful when the input data are fixed and you
-want the same operation to be done on parts of it. For example one input
-image to Crop and multiple crops from various parts of it. In this fashion,
-the image is loaded into memory once, all the crops are divided between the
-number of threads internally and each thread cuts out those parts which are
-assigned to it from the same image. On the other hand, if you have multiple
-images and you want to crop the same region(s) out of all of them, it is
-much more efficient to set @option{--numthreads=1} (so no threads spin off)
-and run Crop multiple times simultaneously, see @ref{How to run
-simultaneous operations}.
address@hidden Wall-clock time
-You can check the boost in speed by first running a program on one of the
-data sets with the maximum number of threads and another time (with
-everything else the same) and only using one thread. You will notice that
-the wall-clock time (reported by most programs at their end) in the former
-is longer than the latter divided by number of physical CPU cores (not
-threads) available to your operating system. Asymptotically these two times
-can be equal (most of the time they aren't). So limiting the programs to
-use only one thread and running them independently on the number of
-available threads will be more efficient.
address@hidden
+* --usage:: View option names and value formats.
+* --help:: List all options with description.
+* Man pages:: Man pages generated from --help.
+* Info:: View complete book in terminal.
+* help-gnuastro mailing list:: Contacting experienced users.
address@hidden menu
address@hidden System Cache
address@hidden Cache, system
-Note that the operating system keeps a cache of recently processed
-data, so usually, the second time you process an identical data set
-(independent of the number of threads used), you will get faster
-results. In order to make an unbiased comparison, you have to first
-clean the system's cache with the following command between the two
-runs.
address@hidden --usage, --help, Getting help, Getting help
address@hidden @option{--usage}
address@hidden --usage
address@hidden Usage pattern
address@hidden Mandatory arguments
address@hidden Optional and mandatory tokens
+If you give this option, the program will not run. It will only print a
+very concise message showing the options and arguments. Everything within
+square brackets (@option{[]}) is optional. For example here are the first
+and last two lines of Crop's @option{--usage} is shown:
@example
-$ sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
+$ astcrop --usage
+Usage: astcrop [-Do?IPqSVW] [-d INT] [-h INT] [-r INT] [-w INT]
+ [-x INT] [-y INT] [-c INT] [-p STR] [-N INT] [--deccol=INT]
+ ....
+ [--setusrconf] [--usage] [--version] [--wcsmode]
+ [ASCIIcatalog] FITSimage(s).fits
@end example
address@hidden
address@hidden
address@hidden: Should I use multiple threads?} Depends:
address@hidden
address@hidden
-If you only have @strong{one} data set (image in most cases!), then
-yes, the more threads you use (with a maximum of the number of threads
-available to your OS) the faster you will get your results.
-
address@hidden
-If you want to run the same operation on @strong{multiple} data sets, it is
-best to set the number of threads to 1 and use Make, or GNU Parallel, as
-explained in @ref{How to run simultaneous operations}.
address@hidden itemize
address@hidden cartouche
+There are no explanations on the options, just their short and long
+names shown separately. After the program name, the short format of
+all the options that don't require a value (on/off options) is
+displayed. Those that do require a value then follow in separate
+brackets, each displaying the format of the input they want, see
address@hidden Since all options are optional, they are shown in
+square brackets, but arguments can also be optional. For example in
+this example, a catalog name is optional and is only required in some
+modes. This is a standard method of displaying optional arguments for
+all GNU software.
address@hidden --help, Man pages, --usage, Getting help
address@hidden @option{--help}
address@hidden --help
+If the command-line includes this option, the program will not be
+run. It will print a complete list of all available options along with
+a short explanation. The options are also grouped by their
+context. Within each context, the options are sorted
+alphabetically. Since the options are shown in detail afterwards, the
+first line of the @option{--help} output shows the arguments and if
+they are optional or not, similar to @ref{--usage}.
+In the @option{--help} output of all programs in Gnuastro, the
+options for each program are classified based on context. The first
+two contexts are always options to do with the input and output
+respectively. For example input image extensions or supplementary
+input files for the inputs. The last class of options is also fixed in
+all of Gnuastro, it shows operating mode options. Most of these
+options are already explained in @ref{Operating mode options}.
address@hidden Long outputs
address@hidden Redirection of output
address@hidden Command-line, long outputs
+The help message will sometimes be longer than the vertical size of
+your terminal. If you are using a graphical user interface terminal
+emulator, you can scroll the terminal with your mouse, but we promised
+no mice distractions! So here are some suggestions:
address@hidden How to run simultaneous operations, , A note on threads,
Multi-threaded operations
address@hidden How to run simultaneous operations
address@hidden
address@hidden
address@hidden Scroll command-line
address@hidden Command-line scroll
address@hidden @key{Shift + PageUP} and @key{Shift + PageDown}
address@hidden + PageUP} to scroll up and @key{Shift + PageDown} to scroll
+down. For most help output this should be enough. The problem is that
+it is limited by the number of lines that your terminal keeps in
+memory and that you can't scroll by lines, only by whole screens.
-There are address@hidden third way would be to open multiple terminal
-emulator windows in your GUI, type the commands separately on each and
-press @key{Enter} once on each terminal, but this is far too frustrating,
-tedious and prone to errors. It's therefore not a realistic solution when
-tens, hundreds or thousands of operations (your research targets,
-multiplied by the operations you do on each) are to be done.} approaches to
-simultaneously execute a program: using GNU Parallel or Make (GNU Make is
-the most common implementation). The first is very useful when you only
-want to do one job multiple times and want to get back to your work without
-actually keeping the command you ran. The second is usually for more
-important operations, with lots of dependencies between the different
-products (for example a full scientific research).
address@hidden
address@hidden Pipe
address@hidden @command{less}
+Pipe to @command{less}. A pipe is a form of shell re-direction. The
address@hidden tool in Unix-like systems was made exactly for such
+outputs of any length. You can pipe (@command{|}) the output of any
+program that is longer than the screen to it and then you can scroll
+through (up and down) with its many tools. For example:
address@hidden
+$ astnoisechisel --help | less
address@hidden example
address@hidden
+Once you have gone through the text, you can quit @command{less} by
+pressing the @key{q} key.
address@hidden @asis
address@hidden GNU Parallel
address@hidden GNU Parallel
-When you only want to run multiple instances of a command on different
-threads and get on with the rest of your work, the best method is to
-use GNU parallel. Surprisingly GNU Parallel is one of the few GNU
-packages that has no Info documentation but only a Man page, see
address@hidden So to see the documentation after installing it please run
address@hidden
address@hidden Save output to file
address@hidden Redirection of output
+Redirect to a file. This is a less convenient way, because you will
+then have to open the file in a text editor! You can do this with the
+shell redirection tool (@command{>}):
address@hidden
+$ astnoisechisel --help > filename.txt
address@hidden example
address@hidden itemize
+
address@hidden GNU Grep
address@hidden Searching text
address@hidden Command-line searching text
+In case you have a special keyword you are looking for in the help, you
+don't have to go through the full list. GNU Grep is made for this job. For
+example if you only want the list of options whose @option{--help} output
+contains the word ``axis'' in Crop, you can run the following command:
@example
-$ man parallel
+$ astcrop --help | grep axis
@end example
address@hidden
-As an example, let's assume we want to crop a region fixed on the
-pixels (500, 600) with the default width from all the FITS images in
-the @file{./data} directory ending with @file{sci.fits} to the current
-directory. To do this, you can run:
address@hidden @code{ARGP_HELP_FMT}
address@hidden Argp argument parser
address@hidden Customize @option{--help} output
address@hidden @option{--help} output customization
+If the output of this option does not fit nicely within the confines
+of your terminal, GNU does enable you to customize its output through
+the environment variable @code{ARGP_HELP_FMT}, you can set various
+parameters which specify the formatting of the help messages. For
+example if your terminals are wider than 70 spaces (say 100) and you
+feel there is too much empty space between the long options and the
+short explanation, you can change these formats by giving values to
+this environment variable before running the program with the
address@hidden output. You can define this environment variable in
+this manner:
@example
-$ parallel astcrop --numthreads=1 --xc=500 --yc=600 ::: \
- ./data/*sci.fits
+$ export ARGP_HELP_FMT=rmargin=100,opt-doc-col=20
@end example
address@hidden @file{.bashrc}
+This will affect all GNU programs using GNU C library's @file{argp.h}
+facilities as long as the environment variable is in memory. You can
+see the full list of these formatting parameters in the ``Argp User
+Customization'' part of the GNU C library manual. If you are more
+comfortable to read the @option{--help} outputs of all GNU software in
+your customized format, you can add your customization (similar to
+the line above, without the @command{$} sign) to your @file{~/.bashrc}
+file. This is a standard option for all GNU software.
+
address@hidden Man pages, Info, --help, Getting help
address@hidden Man pages
address@hidden Man pages
+Man pages were the Unix method of providing command-line documentation
+to a program. With GNU Info, see @ref{Info} the usage of this method
+of documentation is highly discouraged. This is because Info provides
+a much more easier to navigate and read environment.
+However, some operating systems require a man page for packages that
+are installed and some people are still used to this method of command
+line help. So the programs in Gnuastro also have Man pages which are
+automatically generated from the outputs of @option{--version} and
address@hidden using the GNU help2man program. So if you run
address@hidden
+$ man programname
address@hidden example
@noindent
-GNU Parallel can help in many more conditions, this is one of the
-simplest, see the man page for lots of other examples. For absolute
-beginners: the backslash (@command{\}) is only a line breaker to fit
-nicely in the page. If you type the whole command in one line, you
-should remove it.
+You will be provided with a man page listing the options in the
+standard manner.
address@hidden Make
address@hidden Make
-Make is a program for building ``targets'' (e.g., files) using ``recipes''
-(a set of operations) when their known ``prerequisites'' (other files) have
-been updated. It elegantly allows you to define dependency structures for
-building your final output and updating it efficiently when the inputs
-change. It is the most common infra-structure to build software
-today.
-Scientific research methodology is very similar to software development:
-you start by testing a hypothesis on a small sample of objects/targets with
-a simple set of steps. As you are able to get promising results, you
-improve the method and use it on a larger, more general, sample. In the
-process, you will confront many issues that have to be corrected (bugs in
-software development jargon). Make a wonderful tool to manage this style of
-development. It has been used to make reproducible papers, for example see
address@hidden://gitlab.com/makhlaghi/NoiseChisel-paper, the reproduction
-pipeline} of the paper introducing @ref{NoiseChisel} (one of Gnuastro's
-programs).
address@hidden GNU Make
-GNU address@hidden@url{https://www.gnu.org/software/make/}} is the most
-common implementation which (similar to nearly all GNU programs, comes with
-a wonderful
address@hidden@url{https://www.gnu.org/software/make/manual/}}). Make is
-very basic and simple, and thus the manual is short (the most important
-parts are in the first roughly 100 pages) and easy to read/understand.
-Make comes with a @option{--jobs} (@option{-j}) option which allows you to
-specify the maximum number of jobs that can be done simultaneously. For
-example if you have 8 threads available to your operating system. You can
-run:
+
address@hidden Info, help-gnuastro mailing list, Man pages, Getting help
address@hidden Info
+
address@hidden GNU Info
address@hidden Command-line, viewing full book
+Info is the standard documentation format for all GNU software. It is
+a very useful command-line document viewing format, fully equipped
+with links between the various pages and menus and search
+capabilities. As explained before, the best thing about it is that it
+is available for you the moment you need to refresh your memory on any
+command-line tool in the middle of your work without having to take
+your hands off the keyboard. This complete book is available in Info
+format and can be accessed from anywhere on the command-line.
+
+To open the Info format of any installed programs or library on your
+system which has an Info format book, you can simply run the command
+below (change @command{executablename} to the executable name of the
+program or library):
@example
-$ make -j8
+$ info executablename
@end example
-With this command, Make will process your @file{Makefile} and create all
-the targets (can be thousands of FITS images for example) simultaneously on
-8 threads, while fully respecting their dependencies (only building a
-file/target when its prerequisites are successfully built). Make is thus
-strongly recommended for managing scientific research where robustness,
-archiving, reproducibility and address@hidden its multi-threaded
-capabilities, Make will only re-build those targets that depend on a change
-you have made, not the whole work. For example, if you have set the
-prerequisites properly, you can easily test the changing of a parameter on
-your paper's results without having to re-do everything (which is much
-faster). This allows you to be much more productive in easily checking
-various ideas/assumptions of the different stages of your research and thus
-produce a more robust result for your exciting science.} are important.
address@hidden
address@hidden Learning GNU Info
address@hidden GNU software documentation
+In case you are not already familiar with it, run @command{$ info
+info}. It does a fantastic job in explaining all its capabilities its
+self. It is very short and you will become sufficiently fluent in
+about half an hour. Since all GNU software documentation is also
+provided in Info, your whole GNU/Linux life will significantly
+improve.
address@hidden table
address@hidden GNU Emacs
address@hidden GNU C library
+Once you've become an efficient navigator in Info, you can go to any
+part of this book or any other GNU software or library manual, no
+matter how long it is, in a matter of seconds. It also blends nicely
+with GNU Emacs (a text editor) and you can search manuals while you
+are writing your document or programs without taking your hands off
+the keyboard, this is most useful for libraries like the GNU C
+library. To be able to access all the Info manuals installed in your
+GNU/Linux within Emacs, type @key{Ctrl-H + i}.
+To see this whole book from the beginning in Info, you can run
address@hidden
+$ info gnuastro
address@hidden example
address@hidden
+If you run Info with the particular program executable name, for
+example @file{astcrop} or @file{astnoisechisel}:
address@hidden
+$ info astprogramname
address@hidden example
address@hidden Numeric data types, Tables, Multi-threaded operations, Common
program behavior
address@hidden Numeric data types
address@hidden
+you will be taken to the section titled ``Invoking ProgramName'' which
+explains the inputs and outputs along with the command-line options for
+that program. Finally, if you run Info with the official program name, for
+example Crop or NoiseChisel:
address@hidden Bit
address@hidden Type
-At the lowest level, the computer stores everything in terms of @code{1} or
address@hidden For example, each program in Gnuastro, or each astronomical image
-you take with the telescope is actually a string of millions of these zeros
-and ones. The space required to keep a zero or one is the smallest unit of
-storage, and is known as a @emph{bit}. However, understanding and
-manipulating this string of bits is extremely hard for most
-people. Therefore, we define packages of these bits along with a standard
-on how to interpret the bits in each package as a @emph{type}.
address@hidden
+$ info ProgramName
address@hidden example
address@hidden Byte
address@hidden Signed integer
address@hidden Unsigned integer
address@hidden Integer, Signed
-The most basic standard for reading the bits is integer numbers
-(@mymath{..., -2, -1, 0, 1, 2, ...}, more bits will give larger
-limits). The common integer types are 8, 16, 32, and 64 bits wide. For each
-width, there are two standards for reading the bits: signed and unsigned
-integers. In the former, negative numbers are allowed and in the latter,
-they aren't. The @code{unsigned} types thus have larger positive limits
-(one extra bit), but no negative value. When the context of your work
-doesn't involve negative numbers (for example counting, where negative is
-not defined), it is best to use the @code{unsigned} types. For full
-numerical range of all integer types, see below.
address@hidden
+you will be taken to the top section which introduces the
+program. Note that in all cases, Info is not case sensitive.
-Another standard of converting a given number of bits to numbers is the
-floating point standard, this standard can approximately store any real
-number with a given precision. There are two common floating point types:
-32-bit and 64-bit, for single and double precision floating point numbers
-respectively. The former is sufficient for data with less than 8
-significant decimal digits (most astronomical data), while the latter is
-good for less than 16 significant decimal digits. The representation of
-real numbers as bits is much more complex than integers. If you are
-interested, you can start with the
address@hidden://en.wikipedia.org/wiki/Floating_point, Wikipedia article}.
-With the conversion operators in Gnuastro's Arithmetic, you can change the
-types of data to each other, which is necessary in some contexts. For
-example the program/library, that you intend to feed the data into, only
-accepts floating point values, but you have an integer image. Another
-situation that conversion can be helpful is when you know that your data
-only has values that fit within @code{int8} or @code{uint16}. However it is
-currently formatted in the @code{float64} type. Operations involving
-floating point or larger integer types are significantly slower than
-integer or smaller-width types respectively. In the latter case, it also
-requires much more (by 8 or 4 times in the example above) storage space. So
-when you confront such situations and want to store/archive/transfter the
-data, it is best convert them to the most efficient type.
-The short and long names for the recognized numeric data types in Gnuastro
-are listed below. Both short and long names can be used when you want to
-specify a type. For example, as a value to the common option
address@hidden (see @ref{Input output options}), or in the information
-comment lines of @ref{Gnuastro text table format}. The ranges listed below
-are inclusive.
address@hidden help-gnuastro mailing list, , Info, Getting help
address@hidden help-gnuastro mailing list
address@hidden @code
address@hidden u8
address@hidden uint8
-8-bit unsigned integers, range:@*
address@hidden to\ }2^8-1]} or @mymath{[0\rm{\ to\ }255]}.
address@hidden help-gnuastro mailing list
address@hidden Mailing list: help-gnuastro
+Gnuastro maintains the help-gnuastro mailing list for users to ask any
+questions related to Gnuastro. The experienced Gnuastro users and some
+of its developers are subscribed to this mailing list and your email
+will be sent to them immediately. However, when contacting this
+mailing list please have in mind that they are possibly very busy and
+might not be able to answer immediately.
address@hidden i8
address@hidden int8
-8-bit signed integers, range:@*
address@hidden to\ }2^7-1]} or @mymath{[-128\rm{\ to\ }127]}.
address@hidden Mailing list archives
address@hidden @code{help-gnuastro@@gnu.org}
+To ask a question from this mailing list, send a mail to
address@hidden@@gnu.org}. Anyone can view the mailing list
+archives at @url{http://lists.gnu.org/archive/html/help-gnuastro/}. It
+is best that before sending a mail, you search the archives to see if
+anyone has asked a question similar to yours. If you want to make a
+suggestion or report a bug, please don't send a mail to this mailing
+list. We have other mailing lists and tools for those purposes, see
address@hidden a bug} or @ref{Suggest new feature}.
address@hidden u16
address@hidden uint16
-16-bit unsigned integers, range:@*
address@hidden to\ }2^{16}-1]} or @mymath{[0\rm{\ to\ }65535]}.
address@hidden i16
address@hidden int16
-16-bit signed integers, range:@* @mymath{[-2^{15}\rm{\ to\ }2^{15}-1]} or
address@hidden to\ }32767]}.
address@hidden u32
address@hidden uint32
-32-bit unsigned integers, range:@* @mymath{[0\rm{\ to\ }2^{32}-1]} or
address@hidden to\ }4294967295]}.
address@hidden i32
address@hidden int32
-32-bit signed integers, range:@* @mymath{[-2^{31}\rm{\ to\ }2^{31}-1]} or
address@hidden to\ }2147483647]}.
address@hidden u64
address@hidden uint64
-64-bit unsigned integers, address@hidden @mymath{[0\rm{\ to\ }2^{64}-1]} or
address@hidden to\ }18446744073709551615]}.
address@hidden Multi-threaded operations, Numeric data types, Getting help,
Common program behavior
address@hidden Multi-threaded operations
address@hidden i64
address@hidden int64
-64-bit signed integers, range:@* @mymath{[-2^{63}\rm{\ to\ }2^{63}-1]} or
address@hidden to\ }9223372036854775807]}.
address@hidden nproc
address@hidden pthread
address@hidden CPU threads
address@hidden GNU Coreutils
address@hidden Using CPU threads
address@hidden CPU, using all threads
address@hidden Multi-threaded programs
address@hidden Using multiple CPU cores
address@hidden Simultaneous multithreading
+Some of the programs benefit significantly when you use all the threads
+your computer's CPU has to offer to your operating system. The number of
+threads available can be larger than the number of physical (hardware)
+cores in the CPU (also known as Simultaneous multithreading). For example,
+in Intel's CPUs (those that implement its Hyper-threading technology) the
+number of threads is usually double the number of physical cores in your
+CPU. On a GNU/Linux system, the number of threads available can be found
+with the command @command{$ nproc} command (part of GNU Coreutils).
address@hidden f32
address@hidden float32
-32-bit (single-precision) floating point types. The maximum (minimum is its
-negative) possible value is
address@hidden Single-precision floating points can
-accurately represent a floating point number up to @mymath{\sim7.2}
-significant decimals. Given the heavy noise in astronomical data, this is
-usually more than sufficient for storing results.
address@hidden --numthreads
address@hidden Number of threads available
address@hidden Available number of threads
address@hidden Internally stored option value
+Gnuastro's programs can find the number of threads available to your system
+internally at run-time (when you execute the program). However, if a value
+is given to the @option{--numthreads} option, the given number will be
+used, see @ref{Operating mode options} and @ref{Configuration files} for ways
to
+use this option. Thus @option{--numthreads} is the only common option in
+Gnuastro's programs with a value that doesn't have to be specified anywhere
+on the command-line or in the configuration files.
address@hidden f64
address@hidden float64
-64-bit (double-precision) floating point types. The maximum (minimum is its
-negative) possible value is @mymath{\sim10^{308}}. Double-precision
-floating points can accurately represent a floating point number
address@hidden significant decimals. This is usually good for processing
-(mixing) the data internally, for example a sum of single precision data
-(and later storing the result as @code{float32}).
address@hidden table
address@hidden
+* A note on threads:: Caution and suggestion on using threads.
+* How to run simultaneous operations:: How to run things simultaneously.
address@hidden menu
address@hidden
address@hidden
address@hidden file formats don't recognize all types.} Some file formats
-don't recognize all the types, for example the FITS standard (see
address@hidden) does not define @code{uint64} in binary tables or images. When
-a type is not acceptable for output into a given file format, the
-respective Gnuastro program or library will let you know and abort. On the
-command-line, you can use the @ref{Arithmetic} program to convert the
-numerical type of a dataset, in the libraries, you can call
address@hidden
address@hidden cartouche
address@hidden A note on threads, How to run simultaneous operations,
Multi-threaded operations, Multi-threaded operations
address@hidden A note on threads
+
address@hidden Using multiple threads
address@hidden Best use of CPU threads
address@hidden Efficient use of CPU threads
+Spinning off threads is not necessarily the most efficient way to run an
+application. Creating a new thread isn't a cheap operation for the
+operating system. It is most useful when the input data are fixed and you
+want the same operation to be done on parts of it. For example one input
+image to Crop and multiple crops from various parts of it. In this fashion,
+the image is loaded into memory once, all the crops are divided between the
+number of threads internally and each thread cuts out those parts which are
+assigned to it from the same image. On the other hand, if you have multiple
+images and you want to crop the same region(s) out of all of them, it is
+much more efficient to set @option{--numthreads=1} (so no threads spin off)
+and run Crop multiple times simultaneously, see @ref{How to run
+simultaneous operations}.
address@hidden Wall-clock time
+You can check the boost in speed by first running a program on one of the
+data sets with the maximum number of threads and another time (with
+everything else the same) and only using one thread. You will notice that
+the wall-clock time (reported by most programs at their end) in the former
+is longer than the latter divided by number of physical CPU cores (not
+threads) available to your operating system. Asymptotically these two times
+can be equal (most of the time they aren't). So limiting the programs to
+use only one thread and running them independently on the number of
+available threads will be more efficient.
address@hidden System Cache
address@hidden Cache, system
+Note that the operating system keeps a cache of recently processed
+data, so usually, the second time you process an identical data set
+(independent of the number of threads used), you will get faster
+results. In order to make an unbiased comparison, you have to first
+clean the system's cache with the following command between the two
+runs.
address@hidden Tables, Tessellation, Numeric data types, Common program behavior
address@hidden Tables
address@hidden
+$ sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
address@hidden example
-``A table is a collection of related data held in a structured format
-within a database. It consists of columns, and rows.'' (from
-Wikipedia). Each column in the table contains the values of one property
-and each row is a collection of properties (columns) for one target
-object. For example, let's assume you have just ran MakeCatalog (see
address@hidden) on an image to measure some properties for the labeled
-regions (which might be detected galaxies for example) in the image. For
-each labeled region (detected galaxy), there will be a @emph{row} which
-groups its measured properties as @emph{columns}, one column for each
-property. One such property can be the object's magnitude, which is the sum
-of pixels with that label, or its center can be defined as the
-light-weighted average value of those pixels. Many such properties can be
-derived from the raw pixel values and their position, see @ref{Invoking
-astmkcatalog} for a long list.
address@hidden
address@hidden
address@hidden: Should I use multiple threads?} Depends:
address@hidden
address@hidden
+If you only have @strong{one} data set (image in most cases!), then
+yes, the more threads you use (with a maximum of the number of threads
+available to your OS) the faster you will get your results.
-As a summary, for each labeled region (or, galaxy) we have one @emph{row}
-and for each measured property we have one @emph{column}. This high-level
-structure is usually the first step for higher-level analysis, for example
-finding the stellar mass or photometric redshift from magnitudes in
-multiple colors. Thus, tables are not just outputs of programs, in fact it
-is much more common for tables to be inputs of programs. For example, to
-make a mock galaxy image, you need to feed in the properties of each galaxy
-into @ref{MakeProfiles} for it do the inverse of the process above and make
-a simulated image from a catalog, see @ref{Sufi simulates a detection}. In
-other cases, you can feed a table into @ref{Crop} and it will crop out
-regions centered on the positions within the table, see @ref{Hubble
-visually checks and classifies his catalog}. So to end this relatively long
-introduction, tables play a very important role in astronomy, or generally
-all branches of data analysis.
address@hidden
+If you want to run the same operation on @strong{multiple} data sets, it is
+best to set the number of threads to 1 and use Make, or GNU Parallel, as
+explained in @ref{How to run simultaneous operations}.
address@hidden itemize
address@hidden cartouche
-In @ref{Recognized table formats} the currently recognized table formats in
-Gnuastro are discussed. You can use any of these tables as input or ask for
-them to be built as output. The most common type of table format is a
-simple plain text file with each row on one line and columns separated by
-white space characters, this format is easy to read/write by eye/hand. To
-give it the full functionality of more specific table types like the FITS
-tables, Gnuastro has a special convention which you can use to give each
-column a name, type, unit, and comments, while still being readable by
-other plain text table readers. This convention is described in
address@hidden text table format}.
-When tables are input to a program, the program reading it needs to know
-which column(s) it should use for its desired purposes. Gnuastro's programs
-all follow a similar convention, on the way you can select columns in a
-table. They are thoroughly discussed in @ref{Selecting table columns}.
address@hidden
-* Recognized table formats:: Table formats that are recognized in Gnuastro.
-* Gnuastro text table format:: Gnuastro's convention plain text tables.
-* Selecting table columns:: Identify/select certain columns from a table
address@hidden menu
address@hidden Recognized table formats, Gnuastro text table format, Tables,
Tables
address@hidden Recognized table formats
address@hidden How to run simultaneous operations, , A note on threads,
Multi-threaded operations
address@hidden How to run simultaneous operations
-The list of table formats that Gnuastro can currently read from and write
-to are described below. Each has their own advantage and disadvantages, so a
-short review of the format is also provided to help you make the best
-choice based on how you want to define your input tables or later use your
-output tables.
+There are address@hidden third way would be to open multiple terminal
+emulator windows in your GUI, type the commands separately on each and
+press @key{Enter} once on each terminal, but this is far too frustrating,
+tedious and prone to errors. It's therefore not a realistic solution when
+tens, hundreds or thousands of operations (your research targets,
+multiplied by the operations you do on each) are to be done.} approaches to
+simultaneously execute a program: using GNU Parallel or Make (GNU Make is
+the most common implementation). The first is very useful when you only
+want to do one job multiple times and want to get back to your work without
+actually keeping the command you ran. The second is usually for more
+important operations, with lots of dependencies between the different
+products (for example a full scientific research).
@table @asis
address@hidden Plain text table
-This is the most basic and simplest way to create, view, or edit the table
-by hand on a text editor. The other formats described below are less
-eye-friendly and have a more formal structure (for easier computer
-readability). It is fully described in @ref{Gnuastro text table format}.
address@hidden GNU Parallel
address@hidden GNU Parallel
+When you only want to run multiple instances of a command on different
+threads and get on with the rest of your work, the best method is to
+use GNU parallel. Surprisingly GNU Parallel is one of the few GNU
+packages that has no Info documentation but only a Man page, see
address@hidden So to see the documentation after installing it please run
address@hidden FITS Tables
address@hidden Tables FITS
address@hidden ASCII table, FITS
address@hidden FITS ASCII tables
-The FITS ASCII table extension is fully in ASCII encoding and thus easily
-readable on any text editor (assuming it is the only extension in the FITS
-file). If the FITS file also contains binary extensions (for example an
-image or binary table extensions), then there will be many hard to print
-characters. The FITS ASCII format doesn't have new line characters to
-separate rows. In the FITS ASCII table standard, each row is defined as a
-fixed number of characters (value to the @code{NAXIS1} keyword), so to
-visually inspect it properly, you would have to adjust your text editor's
-width to this value. All columns start at given character positions and
-have a fixed width (number of characters).
-
-Numbers in a FITS ASCII table are printed into ASCII format, they are not
-in binary (that the CPU uses). Hence, they can take a larger space in
-memory, loose their precision, and take longer to read into memory. If you
-are dealing with integer type columns (see @ref{Numeric data types}),
-another issue with FITS ASCII tables is that the type information for the
-column will be lost (there is only one integer type in FITS ASCII
-tables). One problem with the binary format on the other hand is that it
-isn't portable (different CPUs/compilers) have different standards for
-translating the zeros and ones. But since ASCII characters are defined on a
-byte and are well recognized, they are better for portability on those
-various systems. Gnuastro's plain text table format described below is much
-more portable and easier to read/write/interpret by humans manually.
-
-Generally, as the name implies, this format is useful for when your table
-mainly contains ASCII columns (for example file names, or
-descriptions). They can be useful when you need to include columns with
-structured ASCII information along with other extensions in one FITS
-file. In such cases, you can also consider header keywords (see
address@hidden).
-
address@hidden Binary table, FITS
address@hidden FITS binary tables
-The FITS binary table is the FITS standard's solution to the issues
-discussed with keeping numbers in ASCII format as described under the FITS
-ASCII table title above. Only columns defined as a string type (a string of
-ASCII characters) are readable in a text editor. The portability problem
-with binary formats discussed above is mostly solved thanks to the
-portability of CFITSIO (see @ref{CFITSIO}) and the very long history of the
-FITS format which has been widely used since the 1970s.
-
-In the case of most numbers, storing them in binary format is more memory
-efficient than ASCII format. For example, to store @code{-25.72034} in
-ASCII format, you need 9 bytes/characters. But if you keep this same number
-(to the approximate precision possible) as a 4-byte (32-bit) floating point
-number, you can keep/transmit it with less than half the amount of
-memory. When catalogs contain thousands/millions of rows in tens/hundreds
-of columns, this can lead to significant improvements in memory/band-width
-usage. Moreover, since the CPU does its operations in the binary formats,
-reading the table in and writing it out is also much faster than an ASCII
-table.
-
-When you are dealing with integer numbers, the compression ratio can be
-even better, for example if you know all of the values in a column are
-positive and less than @code{255}, you can use the @code{unsigned char}
-type which only takes one byte! If they are between @code{-128} and
address@hidden, then you can use the (signed) @code{char} type. So if you are
-thoughtful about the limits of your integer columns, you can greatly reduce
-the size of your file and also the speed at which it is read/written. This
-can be very useful when sharing your results with collaborators or
-publishing them. To decrease the file size even more you can name your
-output as ending in @file{.fits.gz} so it is also compressed after
-creation. Just note that compression/decompressing is CPU intensive and can
-slow down the writing/reading of the file.
-
-Fortunately the FITS Binary table format also accepts ASCII strings as
-column types (along with the various numerical types). So your dataset can
-also contain non-numerical columns.
-
address@hidden table
-
address@hidden
-* Gnuastro text table format:: Reading plain text tables
address@hidden menu
-
address@hidden Gnuastro text table format, Selecting table columns, Recognized
table formats, Tables
address@hidden Gnuastro text table format
-
-Plain text files are the most generic, portable, and easiest way to
-(manually) create, (visually) inspect, or (manually) edit a table. In this
-format, the ending of a row is defined by the new-line character (a line on
-a text editor). So when you view it on a text editor, every row will occupy
-one line. The delimiters (or characters separating the columns) are white
-space characters (space, horizontal tab, vertical tab) and a comma
-(@key{,}). The only further requirement is that all rows/lines must have
-the same number of columns.
-
-The columns don't have to be exactly under each other and the rows can be
-arbitrarily long with different lengths. For example the following contents
-in a file would be interpreted as a table with 4 columns and 2 rows, with
-each element interpreted as a @code{double} type (see @ref{Numeric data
-types}).
address@hidden
+$ man parallel
address@hidden example
address@hidden
+As an example, let's assume we want to crop a region fixed on the
+pixels (500, 600) with the default width from all the FITS images in
+the @file{./data} directory ending with @file{sci.fits} to the current
+directory. To do this, you can run:
@example
-1 2.234948 128 39.8923e8
-2 , 4.454 792 72.98348e7
+$ parallel astcrop --numthreads=1 --xc=500 --yc=600 ::: \
+ ./data/*sci.fits
@end example
-However, the example above has no other information about the columns (it
-is just raw data, with no meta-data). To use this table, you have to
-remember what the numbers in each column represent. Also, when you want to
-select columns, you have to count their position within the table. This can
-become frustrating and prone to bad errors (getting the columns wrong)
-especially as the number of columns increase. It is also bad for sending to
-a colleague, because they will find it hard to remember/use the columns
-properly.
address@hidden
+GNU Parallel can help in many more conditions, this is one of the
+simplest, see the man page for lots of other examples. For absolute
+beginners: the backslash (@command{\}) is only a line breaker to fit
+nicely in the page. If you type the whole command in one line, you
+should remove it.
-To solve these problems in Gnuastro's programs/libraries you aren't limited
-to using the column's number, see @ref{Selecting table columns}. If the
-columns have names, units, or comments you can also select your columns
-based on searches/matches in these fields, for example see @ref{Table}.
-Also, in this manner, you can't guide the program reading the table on how
-to read the numbers. As an example, the first and third columns above can
-be read as integer types: the first column might be an ID and the third can
-be the number of pixels an object occupies in an image. So there is no need
-to read these to columns as a @code{double} type (which takes more memory,
-and is slower).
address@hidden Make
address@hidden Make
+Make is a program for building ``targets'' (e.g., files) using ``recipes''
+(a set of operations) when their known ``prerequisites'' (other files) have
+been updated. It elegantly allows you to define dependency structures for
+building your final output and updating it efficiently when the inputs
+change. It is the most common infra-structure to build software
+today.
-In the bare-minimum example above, you also can't use strings of
-characters, for example the names of filters, or some other identifier that
-includes non-numerical characters. In the absence of any information, only
-numbers can be read robustly. Assuming we read columns with non-numerical
-characters as string, there would still be the problem that the strings
-might contain space (or any delimiter) character for some rows. So, each
-`word' in the string will be interpreted as a column and the program will
-abort with an error that the rows don't have the same number of columns.
+Scientific research methodology is very similar to software development:
+you start by testing a hypothesis on a small sample of objects/targets with
+a simple set of steps. As you are able to get promising results, you
+improve the method and use it on a larger, more general, sample. In the
+process, you will confront many issues that have to be corrected (bugs in
+software development jargon). Make a wonderful tool to manage this style of
+development. It has been used to make reproducible papers, for example see
address@hidden://gitlab.com/makhlaghi/NoiseChisel-paper, the reproduction
+pipeline} of the paper introducing @ref{NoiseChisel} (one of Gnuastro's
+programs).
-To correct for these limitations, Gnuastro defines the following convention
-for storing the table meta-data along with the raw data in one plain text
-file. The format is primarily designed for ease of reading/writing by
-eye/fingers, but is also structured enough to be read by a program.
address@hidden GNU Make
+GNU address@hidden@url{https://www.gnu.org/software/make/}} is the most
+common implementation which (similar to nearly all GNU programs, comes with
+a wonderful
address@hidden@url{https://www.gnu.org/software/make/manual/}}). Make is
+very basic and simple, and thus the manual is short (the most important
+parts are in the first roughly 100 pages) and easy to read/understand.
-When the first non-white character in a line is @key{#}, or there are no
-non-white characters in it, then the line will not be considered as a row
-of data in the table (this is a pretty standard convention in many
-programs, and higher level languages). In the former case, the line is
-interpreted as a @emph{comment}. If the comment line starts with address@hidden
-Column N:}', then it is assumed to contain information about column
address@hidden (a number, counting from 1). Comment lines that don't start with
-this pattern are ignored and you can use them to include any further
-information you want to store with the table in the text file. A column
-information comment is assumed to have the following format:
+Make comes with a @option{--jobs} (@option{-j}) option which allows you to
+specify the maximum number of jobs that can be done simultaneously. For
+example if you have 8 threads available to your operating system. You can
+run:
@example
-# Column N: NAME [UNIT, TYPE, BLANK] COMMENT
+$ make -j8
@end example
address@hidden NaN
address@hidden
-Any sequence of characters between address@hidden:}' and address@hidden' will
be
-interpreted as the column name (so it can contain anything except the
address@hidden' character). Anything between the address@hidden' and the end of
the
-line is defined as a comment. Within the brackets, anything before the
-first address@hidden,}' is the units (physical units, for example km/s, or
erg/s),
-anything before the second address@hidden,}' is the short type identifier (see
-below, and @ref{Numeric data types}). Finally (still within the brackets),
-any non-white characters after the second address@hidden,}' are interpreted as
the
-blank value for that column (see @ref{Blank pixels}). Note that blank
-values will be stored in the same type as the column, not as a
address@hidden floating point types, the @code{nan}, or @code{inf}
-strings (both not case-sensitive) refer to IEEE NaN (not a number) and
-infinity values respectively and will be stored as a floating point, so
-they are acceptable.}.
-
-When a formatting problem occurs (for example you have specified the wrong
-type code, see below), or the the column was already given meta-data in a
-previous comment, or the column number is larger than the actual number of
-columns in the table (the non-commented or empty lines), then the comment
-information line will be ignored.
+With this command, Make will process your @file{Makefile} and create all
+the targets (can be thousands of FITS images for example) simultaneously on
+8 threads, while fully respecting their dependencies (only building a
+file/target when its prerequisites are successfully built). Make is thus
+strongly recommended for managing scientific research where robustness,
+archiving, reproducibility and address@hidden its multi-threaded
+capabilities, Make will only re-build those targets that depend on a change
+you have made, not the whole work. For example, if you have set the
+prerequisites properly, you can easily test the changing of a parameter on
+your paper's results without having to re-do everything (which is much
+faster). This allows you to be much more productive in easily checking
+various ideas/assumptions of the different stages of your research and thus
+produce a more robust result for your exciting science.} are important.
-When a comment information line can be used, the leading and trailing white
-space characters will be stripped from all of the elements. For example in
-this line:
address@hidden table
address@hidden
-# Column 5: column name [km/s, f32,-99] Redshift as speed
address@hidden example
-The @code{NAME} field will be address@hidden name}' and the @code{TYPE}
-field will be address@hidden'. Note how all the white space characters before
-and after strings are not used, but those in the middle remained. Also,
-white space characters aren't mandatory. Hence, in the example above, the
address@hidden field will be given the value of address@hidden'.
-Except for the column number (@code{N}), the rest of the fields are
-optional. Also, the column information comments don't have to be in
-order. In other words, the information for column @mymath{N+m}
-(@mymath{m>0}) can be given in a line before column @mymath{N}. Also, you
-don't have to specify information for all columns. Those columns that don't
-have this information will be interpreted with the default settings (like
-the case above: values are double precision floating point, and the column
-has no name, unit, or comment). So these lines are all acceptable for any
-table (the first one, with nothing but the column number is redundant):
address@hidden
-# Column 5:
-# Column 1: ID [,i] The Clump ID.
-# Column 3: mag_f160w [AB mag, f] Magnitude from the F160W filter
address@hidden example
address@hidden
-The data type of the column should be specified with one of the following
-values:
-
address@hidden
address@hidden
-For a numeric column, you can use any of the numeric types (and their
-recognized identifiers) described in @ref{Numeric data types}.
address@hidden
address@hidden': for strings. The @code{N} value identifies the length of the
-string (how many characters it has). The start of the string on each row is
-the first non-delimiter character of the column that has the string
-type. The next @code{N} characters will be interpreted as a string and all
-leading and trailing white space will be removed.
address@hidden Numeric data types, Tables, Multi-threaded operations, Common
program behavior
address@hidden Numeric data types
-If the next column's characters, are closer than @code{N} characters to the
-start of the string column in that line/row, they will be considered part
-of the string column. If there is a new-line character before the ending of
-the space given to the string column (in other words, the string column is
-the last column), then reading of the string will stop, even if the
address@hidden characters are not complete yet. See @file{tests/table/table.txt}
-for one example. Therefore, the only time you have to pay attention to the
-positioning and spaces given to the string column is when it is not the
-last column in the table.
address@hidden Bit
address@hidden Type
+At the lowest level, the computer stores everything in terms of @code{1} or
address@hidden For example, each program in Gnuastro, or each astronomical image
+you take with the telescope is actually a string of millions of these zeros
+and ones. The space required to keep a zero or one is the smallest unit of
+storage, and is known as a @emph{bit}. However, understanding and
+manipulating this string of bits is extremely hard for most
+people. Therefore, we define packages of these bits along with a standard
+on how to interpret the bits in each package as a @emph{type}.
-The only limitation in this format is that trailing and leading white space
-characters will be removed from the columns that are read. In most cases,
-this is the desired behavior, but if trailing and leading white-spaces are
-critically important to your analysis, define your own starting and ending
-characters and remove them after the table has been read. For example in
-the sample table below, the two address@hidden|}' characters (which are
arbitrary)
-will remain in the value of the second column and you can remove them
-manually later. If only one of the leading or trailing white spaces is
-important for your work, you can only use one of the address@hidden|}'s.
address@hidden Byte
address@hidden Signed integer
address@hidden Unsigned integer
address@hidden Integer, Signed
+The most basic standard for reading the bits is integer numbers
+(@mymath{..., -2, -1, 0, 1, 2, ...}, more bits will give larger
+limits). The common integer types are 8, 16, 32, and 64 bits wide. For each
+width, there are two standards for reading the bits: signed and unsigned
+integers. In the former, negative numbers are allowed and in the latter,
+they aren't. The @code{unsigned} types thus have larger positive limits
+(one extra bit), but no negative value. When the context of your work
+doesn't involve negative numbers (for example counting, where negative is
+not defined), it is best to use the @code{unsigned} types. For full
+numerical range of all integer types, see below.
address@hidden
-# Column 1: ID [label, uc]
-# Column 2: Notes [no unit, str50]
-1 leading and trailing white space is ignored here 2.3442e10
-2 | but they will be preserved here | 8.2964e11
address@hidden example
+Another standard of converting a given number of bits to numbers is the
+floating point standard, this standard can approximately store any real
+number with a given precision. There are two common floating point types:
+32-bit and 64-bit, for single and double precision floating point numbers
+respectively. The former is sufficient for data with less than 8
+significant decimal digits (most astronomical data), while the latter is
+good for less than 16 significant decimal digits. The representation of
+real numbers as bits is much more complex than integers. If you are
+interested, you can start with the
address@hidden://en.wikipedia.org/wiki/Floating_point, Wikipedia article}.
address@hidden itemize
+With the conversion operators in Gnuastro's Arithmetic, you can change the
+types of data to each other, which is necessary in some contexts. For
+example the program/library, that you intend to feed the data into, only
+accepts floating point values, but you have an integer image. Another
+situation that conversion can be helpful is when you know that your data
+only has values that fit within @code{int8} or @code{uint16}. However it is
+currently formatted in the @code{float64} type. Operations involving
+floating point or larger integer types are significantly slower than
+integer or smaller-width types respectively. In the latter case, it also
+requires much more (by 8 or 4 times in the example above) storage space. So
+when you confront such situations and want to store/archive/transfter the
+data, it is best convert them to the most efficient type.
-Note that the FITS binary table standard does not define the @code{unsigned
-int} and @code{unsigned long} types, so if you want to convert your tables
-to FITS binary tables, use other types. Also, note that in the FITS ASCII
-table, there is only one integer type (@code{long}). So if you convert a
-Gnuastro plain text table to a FITS ASCII table with the @ref{Table}
-program, the type information for integers will be lost. Conversely if
-integer types are important for you, you have to manually set them when
-reading a FITS ASCII table (for example with the Table program when
-reading/converting into a file, or with the @file{gnuastro/table.h} library
-functions when reading into memory).
+The short and long names for the recognized numeric data types in Gnuastro
+are listed below. Both short and long names can be used when you want to
+specify a type. For example, as a value to the common option
address@hidden (see @ref{Input output options}), or in the information
+comment lines of @ref{Gnuastro text table format}. The ranges listed below
+are inclusive.
address@hidden @code
address@hidden u8
address@hidden uint8
+8-bit unsigned integers, range:@*
address@hidden to\ }2^8-1]} or @mymath{[0\rm{\ to\ }255]}.
address@hidden Selecting table columns, , Gnuastro text table format, Tables
address@hidden Selecting table columns
address@hidden i8
address@hidden int8
+8-bit signed integers, range:@*
address@hidden to\ }2^7-1]} or @mymath{[-128\rm{\ to\ }127]}.
-At the lowest level, the only defining aspect of a column in a table is its
-number, or position. But selecting columns purely by number is not very
-convenient and, especially when the tables are large it can be very
-frustrating and prone to errors. Hence, table file formats (for example see
address@hidden table formats}) have ways to store additional information
-about the columns (meta-data). Some of the most common pieces of
-information about each column are its @emph{name}, the @emph{units} of data
-in the it, and a @emph{comment} for longer/informal description of the
-column's data.
address@hidden u16
address@hidden uint16
+16-bit unsigned integers, range:@*
address@hidden to\ }2^{16}-1]} or @mymath{[0\rm{\ to\ }65535]}.
-To facilitate research with Gnuastro, you can select columns by matching,
-or searching in these three fields, besides the low-level column number. To
-view the full list of information on the columns in the table, you can use
-the Table program (see @ref{Table}) with the command below (replace
address@hidden with the filename of your table, if its FITS, you might
-also need to specify the HDU/extension which contains the table):
address@hidden i16
address@hidden int16
+16-bit signed integers, range:@* @mymath{[-2^{15}\rm{\ to\ }2^{15}-1]} or
address@hidden to\ }32767]}.
address@hidden
-$ asttable --information table-file
address@hidden example
address@hidden u32
address@hidden uint32
+32-bit unsigned integers, range:@* @mymath{[0\rm{\ to\ }2^{32}-1]} or
address@hidden to\ }4294967295]}.
-Gnuastro's programs need the columns for different purposes, for example in
-Crop, you specify the columns containing the central coordinates of the
-crop centers with the @option{--coordcol} option (see @ref{Crop
-options}). On the other hand, in MakeProfiles, to specify the column
-containing the profile position angles, you must use the @option{--pcol}
-option (see @ref{MakeProfiles catalog}). Thus, there can be no unified
-common option name to select columns for all programs (different columns
-have different purposes). However, when the program expects a column for a
-specific context, the option names end in the @option{col} suffix like the
-examples above. These options accept values in integer (column number), or
-string (metadata match/search) format.
address@hidden i32
address@hidden int32
+32-bit signed integers, range:@* @mymath{[-2^{31}\rm{\ to\ }2^{31}-1]} or
address@hidden to\ }2147483647]}.
-If the value can be parsed as a positive integer, it will be seen as the
-low-level column number. Note that column counting starts from 1, so if you
-ask for column 0, the respective program will abort with an error. When the
-value can't be interpreted as an a integer number, it will be seen as a
-string of characters which will be used to match/search in the table's
-meta-data. The meta-data field which the value will be compared with can be
-selected through the @option{--searchin} option, see @ref{Input output
-options}. @option{--searchin} can take three values: @code{name},
address@hidden, @code{comment}. The matching will be done following this
-convention:
address@hidden u64
address@hidden uint64
+64-bit unsigned integers, address@hidden @mymath{[0\rm{\ to\ }2^{64}-1]} or
address@hidden to\ }18446744073709551615]}.
address@hidden
address@hidden
-If the value is enclosed in two slashes (for example @command{-x/RA_/}, or
address@hidden/RA_/}, see @ref{Crop options}), then it is assumed to
-be a regular expression with the same convention as GNU AWK. GNU AWK has a
-very well written
address@hidden://www.gnu.org/software/gawk/manual/html_node/Regexp.html,
-chapter} describing regular expressions, so we we will not continue
-discussing them here. Regular expressions are a very powerful tool in
-matching text and useful in many contexts. We thus strongly encourage
-reviewing this chapter for greatly improving the quality of your work in
-many cases, not just for searching column meta-data in Gnuastro.
address@hidden i64
address@hidden int64
+64-bit signed integers, range:@* @mymath{[-2^{63}\rm{\ to\ }2^{63}-1]} or
address@hidden to\ }9223372036854775807]}.
address@hidden
-When the string isn't enclosed between address@hidden/}'s, any column that
exactly
-matches the given value in the given field will be selected.
address@hidden itemize
address@hidden f32
address@hidden float32
+32-bit (single-precision) floating point types. The maximum (minimum is its
+negative) possible value is
address@hidden Single-precision floating points can
+accurately represent a floating point number up to @mymath{\sim7.2}
+significant decimals. Given the heavy noise in astronomical data, this is
+usually more than sufficient for storing results.
-Note that in both cases, you can ignore the case of alphabetic characters
-with the @option{--ignorecase} option, see @ref{Input output options}. Also, in
-both cases, multiple columns may be selected with one call to this
-function. In this case, the order of the selected columns (with one call)
-will be the same order as they appear in the table.
address@hidden f64
address@hidden float64
+64-bit (double-precision) floating point types. The maximum (minimum is its
+negative) possible value is @mymath{\sim10^{308}}. Double-precision
+floating points can accurately represent a floating point number
address@hidden significant decimals. This is usually good for processing
+(mixing) the data internally, for example a sum of single precision data
+(and later storing the result as @code{float32}).
address@hidden table
address@hidden
address@hidden
address@hidden file formats don't recognize all types.} Some file formats
+don't recognize all the types, for example the FITS standard (see
address@hidden) does not define @code{uint64} in binary tables or images. When
+a type is not acceptable for output into a given file format, the
+respective Gnuastro program or library will let you know and abort. On the
+command-line, you can use the @ref{Arithmetic} program to convert the
+numerical type of a dataset, in the libraries, you can call
address@hidden
address@hidden cartouche
address@hidden Tables, Tessellation, Numeric data types, Common program behavior
address@hidden Tables
address@hidden Tessellation, Getting help, Tables, Common program behavior
address@hidden Tessellation
+``A table is a collection of related data held in a structured format
+within a database. It consists of columns, and rows.'' (from
+Wikipedia). Each column in the table contains the values of one property
+and each row is a collection of properties (columns) for one target
+object. For example, let's assume you have just ran MakeCatalog (see
address@hidden) on an image to measure some properties for the labeled
+regions (which might be detected galaxies for example) in the image. For
+each labeled region (detected galaxy), there will be a @emph{row} which
+groups its measured properties as @emph{columns}, one column for each
+property. One such property can be the object's magnitude, which is the sum
+of pixels with that label, or its center can be defined as the
+light-weighted average value of those pixels. Many such properties can be
+derived from the raw pixel values and their position, see @ref{Invoking
+astmkcatalog} for a long list.
-It is sometimes necessary to classify the elements in a dataset (for
-example pixels in an image) into a grid of individual, non-overlapping
-tiles. For example when background sky gradients are present in an image,
-you can define a tile grid over the image. When the tile sizes are set
-properly, the background's variation over each tile will be negligible,
-allowing you to measure (and subtract) it. In other cases (for example
-spatial domain convolution in Gnuastro, see @ref{Convolve}), it might
-simply be for speed of processing: each tile can be processed independently
-on a separate CPU thread. In the arts and mathematics, this process is
-formally known as @url{https://en.wikipedia.org/wiki/Tessellation,
-tessellation}.
+As a summary, for each labeled region (or, galaxy) we have one @emph{row}
+and for each measured property we have one @emph{column}. This high-level
+structure is usually the first step for higher-level analysis, for example
+finding the stellar mass or photometric redshift from magnitudes in
+multiple colors. Thus, tables are not just outputs of programs, in fact it
+is much more common for tables to be inputs of programs. For example, to
+make a mock galaxy image, you need to feed in the properties of each galaxy
+into @ref{MakeProfiles} for it do the inverse of the process above and make
+a simulated image from a catalog, see @ref{Sufi simulates a detection}. In
+other cases, you can feed a table into @ref{Crop} and it will crop out
+regions centered on the positions within the table, see @ref{Hubble
+visually checks and classifies his catalog}. So to end this relatively long
+introduction, tables play a very important role in astronomy, or generally
+all branches of data analysis.
-The size of the regular tiles (in units of data-elements, or pixels in an
-image) can be defined with the @option{--tilesize} option. It takes
-multiple numbers (separated by a comma) which will be the length along the
-respective dimension (in FORTRAN/FITS dimension order). Divisions are also
-acceptable, but must result in an integer. For example
address@hidden,40} can be used for an image (a 2D dataset). The
-regular tile size along the first FITS axis (horizontal when viewed in SAO
-ds9) will be 30 pixels and along the second it will be 40 pixels. Ideally,
address@hidden should be selected such that all tiles in the image
-have exactly the same size. In other words, that the dataset length in each
-dimension is divisible by the tile size in that dimension.
+In @ref{Recognized table formats} the currently recognized table formats in
+Gnuastro are discussed. You can use any of these tables as input or ask for
+them to be built as output. The most common type of table format is a
+simple plain text file with each row on one line and columns separated by
+white space characters, this format is easy to read/write by eye/hand. To
+give it the full functionality of more specific table types like the FITS
+tables, Gnuastro has a special convention which you can use to give each
+column a name, type, unit, and comments, while still being readable by
+other plain text table readers. This convention is described in
address@hidden text table format}.
-However, this is not always possible: the dataset can be any size and every
-pixel in it is valuable. In such cases, Gnuastro will look at the
-significance of the remainder length, if it is not significant (for example
-one or two pixels), then it will just increase the size of the first tile
-in the respective dimension and allow the rest of the tiles to have the
-required size. When the remainder is significant (for example one pixel
-less than the size along that dimension), the remainder will be added to
-one regular tile's size and the large tile will be cut in half and put in
-the two ends of the grid/tessellation. In this way, all the tiles in the
-central regions of the dataset will have the regular tile sizes and the
-tiles on the edge will be slightly larger/smaller depending on the
-remainder significance. The fraction which defines the remainder
-significance along all dimensions can be set through
address@hidden
+When tables are input to a program, the program reading it needs to know
+which column(s) it should use for its desired purposes. Gnuastro's programs
+all follow a similar convention, on the way you can select columns in a
+table. They are thoroughly discussed in @ref{Selecting table columns}.
-The best tile size is directly related to the spatial properties of the
-property you want to study (for example, gradient on the image). In
-practice we assume that the gradient is not present over each tile. So if
-there is a strong gradient (for example in long wavelength ground based
-images) or the image is of a crowded area where there isn't too much blank
-area, you have to choose a smaller tile size. A larger mesh will give more
-pixels and and so the scatter in the results will be less (better
-statistics).
address@hidden CCD
address@hidden Amplifier
address@hidden Bias current
address@hidden Subaru Telescope
address@hidden Hyper Suprime-Cam
address@hidden Hubble Space Telescope (HST)
-For raw image processing, a single tessellation/grid is not sufficient. Raw
-images are the unprocessed outputs of the camera detectors. Modern
-detectors usually have multiple readout channels each with its own
-amplifier. For example the Hubble Space Telescope Advanced Camera for
-Surveys (ACS) has four amplifiers over its full detector area dividing the
-square field of view to four smaller squares. Ground based image detectors
-are not exempt, for example each CCD of Subaru Telescope's Hyper
-Suprime-Cam camera (which has 104 CCDs) has four amplifiers, but they have
-the same height of the CCD and divide the width by four parts.
address@hidden
+* Recognized table formats:: Table formats that are recognized in Gnuastro.
+* Gnuastro text table format:: Gnuastro's convention plain text tables.
+* Selecting table columns:: Identify/select certain columns from a table
address@hidden menu
address@hidden Channel
-The bias current on each amplifier is different, and initial bias
-subtraction is not perfect. So even after subtracting the measured bias
-current, you can usually still identify the boundaries of different
-amplifiers by eye. See Figure 11(a) in Akhlaghi and Ichikawa (2015) for an
-example. This results in the final reduced data to have non-uniform
-amplifier-shaped regions with higher or lower background flux values. Such
-systematic biases will then propagate to all subsequent measurements we do
-on the data (for example photometry and subsequent stellar mass and star
-formation rate measurements in the case of galaxies).
address@hidden Recognized table formats, Gnuastro text table format, Tables,
Tables
address@hidden Recognized table formats
-Therefore an accurate analysis requires a two layer tessellation: the top
-layer contains larger tiles, each covering one amplifier channel. For
-clarity we'll call these larger tiles ``channels''. The number of channels
-along each dimension is defined through the @option{--numchannels}. Each
-channel is then covered by its own individual smaller tessellation (with
-tile sizes determined by the @option{--tilesize} option). This will allow
-independent analysis of two adjacent pixels from different channels if
-necessary. If the image is processed or the detector only has one
-amplifier, you can set the number of channels in both dimension to 1.
+The list of table formats that Gnuastro can currently read from and write
+to are described below. Each has their own advantage and disadvantages, so a
+short review of the format is also provided to help you make the best
+choice based on how you want to define your input tables or later use your
+output tables.
-The final tessellation can be inspected on the image with the
address@hidden option that is available to all programs which use
-tessellation for localized operations. When this option is called, a FITS
-file with a @file{_tiled.fits} suffix will be created along with the
-outputs, see @ref{Automatic output}. Each pixel in this image has the
-number of the tile that covers it. If the number of channels in any
-dimension are larger than unity, you will notice that the tile IDs are
-defined such that the first channels is covered first, then the second and
-so on. For the full list of processing-related common options (including
-tessellation options), please see @ref{Processing options}.
address@hidden @asis
address@hidden Plain text table
+This is the most basic and simplest way to create, view, or edit the table
+by hand on a text editor. The other formats described below are less
+eye-friendly and have a more formal structure (for easier computer
+readability). It is fully described in @ref{Gnuastro text table format}.
address@hidden FITS Tables
address@hidden Tables FITS
address@hidden ASCII table, FITS
address@hidden FITS ASCII tables
+The FITS ASCII table extension is fully in ASCII encoding and thus easily
+readable on any text editor (assuming it is the only extension in the FITS
+file). If the FITS file also contains binary extensions (for example an
+image or binary table extensions), then there will be many hard to print
+characters. The FITS ASCII format doesn't have new line characters to
+separate rows. In the FITS ASCII table standard, each row is defined as a
+fixed number of characters (value to the @code{NAXIS1} keyword), so to
+visually inspect it properly, you would have to adjust your text editor's
+width to this value. All columns start at given character positions and
+have a fixed width (number of characters).
+Numbers in a FITS ASCII table are printed into ASCII format, they are not
+in binary (that the CPU uses). Hence, they can take a larger space in
+memory, loose their precision, and take longer to read into memory. If you
+are dealing with integer type columns (see @ref{Numeric data types}),
+another issue with FITS ASCII tables is that the type information for the
+column will be lost (there is only one integer type in FITS ASCII
+tables). One problem with the binary format on the other hand is that it
+isn't portable (different CPUs/compilers) have different standards for
+translating the zeros and ones. But since ASCII characters are defined on a
+byte and are well recognized, they are better for portability on those
+various systems. Gnuastro's plain text table format described below is much
+more portable and easier to read/write/interpret by humans manually.
address@hidden Getting help, Automatic output, Tessellation, Common program
behavior
address@hidden Getting help
+Generally, as the name implies, this format is useful for when your table
+mainly contains ASCII columns (for example file names, or
+descriptions). They can be useful when you need to include columns with
+structured ASCII information along with other extensions in one FITS
+file. In such cases, you can also consider header keywords (see
address@hidden).
address@hidden Help
address@hidden Book formats
address@hidden Remembering options
address@hidden Convenient book formats
-Probably the first time you read this book, it is either in the PDF
-or HTML formats. These two formats are very convenient for when you
-are not actually working, but when you are only reading. Later on,
-when you start to use the programs and you are deep in the middle of
-your work, some of the details will inevitably be forgotten. Going to
-find the PDF file (printed or digital) or the HTML webpage is a major
-distraction.
address@hidden Binary table, FITS
address@hidden FITS binary tables
+The FITS binary table is the FITS standard's solution to the issues
+discussed with keeping numbers in ASCII format as described under the FITS
+ASCII table title above. Only columns defined as a string type (a string of
+ASCII characters) are readable in a text editor. The portability problem
+with binary formats discussed above is mostly solved thanks to the
+portability of CFITSIO (see @ref{CFITSIO}) and the very long history of the
+FITS format which has been widely used since the 1970s.
address@hidden Online help
address@hidden Command-line help
-GNU software have a very unique set of tools for aiding your memory on
-the command-line, where you are working, depending how much of it you
-need to remember. In the past, such command-line help was known as
-``online'' help, because they were literally provided to you `on'
-the command `line'. However, nowadays the word ``online'' refers to
-something on the internet, so that term will not be used. With this
-type of help, you can resume your exciting research without taking
-your hands off the keyboard.
+In the case of most numbers, storing them in binary format is more memory
+efficient than ASCII format. For example, to store @code{-25.72034} in
+ASCII format, you need 9 bytes/characters. But if you keep this same number
+(to the approximate precision possible) as a 4-byte (32-bit) floating point
+number, you can keep/transmit it with less than half the amount of
+memory. When catalogs contain thousands/millions of rows in tens/hundreds
+of columns, this can lead to significant improvements in memory/band-width
+usage. Moreover, since the CPU does its operations in the binary formats,
+reading the table in and writing it out is also much faster than an ASCII
+table.
address@hidden Installed help methods
-Another major advantage of such command-line based help routines is
-that they are installed with the software in your computer, therefore
-they are always in sync with the executable you are actually
-running. Three of them are actually part of the executable. You don't
-have to worry about the version of the book or program. If you rely
-on external help (a PDF in your personal print or digital archive or
-HTML from the official webpage) you have to check to see if their
-versions fit with your installed program.
+When you are dealing with integer numbers, the compression ratio can be
+even better, for example if you know all of the values in a column are
+positive and less than @code{255}, you can use the @code{unsigned char}
+type which only takes one byte! If they are between @code{-128} and
address@hidden, then you can use the (signed) @code{char} type. So if you are
+thoughtful about the limits of your integer columns, you can greatly reduce
+the size of your file and also the speed at which it is read/written. This
+can be very useful when sharing your results with collaborators or
+publishing them. To decrease the file size even more you can name your
+output as ending in @file{.fits.gz} so it is also compressed after
+creation. Just note that compression/decompressing is CPU intensive and can
+slow down the writing/reading of the file.
-If you only need to remember the short or long names of the options,
address@hidden is advised. If it is what the options do, then
address@hidden is a great tool. Man pages are also provided for those
-who are use to this older system of documentation. This full book is
-also available to you on the command-line in Info format. If none of
-these seems to resolve the problems, there is a mailing list which
-enables you to get in touch with experienced Gnuastro users. In the
-subsections below each of these methods are reviewed.
+Fortunately the FITS Binary table format also accepts ASCII strings as
+column types (along with the various numerical types). So your dataset can
+also contain non-numerical columns.
address@hidden table
@menu
-* --usage:: View option names and value formats.
-* --help:: List all options with description.
-* Man pages:: Man pages generated from --help.
-* Info:: View complete book in terminal.
-* help-gnuastro mailing list:: Contacting experienced users.
+* Gnuastro text table format:: Reading plain text tables
@end menu
address@hidden --usage, --help, Getting help, Getting help
address@hidden @option{--usage}
address@hidden --usage
address@hidden Usage pattern
address@hidden Mandatory arguments
address@hidden Optional and mandatory tokens
-If you give this option, the program will not run. It will only print a
-very concise message showing the options and arguments. Everything within
-square brackets (@option{[]}) is optional. For example here are the first
-and last two lines of Crop's @option{--usage} is shown:
address@hidden Gnuastro text table format, Selecting table columns, Recognized
table formats, Tables
address@hidden Gnuastro text table format
+
+Plain text files are the most generic, portable, and easiest way to
+(manually) create, (visually) inspect, or (manually) edit a table. In this
+format, the ending of a row is defined by the new-line character (a line on
+a text editor). So when you view it on a text editor, every row will occupy
+one line. The delimiters (or characters separating the columns) are white
+space characters (space, horizontal tab, vertical tab) and a comma
+(@key{,}). The only further requirement is that all rows/lines must have
+the same number of columns.
+
+The columns don't have to be exactly under each other and the rows can be
+arbitrarily long with different lengths. For example the following contents
+in a file would be interpreted as a table with 4 columns and 2 rows, with
+each element interpreted as a @code{double} type (see @ref{Numeric data
+types}).
@example
-$ astcrop --usage
-Usage: astcrop [-Do?IPqSVW] [-d INT] [-h INT] [-r INT] [-w INT]
- [-x INT] [-y INT] [-c INT] [-p STR] [-N INT] [--deccol=INT]
- ....
- [--setusrconf] [--usage] [--version] [--wcsmode]
- [ASCIIcatalog] FITSimage(s).fits
+1 2.234948 128 39.8923e8
+2 , 4.454 792 72.98348e7
@end example
-There are no explanations on the options, just their short and long
-names shown separately. After the program name, the short format of
-all the options that don't require a value (on/off options) is
-displayed. Those that do require a value then follow in separate
-brackets, each displaying the format of the input they want, see
address@hidden Since all options are optional, they are shown in
-square brackets, but arguments can also be optional. For example in
-this example, a catalog name is optional and is only required in some
-modes. This is a standard method of displaying optional arguments for
-all GNU software.
-
address@hidden --help, Man pages, --usage, Getting help
address@hidden @option{--help}
+However, the example above has no other information about the columns (it
+is just raw data, with no meta-data). To use this table, you have to
+remember what the numbers in each column represent. Also, when you want to
+select columns, you have to count their position within the table. This can
+become frustrating and prone to bad errors (getting the columns wrong)
+especially as the number of columns increase. It is also bad for sending to
+a colleague, because they will find it hard to remember/use the columns
+properly.
address@hidden --help
-If the command-line includes this option, the program will not be
-run. It will print a complete list of all available options along with
-a short explanation. The options are also grouped by their
-context. Within each context, the options are sorted
-alphabetically. Since the options are shown in detail afterwards, the
-first line of the @option{--help} output shows the arguments and if
-they are optional or not, similar to @ref{--usage}.
+To solve these problems in Gnuastro's programs/libraries you aren't limited
+to using the column's number, see @ref{Selecting table columns}. If the
+columns have names, units, or comments you can also select your columns
+based on searches/matches in these fields, for example see @ref{Table}.
+Also, in this manner, you can't guide the program reading the table on how
+to read the numbers. As an example, the first and third columns above can
+be read as integer types: the first column might be an ID and the third can
+be the number of pixels an object occupies in an image. So there is no need
+to read these to columns as a @code{double} type (which takes more memory,
+and is slower).
-In the @option{--help} output of all programs in Gnuastro, the
-options for each program are classified based on context. The first
-two contexts are always options to do with the input and output
-respectively. For example input image extensions or supplementary
-input files for the inputs. The last class of options is also fixed in
-all of Gnuastro, it shows operating mode options. Most of these
-options are already explained in @ref{Operating mode options}.
+In the bare-minimum example above, you also can't use strings of
+characters, for example the names of filters, or some other identifier that
+includes non-numerical characters. In the absence of any information, only
+numbers can be read robustly. Assuming we read columns with non-numerical
+characters as string, there would still be the problem that the strings
+might contain space (or any delimiter) character for some rows. So, each
+`word' in the string will be interpreted as a column and the program will
+abort with an error that the rows don't have the same number of columns.
address@hidden Long outputs
address@hidden Redirection of output
address@hidden Command-line, long outputs
-The help message will sometimes be longer than the vertical size of
-your terminal. If you are using a graphical user interface terminal
-emulator, you can scroll the terminal with your mouse, but we promised
-no mice distractions! So here are some suggestions:
+To correct for these limitations, Gnuastro defines the following convention
+for storing the table meta-data along with the raw data in one plain text
+file. The format is primarily designed for ease of reading/writing by
+eye/fingers, but is also structured enough to be read by a program.
address@hidden
address@hidden
address@hidden Scroll command-line
address@hidden Command-line scroll
address@hidden @key{Shift + PageUP} and @key{Shift + PageDown}
address@hidden + PageUP} to scroll up and @key{Shift + PageDown} to scroll
-down. For most help output this should be enough. The problem is that
-it is limited by the number of lines that your terminal keeps in
-memory and that you can't scroll by lines, only by whole screens.
+When the first non-white character in a line is @key{#}, or there are no
+non-white characters in it, then the line will not be considered as a row
+of data in the table (this is a pretty standard convention in many
+programs, and higher level languages). In the former case, the line is
+interpreted as a @emph{comment}. If the comment line starts with address@hidden
+Column N:}', then it is assumed to contain information about column
address@hidden (a number, counting from 1). Comment lines that don't start with
+this pattern are ignored and you can use them to include any further
+information you want to store with the table in the text file. A column
+information comment is assumed to have the following format:
address@hidden
address@hidden Pipe
address@hidden @command{less}
-Pipe to @command{less}. A pipe is a form of shell re-direction. The
address@hidden tool in Unix-like systems was made exactly for such
-outputs of any length. You can pipe (@command{|}) the output of any
-program that is longer than the screen to it and then you can scroll
-through (up and down) with its many tools. For example:
@example
-$ astnoisechisel --help | less
+# Column N: NAME [UNIT, TYPE, BLANK] COMMENT
@end example
+
address@hidden NaN
@noindent
-Once you have gone through the text, you can quit @command{less} by
-pressing the @key{q} key.
+Any sequence of characters between address@hidden:}' and address@hidden' will
be
+interpreted as the column name (so it can contain anything except the
address@hidden' character). Anything between the address@hidden' and the end of
the
+line is defined as a comment. Within the brackets, anything before the
+first address@hidden,}' is the units (physical units, for example km/s, or
erg/s),
+anything before the second address@hidden,}' is the short type identifier (see
+below, and @ref{Numeric data types}). Finally (still within the brackets),
+any non-white characters after the second address@hidden,}' are interpreted as
the
+blank value for that column (see @ref{Blank pixels}). Note that blank
+values will be stored in the same type as the column, not as a
address@hidden floating point types, the @code{nan}, or @code{inf}
+strings (both not case-sensitive) refer to IEEE NaN (not a number) and
+infinity values respectively and will be stored as a floating point, so
+they are acceptable.}.
+When a formatting problem occurs (for example you have specified the wrong
+type code, see below), or the the column was already given meta-data in a
+previous comment, or the column number is larger than the actual number of
+columns in the table (the non-commented or empty lines), then the comment
+information line will be ignored.
+
+When a comment information line can be used, the leading and trailing white
+space characters will be stripped from all of the elements. For example in
+this line:
address@hidden
address@hidden Save output to file
address@hidden Redirection of output
-Redirect to a file. This is a less convenient way, because you will
-then have to open the file in a text editor! You can do this with the
-shell redirection tool (@command{>}):
@example
-$ astnoisechisel --help > filename.txt
+# Column 5: column name [km/s, f32,-99] Redshift as speed
@end example
address@hidden itemize
address@hidden GNU Grep
address@hidden Searching text
address@hidden Command-line searching text
-In case you have a special keyword you are looking for in the help, you
-don't have to go through the full list. GNU Grep is made for this job. For
-example if you only want the list of options whose @option{--help} output
-contains the word ``axis'' in Crop, you can run the following command:
+The @code{NAME} field will be address@hidden name}' and the @code{TYPE}
+field will be address@hidden'. Note how all the white space characters before
+and after strings are not used, but those in the middle remained. Also,
+white space characters aren't mandatory. Hence, in the example above, the
address@hidden field will be given the value of address@hidden'.
address@hidden
-$ astcrop --help | grep axis
address@hidden example
+Except for the column number (@code{N}), the rest of the fields are
+optional. Also, the column information comments don't have to be in
+order. In other words, the information for column @mymath{N+m}
+(@mymath{m>0}) can be given in a line before column @mymath{N}. Also, you
+don't have to specify information for all columns. Those columns that don't
+have this information will be interpreted with the default settings (like
+the case above: values are double precision floating point, and the column
+has no name, unit, or comment). So these lines are all acceptable for any
+table (the first one, with nothing but the column number is redundant):
address@hidden @code{ARGP_HELP_FMT}
address@hidden Argp argument parser
address@hidden Customize @option{--help} output
address@hidden @option{--help} output customization
-If the output of this option does not fit nicely within the confines
-of your terminal, GNU does enable you to customize its output through
-the environment variable @code{ARGP_HELP_FMT}, you can set various
-parameters which specify the formatting of the help messages. For
-example if your terminals are wider than 70 spaces (say 100) and you
-feel there is too much empty space between the long options and the
-short explanation, you can change these formats by giving values to
-this environment variable before running the program with the
address@hidden output. You can define this environment variable in
-this manner:
@example
-$ export ARGP_HELP_FMT=rmargin=100,opt-doc-col=20
+# Column 5:
+# Column 1: ID [,i] The Clump ID.
+# Column 3: mag_f160w [AB mag, f] Magnitude from the F160W filter
@end example
address@hidden @file{.bashrc}
-This will affect all GNU programs using GNU C library's @file{argp.h}
-facilities as long as the environment variable is in memory. You can
-see the full list of these formatting parameters in the ``Argp User
-Customization'' part of the GNU C library manual. If you are more
-comfortable to read the @option{--help} outputs of all GNU software in
-your customized format, you can add your customization (similar to
-the line above, without the @command{$} sign) to your @file{~/.bashrc}
-file. This is a standard option for all GNU software.
address@hidden Man pages, Info, --help, Getting help
address@hidden Man pages
address@hidden Man pages
-Man pages were the Unix method of providing command-line documentation
-to a program. With GNU Info, see @ref{Info} the usage of this method
-of documentation is highly discouraged. This is because Info provides
-a much more easier to navigate and read environment.
address@hidden
+The data type of the column should be specified with one of the following
+values:
+
address@hidden
address@hidden
+For a numeric column, you can use any of the numeric types (and their
+recognized identifiers) described in @ref{Numeric data types}.
address@hidden
address@hidden': for strings. The @code{N} value identifies the length of the
+string (how many characters it has). The start of the string on each row is
+the first non-delimiter character of the column that has the string
+type. The next @code{N} characters will be interpreted as a string and all
+leading and trailing white space will be removed.
+
+If the next column's characters, are closer than @code{N} characters to the
+start of the string column in that line/row, they will be considered part
+of the string column. If there is a new-line character before the ending of
+the space given to the string column (in other words, the string column is
+the last column), then reading of the string will stop, even if the
address@hidden characters are not complete yet. See @file{tests/table/table.txt}
+for one example. Therefore, the only time you have to pay attention to the
+positioning and spaces given to the string column is when it is not the
+last column in the table.
+
+The only limitation in this format is that trailing and leading white space
+characters will be removed from the columns that are read. In most cases,
+this is the desired behavior, but if trailing and leading white-spaces are
+critically important to your analysis, define your own starting and ending
+characters and remove them after the table has been read. For example in
+the sample table below, the two address@hidden|}' characters (which are
arbitrary)
+will remain in the value of the second column and you can remove them
+manually later. If only one of the leading or trailing white spaces is
+important for your work, you can only use one of the address@hidden|}'s.
-However, some operating systems require a man page for packages that
-are installed and some people are still used to this method of command
-line help. So the programs in Gnuastro also have Man pages which are
-automatically generated from the outputs of @option{--version} and
address@hidden using the GNU help2man program. So if you run
@example
-$ man programname
+# Column 1: ID [label, uc]
+# Column 2: Notes [no unit, str50]
+1 leading and trailing white space is ignored here 2.3442e10
+2 | but they will be preserved here | 8.2964e11
@end example
address@hidden
-You will be provided with a man page listing the options in the
-standard manner.
-
address@hidden itemize
+Note that the FITS binary table standard does not define the @code{unsigned
+int} and @code{unsigned long} types, so if you want to convert your tables
+to FITS binary tables, use other types. Also, note that in the FITS ASCII
+table, there is only one integer type (@code{long}). So if you convert a
+Gnuastro plain text table to a FITS ASCII table with the @ref{Table}
+program, the type information for integers will be lost. Conversely if
+integer types are important for you, you have to manually set them when
+reading a FITS ASCII table (for example with the Table program when
+reading/converting into a file, or with the @file{gnuastro/table.h} library
+functions when reading into memory).
address@hidden Info, help-gnuastro mailing list, Man pages, Getting help
address@hidden Info
address@hidden Selecting table columns, , Gnuastro text table format, Tables
address@hidden Selecting table columns
address@hidden GNU Info
address@hidden Command-line, viewing full book
-Info is the standard documentation format for all GNU software. It is
-a very useful command-line document viewing format, fully equipped
-with links between the various pages and menus and search
-capabilities. As explained before, the best thing about it is that it
-is available for you the moment you need to refresh your memory on any
-command-line tool in the middle of your work without having to take
-your hands off the keyboard. This complete book is available in Info
-format and can be accessed from anywhere on the command-line.
+At the lowest level, the only defining aspect of a column in a table is its
+number, or position. But selecting columns purely by number is not very
+convenient and, especially when the tables are large it can be very
+frustrating and prone to errors. Hence, table file formats (for example see
address@hidden table formats}) have ways to store additional information
+about the columns (meta-data). Some of the most common pieces of
+information about each column are its @emph{name}, the @emph{units} of data
+in the it, and a @emph{comment} for longer/informal description of the
+column's data.
-To open the Info format of any installed programs or library on your
-system which has an Info format book, you can simply run the command
-below (change @command{executablename} to the executable name of the
-program or library):
+To facilitate research with Gnuastro, you can select columns by matching,
+or searching in these three fields, besides the low-level column number. To
+view the full list of information on the columns in the table, you can use
+the Table program (see @ref{Table}) with the command below (replace
address@hidden with the filename of your table, if its FITS, you might
+also need to specify the HDU/extension which contains the table):
@example
-$ info executablename
+$ asttable --information table-file
@end example
address@hidden
address@hidden Learning GNU Info
address@hidden GNU software documentation
-In case you are not already familiar with it, run @command{$ info
-info}. It does a fantastic job in explaining all its capabilities its
-self. It is very short and you will become sufficiently fluent in
-about half an hour. Since all GNU software documentation is also
-provided in Info, your whole GNU/Linux life will significantly
-improve.
+Gnuastro's programs need the columns for different purposes, for example in
+Crop, you specify the columns containing the central coordinates of the
+crop centers with the @option{--coordcol} option (see @ref{Crop
+options}). On the other hand, in MakeProfiles, to specify the column
+containing the profile position angles, you must use the @option{--pcol}
+option (see @ref{MakeProfiles catalog}). Thus, there can be no unified
+common option name to select columns for all programs (different columns
+have different purposes). However, when the program expects a column for a
+specific context, the option names end in the @option{col} suffix like the
+examples above. These options accept values in integer (column number), or
+string (metadata match/search) format.
address@hidden GNU Emacs
address@hidden GNU C library
-Once you've become an efficient navigator in Info, you can go to any
-part of this book or any other GNU software or library manual, no
-matter how long it is, in a matter of seconds. It also blends nicely
-with GNU Emacs (a text editor) and you can search manuals while you
-are writing your document or programs without taking your hands off
-the keyboard, this is most useful for libraries like the GNU C
-library. To be able to access all the Info manuals installed in your
-GNU/Linux within Emacs, type @key{Ctrl-H + i}.
+If the value can be parsed as a positive integer, it will be seen as the
+low-level column number. Note that column counting starts from 1, so if you
+ask for column 0, the respective program will abort with an error. When the
+value can't be interpreted as an a integer number, it will be seen as a
+string of characters which will be used to match/search in the table's
+meta-data. The meta-data field which the value will be compared with can be
+selected through the @option{--searchin} option, see @ref{Input output
+options}. @option{--searchin} can take three values: @code{name},
address@hidden, @code{comment}. The matching will be done following this
+convention:
-To see this whole book from the beginning in Info, you can run
address@hidden
address@hidden
+If the value is enclosed in two slashes (for example @command{-x/RA_/}, or
address@hidden/RA_/}, see @ref{Crop options}), then it is assumed to
+be a regular expression with the same convention as GNU AWK. GNU AWK has a
+very well written
address@hidden://www.gnu.org/software/gawk/manual/html_node/Regexp.html,
+chapter} describing regular expressions, so we we will not continue
+discussing them here. Regular expressions are a very powerful tool in
+matching text and useful in many contexts. We thus strongly encourage
+reviewing this chapter for greatly improving the quality of your work in
+many cases, not just for searching column meta-data in Gnuastro.
address@hidden
-$ info gnuastro
address@hidden example
address@hidden
+When the string isn't enclosed between address@hidden/}'s, any column that
exactly
+matches the given value in the given field will be selected.
address@hidden itemize
address@hidden
-If you run Info with the particular program executable name, for
-example @file{astcrop} or @file{astnoisechisel}:
+Note that in both cases, you can ignore the case of alphabetic characters
+with the @option{--ignorecase} option, see @ref{Input output options}. Also, in
+both cases, multiple columns may be selected with one call to this
+function. In this case, the order of the selected columns (with one call)
+will be the same order as they appear in the table.
address@hidden
-$ info astprogramname
address@hidden example
address@hidden
-you will be taken to the section titled ``Invoking ProgramName'' which
-explains the inputs and outputs along with the command-line options for
-that program. Finally, if you run Info with the official program name, for
-example Crop or NoiseChisel:
address@hidden
-$ info ProgramName
address@hidden example
address@hidden
-you will be taken to the top section which introduces the
-program. Note that in all cases, Info is not case sensitive.
address@hidden Tessellation, Automatic output, Tables, Common program behavior
address@hidden Tessellation
+It is sometimes necessary to classify the elements in a dataset (for
+example pixels in an image) into a grid of individual, non-overlapping
+tiles. For example when background sky gradients are present in an image,
+you can define a tile grid over the image. When the tile sizes are set
+properly, the background's variation over each tile will be negligible,
+allowing you to measure (and subtract) it. In other cases (for example
+spatial domain convolution in Gnuastro, see @ref{Convolve}), it might
+simply be for speed of processing: each tile can be processed independently
+on a separate CPU thread. In the arts and mathematics, this process is
+formally known as @url{https://en.wikipedia.org/wiki/Tessellation,
+tessellation}.
address@hidden help-gnuastro mailing list, , Info, Getting help
address@hidden help-gnuastro mailing list
+The size of the regular tiles (in units of data-elements, or pixels in an
+image) can be defined with the @option{--tilesize} option. It takes
+multiple numbers (separated by a comma) which will be the length along the
+respective dimension (in FORTRAN/FITS dimension order). Divisions are also
+acceptable, but must result in an integer. For example
address@hidden,40} can be used for an image (a 2D dataset). The
+regular tile size along the first FITS axis (horizontal when viewed in SAO
+ds9) will be 30 pixels and along the second it will be 40 pixels. Ideally,
address@hidden should be selected such that all tiles in the image
+have exactly the same size. In other words, that the dataset length in each
+dimension is divisible by the tile size in that dimension.
address@hidden help-gnuastro mailing list
address@hidden Mailing list: help-gnuastro
-Gnuastro maintains the help-gnuastro mailing list for users to ask any
-questions related to Gnuastro. The experienced Gnuastro users and some
-of its developers are subscribed to this mailing list and your email
-will be sent to them immediately. However, when contacting this
-mailing list please have in mind that they are possibly very busy and
-might not be able to answer immediately.
+However, this is not always possible: the dataset can be any size and every
+pixel in it is valuable. In such cases, Gnuastro will look at the
+significance of the remainder length, if it is not significant (for example
+one or two pixels), then it will just increase the size of the first tile
+in the respective dimension and allow the rest of the tiles to have the
+required size. When the remainder is significant (for example one pixel
+less than the size along that dimension), the remainder will be added to
+one regular tile's size and the large tile will be cut in half and put in
+the two ends of the grid/tessellation. In this way, all the tiles in the
+central regions of the dataset will have the regular tile sizes and the
+tiles on the edge will be slightly larger/smaller depending on the
+remainder significance. The fraction which defines the remainder
+significance along all dimensions can be set through
address@hidden
address@hidden Mailing list archives
address@hidden @code{help-gnuastro@@gnu.org}
-To ask a question from this mailing list, send a mail to
address@hidden@@gnu.org}. Anyone can view the mailing list
-archives at @url{http://lists.gnu.org/archive/html/help-gnuastro/}. It
-is best that before sending a mail, you search the archives to see if
-anyone has asked a question similar to yours. If you want to make a
-suggestion or report a bug, please don't send a mail to this mailing
-list. We have other mailing lists and tools for those purposes, see
address@hidden a bug} or @ref{Suggest new feature}.
+The best tile size is directly related to the spatial properties of the
+property you want to study (for example, gradient on the image). In
+practice we assume that the gradient is not present over each tile. So if
+there is a strong gradient (for example in long wavelength ground based
+images) or the image is of a crowded area where there isn't too much blank
+area, you have to choose a smaller tile size. A larger mesh will give more
+pixels and and so the scatter in the results will be less (better
+statistics).
+
address@hidden CCD
address@hidden Amplifier
address@hidden Bias current
address@hidden Subaru Telescope
address@hidden Hyper Suprime-Cam
address@hidden Hubble Space Telescope (HST)
+For raw image processing, a single tessellation/grid is not sufficient. Raw
+images are the unprocessed outputs of the camera detectors. Modern
+detectors usually have multiple readout channels each with its own
+amplifier. For example the Hubble Space Telescope Advanced Camera for
+Surveys (ACS) has four amplifiers over its full detector area dividing the
+square field of view to four smaller squares. Ground based image detectors
+are not exempt, for example each CCD of Subaru Telescope's Hyper
+Suprime-Cam camera (which has 104 CCDs) has four amplifiers, but they have
+the same height of the CCD and divide the width by four parts.
+
address@hidden Channel
+The bias current on each amplifier is different, and initial bias
+subtraction is not perfect. So even after subtracting the measured bias
+current, you can usually still identify the boundaries of different
+amplifiers by eye. See Figure 11(a) in Akhlaghi and Ichikawa (2015) for an
+example. This results in the final reduced data to have non-uniform
+amplifier-shaped regions with higher or lower background flux values. Such
+systematic biases will then propagate to all subsequent measurements we do
+on the data (for example photometry and subsequent stellar mass and star
+formation rate measurements in the case of galaxies).
+
+Therefore an accurate analysis requires a two layer tessellation: the top
+layer contains larger tiles, each covering one amplifier channel. For
+clarity we'll call these larger tiles ``channels''. The number of channels
+along each dimension is defined through the @option{--numchannels}. Each
+channel is then covered by its own individual smaller tessellation (with
+tile sizes determined by the @option{--tilesize} option). This will allow
+independent analysis of two adjacent pixels from different channels if
+necessary. If the image is processed or the detector only has one
+amplifier, you can set the number of channels in both dimension to 1.
+
+The final tessellation can be inspected on the image with the
address@hidden option that is available to all programs which use
+tessellation for localized operations. When this option is called, a FITS
+file with a @file{_tiled.fits} suffix will be created along with the
+outputs, see @ref{Automatic output}. Each pixel in this image has the
+number of the tile that covers it. If the number of channels in any
+dimension are larger than unity, you will notice that the tile IDs are
+defined such that the first channels is covered first, then the second and
+so on. For the full list of processing-related common options (including
+tessellation options), please see @ref{Processing options}.
address@hidden Automatic output, Output FITS files, Getting help, Common
program behavior
+
address@hidden Automatic output, Output FITS files, Tessellation, Common
program behavior
@section Automatic output
address@hidden Standard input
@cindex Automatic output file names
@cindex Output file names, automatic
@cindex Setting output file names automatically
-All the programs in Gnuastro are designed such that specifying an
-output file or directory (based on the program context) is optional.
-The outputs of most programs are automatically found based on the
-input and what the program does. For example when you are converting a
-FITS image named @file{FITSimage.fits} to a JPEG image, the JPEG image
-will be saved in @file{FITSimage.jpg}.
+All the programs in Gnuastro are designed such that specifying an output
+file or directory (based on the program context) is optional. When no
+output name is explicitly given (with @option{--output}, see @ref{Input
+output options}), the programs will automatically set an output name based
+on the input name(s) and what the program does. For example when you are
+using ConvertType to save FITS image named @file{dataset.fits} to a JPEG
+image and don't specify a name for it, the JPEG output file will be name
address@hidden When the input is from the standard input (for example
+a pipe, see @ref{Standard input}), and @option{--output} isn't given, the
+output name will be the program's name (for example
address@hidden).
@vindex --keepinputdir
-Another very important part of the automatic output generation is that
-all the directory information of the input file name is stripped off
-of it. This feature can be disabled with the @option{--keepinputdir}
-option, see @ref{Common options}. It is the default because
-astronomical data are usually very large and organized specially with
-special file names. In some cases, the user might not have write
-permissions in those directories. In fact, even if the data is stored
-on your own computer, it is advised to only grant write permissions to
-the super user or root. This way, you won't accidentally delete or
-modify your valuable data!
+Another very important part of the automatic output generation is that all
+the directory information of the input file name is stripped off of
+it. This feature can be disabled with the @option{--keepinputdir} option,
+see @ref{Input output options}. It is the default because astronomical data
+are usually very large and organized specially with special file names. In
+some cases, the user might not have write permissions in those
address@hidden fact, even if the data is stored on your own
+computer, it is advised to only grant write permissions to the super user
+or root. This way, you won't accidentally delete or modify your valuable
+data!}.
Let's assume that we are working on a report and want to process the
FITS images from two projects (ABC and DEF), which are stored in the
@@ -10156,24 +10293,23 @@ continue with the rest of actions.
@cindex Image format conversion
@cindex Converting image formats
@pindex @r{ConvertType (address@hidden)}
-The formats of astronomical data were defined mainly for archiving and
-processing. In other situations, the data might be useful in other
-formats. For example, when you are writing a paper or report or if you
-are making slides for a talk, you can't use a FITS image. Other image
-formats should be used. In other cases you might want your pixel
-values in a table format as plain text for input to other programs
-that don't recognize FITS, or to include as a table in your
-report. ConvertType is created for such situations. The various types
-will increase with future updates and based on need.
-
-The conversion is not only one way (from FITS to other formats), but
-two ways (except the EPS and PDF formats). So you can convert a JPEG
-image or text file into a FITS image. Basically, other than EPS, you
-can use any of the recognized formats as different color channel
-inputs to get any of the recognized outputs. So before explaining the
-options and arguments, first a short description of the recognized
-files types will be given followed a short introduction to digital
-color.
+The FITS format used in astronomy was defined mainly for archiving,
+transmission, and processing. In other situations, the data might be useful
+in other formats. For example, when you are writing a paper or report or if
+you are making slides for a talk, you can't use a FITS image. Other image
+formats should be used. In other cases you might want your pixel values in
+a table format as plain text for input to other programs that don't
+recognize FITS, or to include as a table in your report. ConvertType is
+created for such situations. The various types will increase with future
+updates and based on need.
+
+The conversion is not only one way (from FITS to other formats), but two
+ways (except the EPS and PDF formats). So you can convert a JPEG image or
+text file into a FITS image. Basically, other than EPS, you can use any of
+the recognized formats as different color channel inputs to get any of the
+recognized outputs. So before explaining the options and arguments, first a
+short description of the recognized files types will be given followed a
+short introduction to digital color.
@menu
* Recognized file formats:: Recognized file formats
@@ -10442,9 +10578,9 @@ are approximately the same file size.
@node Invoking astconvertt, , Color, ConvertType
@subsection Invoking ConvertType
-ConvertType will convert any recognized input file type to any
-specified output type. The executable name is @file{astconvertt} with
-the following general template
+ConvertType will convert any recognized input file type to any specified
+output type. The executable name is @file{astconvertt} with the following
+general template
@example
$ astconvertt [OPTION...] InputFile [InputFile2] ... [InputFile4]
@@ -10465,31 +10601,39 @@ $ astconvertt f1.txt f2.txt f3.fits -o.jpg
## Use two images and one blank for an RGB EPS output:
$ astconvertt M31_r.fits M31_g.fits blank -oeps
+
+## Directly pass input from output of another program through Standard
+## input (not a file).
+$ cat 2darray.txt | astconvertt -oimg.fits
@end example
@noindent
-The file type of the output will be specified with the (possibly
-complete) file name given to the @option{--output} option, which can
-either be given on the command-line or in any of the configuration
-files (see @ref{Configuration files}). Note that if the output suffix
-is not recognized, it will default to plain text format, see
address@hidden file formats}.
-
-The order of multiple input files is important. After reading the
-input file(s) the number of color channels in all the inputs will be
-used to define which color space is being used for the outputs and how
-each color channel is interpreted. Note that one file might have more
-than one color channel (for example in the JPEG format). If there is
-one color channel the output is gray-scale, if three input color
-channels are given they are respectively considered to be the red,
-green and blue color channels and if there are four color channels
-they are respectively considered to be cyan, magenta, yellow and
-black.
+The output's file format will be interpretted from the value given to the
address@hidden option. It can either be given on the command-line or in
+any of the configuration files (see @ref{Configuration files}). Note that
+if the output suffix is not recognized, it will default to plain text
+format, see @ref{Recognized file formats}.
+
address@hidden Standard input
+At most four input files (one for each color channel for formats that allow
+it) are allowed in ConvertType. The first input dataset can either be a
+file or come from Standard input (see @ref{Standard input}). The order of
+multiple input files is important. After reading the input file(s) the
+number of color channels in all the inputs will be used to define which
+color space to use for the outputs and how each color channel is
+interpreted.
+
+Some formats can allow more than one color channel (for example in the JPEG
+format, see @ref{Recognized file formats}). If there is one input dataset
+(color channel) the output will be gray-scale, if three input datasets
+(color channels) are given, they are respectively considered to be the red,
+green and blue color channels. Finally, if there are four color channels
+they will be be cyan, magenta, yellow and black (CMYK colors).
The value to @option{--output} (or @option{-o}) can be either a full file
name or just the suffix of the desired output format. In the former case,
-that same name will be used for the output. In the latter case, the name of
-the output file will be set based on the automatic output guidelines, see
+it will used for the output. In the latter case, the name of the output
+file will be set based on the automatic output guidelines, see
@ref{Automatic output}. Note that the suffix name can optionally start a
@file{.} (dot), so for example @option{--output=.jpg} and
@option{--output=jpg} are equivalent. See @ref{Recognized file formats}
@@ -10771,19 +10915,24 @@ $ asttable bintab.fits | awk '$10>10e5
@address@hidden'
## Sort the output columns by the third column, save output:
$ asttable bintab.fits | 'sort -nk3 > output.txt
+## Subtract the first column from the second in `cat.txt' and keep the
+## third and fourth columns. Feed the columns to Table to write as a
+## FITS table.
+$ awk '!/^#/@{print $2-$1, $3, address@hidden' cat.txt | asttable -ocat.fits
+
## Convert a plain text table to a binary FITS table:
$ asttable plaintext.txt --output=table.fits --tabletype=fits-binary
@end example
address@hidden GNU AWK
-In the absence of selected columns, all the input file's columns will be
-output. If the specified output is a FITS file, the type of FITS table
-(binary or ASCII) will be determined from the @option{--tabletype}
-option. If the output is not a FITS file, it will be printed as a plain
-text table (with space characters between the columns). When the columns
-are accompanied by meta-data (like column name, units, or comments), this
-information will also printed in the plain text file before the table, as
-described in @ref{Gnuastro text table format}.
+Table's input dataset can be given either as a file or from Standard input
+(see @ref{Standard input}). In the absence of selected columns, all the
+input's columns will be written to the output. If the specified output is a
+FITS file, the type of FITS table (binary or ASCII) will be determined from
+the @option{--tabletype} option. If the output is not a FITS file, it will
+be printed as a plain text table (with space characters between the
+columns). When the columns are accompanied by meta-data (like column name,
+units, or comments), this information will also printed in the plain text
+file before the table, as described in @ref{Gnuastro text table format}.
For the full list of options common to all Gnuastro programs please see
@ref{Common options}. Options can also be stored in directory, user or
@@ -15125,16 +15274,16 @@ $ awk '($1+$2)/2 > 5 @{print address@hidden'
table.txt | aststatistics --median
@noindent
@cindex Standard input
Statistics can take its input dataset either from a file (image or table)
-or the Standard input. If any output file is to be created, the value to
-the @option{--output} option, is used as the base name for the generated
-files. Without @option{--output}, the input name will be used to generate
-an output name, see @ref{Automatic output}. The options described below are
-particular to Statistics, but for general operations, it shares a large
-collection of options with the other Gnuastro programs, see @ref{Common
-options} for the full list. For more on reading from standard input, please
-see the description of @code{--stdintimeout} option in @ref{Input output
-options}. Options can also be given in configuration files, for more,
-please see @ref{Configuration files}.
+or the Standard input (see @ref{Standard input}). If any output file is to
+be created, the value to the @option{--output} option, is used as the base
+name for the generated files. Without @option{--output}, the input name
+will be used to generate an output name, see @ref{Automatic output}. The
+options described below are particular to Statistics, but for general
+operations, it shares a large collection of options with the other Gnuastro
+programs, see @ref{Common options} for the full list. For more on reading
+from standard input, please see the description of @code{--stdintimeout}
+option in @ref{Input output options}. Options can also be given in
+configuration files, for more, please see @ref{Configuration files}.
The input dataset may have blank values (see @ref{Blank pixels}), in this
case, all blank pixels are ignored during the calculation. Initially, the
@@ -19108,12 +19257,15 @@ $ astmatch --ccol1=RA,DEC --ccol2=RA_D,DEC_D
--aperture=0.5/3600 \
@end example
Two inputs are necessary for Match to start processing. The inputs can be
-plain text tables or FITS tables, see @ref{Tables}. Match follows the same
-basic behavior of all Gnuastro programs as fully described in @ref{Common
-program behavior}. If the first input is a FITS file, the common
address@hidden option (see @ref{Input output options}) should be used to
-identify the extension. When the second input is FITS, the extension must
-be specified with @option{--hdu2}.
+plain text tables or FITS tables, see @ref{Tables}. If only one argument is
+provided, Match will assume look for the first input in Stanard input (see
address@hidden input}).
+
+Match follows the same basic behavior of all Gnuastro programs as fully
+described in @ref{Common program behavior}. If the first input is a FITS
+file, the common @option{--hdu} option (see @ref{Input output options})
+should be used to identify the extension. When the second input is FITS,
+the extension must be specified with @option{--hdu2}.
When @option{--quiet} is not called, Match will print the number of matches
found in standard output (on the command-line). When matches are found, by
@@ -19927,10 +20079,11 @@ The parameters of the mock profiles can either be
given through a catalog
(which stores the parameters of many mock profiles, see @ref{MakeProfiles
catalog}), or the @option{--kernel} option (see @ref{MakeProfiles output
dataset}). The catalog can be in the FITS ASCII, FITS binary format, or
-plain text formats (see @ref{Tables}). The columns related to each
-parameter can be determined both by number, or by match/search criteria
-using the column names, units, or comments. with the options ending in
address@hidden, see below.
+plain text formats (see @ref{Tables}). A plain text catalog can also be
+provided using the Standard input (see @ref{Standard input}). The columns
+related to each parameter can be determined both by number, or by
+match/search criteria using the column names, units, or comments. with the
+options ending in @option{col}, see below.
Without any file given to the @option{--background} option, MakeProfiles
will make a zero-valued image and build the profiles on that (its size and
@@ -19973,13 +20126,14 @@ Gnuastro's programs to make a complete simulated
image of a mock galaxy.
@node MakeProfiles catalog, MakeProfiles profile settings, Invoking astmkprof,
Invoking astmkprof
@subsubsection MakeProfiles catalog
The catalog containing information about each profile can be in the FITS
-ASCII, FITS binary, or plain text formats (see @ref{Tables}). Its columns
-can be ordered in any desired manner. You can specify which columns belong
-to which parameters using the set of options discussed below. For example
-through the @option{--rcol} and @option{--tcol} options, you can specify
-the column that contains the radial parameter for each profile and its
-truncation respectively. See @ref{Selecting table columns} for a thorough
-discussion on the values to these options.
+ASCII, FITS binary, or plain text formats (see @ref{Tables}). The latter
+can also be provided using standard input (see @ref{Standard input}). Its
+columns can be ordered in any desired manner. You can specify which columns
+belong to which parameters using the set of options discussed below. For
+example through the @option{--rcol} and @option{--tcol} options, you can
+specify the column that contains the radial parameter for each profile and
+its truncation respectively. See @ref{Selecting table columns} for a
+thorough discussion on the values to these options.
The value for the profile center in the catalog (the @option{--ccol}
option) can be a floating point number so the profile center can be on any
diff --git a/lib/gnuastro-internal/options.h b/lib/gnuastro-internal/options.h
index bc71b5c..d00f8ed 100644
--- a/lib/gnuastro-internal/options.h
+++ b/lib/gnuastro-internal/options.h
@@ -303,6 +303,9 @@ gal_options_set_from_key(int key, char *arg, struct
argp_option *options,
error_t
gal_options_common_argp_parse(int key, char *arg, struct argp_state *state);
+char *
+gal_options_stdin_error(long stdintimeout, int precedence);
+
gal_list_str_t *
gal_options_check_stdin(char *inputname, long stdintimeout);
diff --git a/lib/options.c b/lib/options.c
index 8facca8..9646062 100644
--- a/lib/options.c
+++ b/lib/options.c
@@ -1358,6 +1358,31 @@ gal_options_common_argp_parse(int key, char *arg, struct
argp_state *state)
+
+char *
+gal_options_stdin_error(long stdintimeout, int precedence)
+{
+ char *out;
+
+ if( asprintf(&out, "no input!\n\n"
+ "The (first) input dataset can be read from a file "
+ "(specified as an argument), or the standard input.%s "
+ "Standard input can come from a pipe (output of another "
+ "program) or typed on the command-line before %ld "
+ "micro-seconds (configurable with the `--stdintimeout' "
+ "option).", ( precedence
+ ? "If both are provided, a file takes precedence"
+ : " " ),
+ stdintimeout )<0 )
+ error(EXIT_FAILURE, 0, "%s: `asprintf' allocation error", __func__);
+
+ return out;
+}
+
+
+
+
+
/* Make the notice that is printed before program terminates, when no input
is given and Standard input is also available. */
gal_list_str_t *
@@ -1367,14 +1392,7 @@ gal_options_check_stdin(char *inputname, long
stdintimeout)
/* See if atleast one of the two inputs is given. */
if(inputname==NULL && lines==NULL)
- error( EXIT_FAILURE, 0, "no input!\n\n"
- "The (first) input dataset can be read from a file "
- "(specified as an argument) or the standard input. If "
- "both are provided, a file takes precedence. Standard "
- "input can come from a pipe (output of another program) "
- "or typed on the command-line before %ld micro-seconds "
- "(configurable with the `--stdintimeout' option).",
- stdintimeout );
+ error( EXIT_FAILURE, 0, gal_options_stdin_error(stdintimeout,1));
/* Return the output. */
return lines;
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [gnuastro-commits] master 596b88f: Stdin in ConvertType, Match, MakeProfiles, Table, and book,
Mohammad Akhlaghi <=