[Bug-wget] [PATCH 2/3] Add --compression argument

[Tim Schlueter]

Adds a --compression argument for later use. It currently has 3 possible
values: auto, gzip, or none.

Since for the time being, I'm only planning on adding gzip support via
zlib, the compression option is disabled when wget is built without zlib.

It is worth mentioning that --continue and --start-pos will not work
with on the fly decompression of stream compression algorithms like gzip
that do not have a way to translate a compressed size to an uncompressed
size and vice versa.

If a web server only serves gzipped content, on the fly gzip
decompression is enabled, and the download is interrupted, there is no
way to continue the download from where it left off.

Conversely, in such a scenario if gzip decompression is disabled so that
the compressed data is being stored locally, the gzipped download can be
continued in some cases (depends on the server).

---
 src/init.c    | 29 +++++++++++++++++++++++++++++
 src/main.c    | 27 +++++++++++++++++++++++++++
 src/options.h |  8 ++++++++
 3 files changed, 64 insertions(+)

diff --git a/src/init.c b/src/init.c
index 5f4eefa..1064883 100644
--- a/src/init.c
+++ b/src/init.c
@@ -99,6 +99,9 @@ CMD_DECLARE (cmd_vector);

 CMD_DECLARE (cmd_use_askpass);

+#ifdef HAVE_LIBZ
+CMD_DECLARE (cmd_spec_compression);
+#endif
 CMD_DECLARE (cmd_spec_dirstruct);
 CMD_DECLARE (cmd_spec_header);
 CMD_DECLARE (cmd_spec_warc_header);
@@ -161,6 +164,9 @@ static const struct {
   { "checkcertificate", &opt.check_cert,        cmd_check_cert },
 #endif
   { "chooseconfig",     &opt.choose_config,     cmd_file },
+#ifdef HAVE_LIBZ
+  { "compression",      &opt.compression,       cmd_spec_compression },
+#endif
   { "connecttimeout",   &opt.connect_timeout,   cmd_time },
   { "contentdisposition", &opt.content_disposition, cmd_boolean },
   { "contentonerror",   &opt.content_on_error,  cmd_boolean },
@@ -445,6 +451,10 @@ defaults (void)
   opt.ftps_clear_data_connection = false;
 #endif

+#ifdef HAVE_LIBZ
+  opt.compression = compression_auto;
+#endif
+
   /* The default for file name restriction defaults to the OS type. */
 #if defined(WINDOWS) || defined(MSDOS) || defined(__CYGWIN__)
   opt.restrict_files_os = restrict_windows;
@@ -1445,6 +1455,25 @@ cmd_cert_type (const char *com, const char *val,
void *place)

 static bool check_user_specified_header (const char *);

+#ifdef HAVE_LIBZ
+static bool
+cmd_spec_compression (const char *com, const char *val, void *place)
+{
+  static const struct decode_item choices[] = {
+    { "auto", compression_auto },
+    { "gzip", compression_gzip },
+    { "none", compression_none },
+  };
+  int ok = decode_string (val, choices, countof (choices), place);
+  if (!ok)
+    {
+      fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com,
+               quote (val));
+    }
+  return ok;
+}
+#endif
+
 static bool
 cmd_spec_dirstruct (const char *com, const char *val, void
*place_ignored _GL_UNUSED)
 {
diff --git a/src/main.c b/src/main.c
index 297499e..f9759c3 100644
--- a/src/main.c
+++ b/src/main.c
@@ -275,6 +275,9 @@ static struct cmdline_option option_data[] =
     { IF_SSL ("certificate-type"), 0, OPT_VALUE, "certificatetype", -1 },
     { IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate",
-1 },
     { "clobber", 0, OPT__CLOBBER, NULL, optional_argument },
+#ifdef HAVE_LIBZ
+    { "compression", 0, OPT_VALUE, "compression", -1 },
+#endif
     { "config", 0, OPT_VALUE, "chooseconfig", -1 },
     { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 },
     { "continue", 'c', OPT_BOOLEAN, "continue", -1 },
@@ -763,6 +766,10 @@ HTTP options:\n"),
        --ignore-length             ignore 'Content-Length' header
field\n"),
     N_("\
        --header=STRING             insert STRING among the headers\n"),
+#ifdef HAVE_LIBZ
+    N_("\
+       --compression=TYPE          choose compression, one of auto,
gzip and none\n"),
+#endif
     N_("\
        --max-redirect              maximum redirections allowed per
page\n"),
     N_("\
@@ -1675,6 +1682,26 @@ for details.\n\n"));
         }
     }

+#ifdef HAVE_LIBZ
+  if (opt.always_rest || opt.start_pos >= 0)
+    {
+      if (opt.compression == compression_auto)
+        {
+          /* Compression does not work with --continue or --start-pos.
+             Since compression was not explicitly set, it will be
disabled. */
+          opt.compression = compression_none;
+        }
+      else if (opt.compression != compression_none)
+        {
+          fprintf (stderr,
+                   _("Compression does not work with --continue or"
+                     " --start-pos, they will be disabled.\n"));
+          opt.always_rest = false;
+          opt.start_pos = -1;
+        }
+    }
+#endif
+
   if (opt.ask_passwd && opt.passwd)
     {
       fprintf (stderr,
diff --git a/src/options.h b/src/options.h
index 3972945..cf945c1 100644
--- a/src/options.h
+++ b/src/options.h
@@ -326,6 +326,14 @@ struct options
                                    name. */
   bool report_bps;              /*Output bandwidth in bits format*/

+#ifdef HAVE_LIBZ
+  enum compression_options {
+    compression_auto,
+    compression_gzip,
+    compression_none
+  } compression;                /* type of HTTP compression to use */
+#endif
+
   char *rejected_log;           /* The file to log rejected URLS to. */

 #ifdef HAVE_HSTS
--

signature.asc
Description: OpenPGP digital signature