[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Bug-wget] [PATCH] Disable automatic wget headers.
|
From: |
adham elkarn |
|
Subject: |
[Bug-wget] [PATCH] Disable automatic wget headers. |
|
Date: |
Fri, 3 May 2019 23:44:26 +0000 |
From: sulfastor <address@hidden>
Hi,
This our update for the disable-header feature [bug #54769
(https://savannah.gnu.org/bugs/?54769)].
We took in account Dale Worley's comment to avoid regression since a header can
be sent empty, although is rare.
* doc/wget.texi: Added --disable-header documentation.
* fuzz/wget_options_fuzzer.dict: Update with --disable-header inputs.
* src/http.c: removed disabled headers before its creation
* src/init.c: added new functions to check user disabled headers,
disable headers, remove user headers
* src/main.c: added new option disable-header, added help description
* src/options.h: added new option disable-header
* testenv/Makefile.am: Added new test files
* testenv/server/http/http_server.py: Added new rule RejectHeaderField
* testenv/conf/reject_header_field.py: Added new rule RejectHeaderField
* testenv/README: Added help description for new rule
* testenv/Test-disable-default-headers.py: Test without using --header
* testenv/Test-disable-headers-after.py: Test using --header before
--disable-header
* testenv/Test-disable-headers-before.py: Test using --header after
--disable-header
Signed-off-by: sulfastor <address@hidden>, adham elkarn <address@hidden>
---
doc/wget.texi | 19 ++++
fuzz/wget_options_fuzzer.dict | 17 +++
src/http.c | 140 ++++++++++++++++--------
src/init.c | 88 +++++++++++++++
src/main.c | 4 +
src/options.h | 1 +
testenv/Makefile.am | 3 +
testenv/README | 4 +
testenv/Test-disable-default-headers.py | 73 ++++++++++++
testenv/Test-disable-headers-after.py | 77 +++++++++++++
testenv/Test-disable-headers-before.py | 77 +++++++++++++
testenv/conf/reject_header_field.py | 12 ++
testenv/server/http/http_server.py | 8 ++
13 files changed, 476 insertions(+), 47 deletions(-)
create mode 100644 testenv/Test-disable-default-headers.py
create mode 100644 testenv/Test-disable-headers-after.py
create mode 100644 testenv/Test-disable-headers-before.py
create mode 100644 testenv/conf/reject_header_field.py
diff --git a/doc/wget.texi b/doc/wget.texi
index 7eada2dd..7a194bb2 100644
--- a/doc/wget.texi
+++ b/doc/wget.texi
@@ -1542,6 +1542,25 @@ wget --header="Host: foo.bar" http://localhost/
In versions of Wget prior to 1.10 such use of @samp{--header} caused
sending of duplicate headers.
address@hidden disable header, choose
address@hidden address@hidden
+Remove @var{header-field} among the headers in each @sc{http} request.
+
+You may define more than one additional header field by specifying
address@hidden more than once as in @samp{--header}.
+
address@hidden
address@hidden
+wget --disable-header='Accept' \
+ --disable-header='User-Agent' \
+ --disable-header='Accept-Encoding' \
+ http://fly.srk.fer.hr/
address@hidden group
address@hidden example
+
+Specificating a header field with @samp{--header} after disabling it
+will override it and include it in the @sc{http} request headers.
+
@cindex Content-Encoding, choose
@item address@hidden
Choose the type of compression to be used. Legal values are
diff --git a/fuzz/wget_options_fuzzer.dict b/fuzz/wget_options_fuzzer.dict
index 9a2dbd8e..12d54d60 100644
--- a/fuzz/wget_options_fuzzer.dict
+++ b/fuzz/wget_options_fuzzer.dict
@@ -30,6 +30,22 @@
"human"
"csv"
"json"
+"Authorization"
+"User-Agent"
+"Referer"
+"Cache-Control"
+"Pragma"
+"If-Modified-Since"
+"Range"
+"Accept"
+"Accept-Encoding"
+"Host"
+"Connection"
+"Proxy-Connection"
+"Content-Type"
+"Content-Length"
+"Proxy-Authorization"
+"Cookie"
"accept="
"accept-regex="
"adjust-extension="
@@ -66,6 +82,7 @@
"delete-after="
"directories="
"directory-prefix="
+"disable-header="
"dns-caching="
"dns-timeout="
"domains="
diff --git a/src/http.c b/src/http.c
index 289d1101..0ccba4bf 100644
--- a/src/http.c
+++ b/src/http.c
@@ -88,6 +88,7 @@ static char *basic_authentication_encode (const char *, const
char *);
static bool known_authentication_scheme_p (const char *, const char *);
static void ensure_extension (struct http_stat *, const char *, int *);
static void load_cookies (void);
+static bool disabled_header (char*);
static bool cookies_loaded_p;
static struct cookie_jar *wget_cookie_jar;
@@ -152,6 +153,8 @@ struct request {
int hcount, hcapacity;
};
+/* Forward decls. */
+static bool request_remove_header (struct request*, const char*);
/* Create a new, empty request. Set the request's method and its
arguments. METHOD should be a literal string (or it should outlive
@@ -436,7 +439,7 @@ maybe_send_basic_creds (const char *hostname, const char
*user,
DEBUGP (("Host %s has not issued a general basic challenge.\n",
quote (hostname)));
}
- if (do_challenge)
+ if (!disabled_header ("Authorization") && do_challenge)
{
request_set_header (req, "Authorization",
basic_authentication_encode (user, passwd),
@@ -1770,23 +1773,29 @@ read_response_body (struct http_stat *hs, int sock,
FILE *fp, wgint contlen,
#ifdef __VMS
#define SET_USER_AGENT(req) do { \
- if (!opt.useragent) \
- request_set_header (req, "User-Agent", \
- aprintf ("Wget/%s (VMS %s %s)", \
- version_string, vms_arch(), vms_vers()), \
- rel_value); \
- else if (*opt.useragent) \
- request_set_header (req, "User-Agent", opt.useragent, rel_none); \
+ if (!disabled_header ("User-Agent")) \
+ { \
+ if (!opt.useragent) \
+ request_set_header (req, "User-Agent", \
+ aprintf ("Wget/%s (VMS %s %s)", \
+ version_string, vms_arch(), vms_vers()), \
+ rel_value); \
+ else if (*opt.useragent) \
+ request_set_header (req, "User-Agent", opt.useragent, rel_none); \
+ } \
} while (0)
#else /* def __VMS */
#define SET_USER_AGENT(req) do { \
- if (!opt.useragent) \
- request_set_header (req, "User-Agent", \
- aprintf ("Wget/%s (%s)", \
- version_string, OS_TYPE), \
- rel_value); \
- else if (*opt.useragent) \
- request_set_header (req, "User-Agent", opt.useragent, rel_none); \
+ if (!disabled_header ("User-Agent")) \
+ { \
+ if (!opt.useragent) \
+ request_set_header (req, "User-Agent", \
+ aprintf ("Wget/%s (%s)", \
+ version_string, OS_TYPE), \
+ rel_value); \
+ else if (*opt.useragent) \
+ request_set_header (req, "User-Agent", opt.useragent, rel_none); \
+ } \
} while (0)
#endif /* def __VMS [else] */
@@ -1842,6 +1851,26 @@ time_to_rfc1123 (time_t time, char *buf, size_t bufsize)
return RETROK;
}
+static bool
+disabled_header (char* header_name)
+{
+ char** p = opt.disabled_headers;
+ char *s;
+ size_t n;
+
+ if (!p)
+ return 0;
+
+ for (; *p; ++p) {
+ s = strchrnul (header_name, ':');
+ n = (size_t) (s - header_name);
+ if (n == strlen (*p) && 0 == strncmp (header_name, *p, n))
+ return 1;
+ }
+
+ return 0;
+}
+
static struct request *
initialize_request (const struct url *u, struct http_stat *hs, int *dt, struct
url *proxy,
bool inhibit_keep_alive, bool *basic_auth_finished,
@@ -1874,15 +1903,17 @@ initialize_request (const struct url *u, struct
http_stat *hs, int *dt, struct u
meth_arg = url_full_path (u);
req = request_new (meth, meth_arg);
}
-
- request_set_header (req, "Referer", (char *) hs->referer, rel_none);
+ if (!disabled_header ("Referer"))
+ request_set_header (req, "Referer", (char *) hs->referer, rel_none);
if (*dt & SEND_NOCACHE)
{
/* Cache-Control MUST be obeyed by all HTTP/1.1 caching mechanisms... */
- request_set_header (req, "Cache-Control", "no-cache", rel_none);
+ if (!disabled_header ("Cache-Control"))
+ request_set_header (req, "Cache-Control", "no-cache", rel_none);
/* ... but some HTTP/1.0 caches doesn't implement Cache-Control. */
- request_set_header (req, "Pragma", "no-cache", rel_none);
+ if (!disabled_header ("Pragma"))
+ request_set_header (req, "Pragma", "no-cache", rel_none);
}
if (*dt & IF_MODIFIED_SINCE)
{
@@ -1896,21 +1927,27 @@ initialize_request (const struct url *u, struct
http_stat *hs, int *dt, struct u
"time.\n"));
strcpy (strtime, "Thu, 01 Jan 1970 00:00:00 GMT");
}
- request_set_header (req, "If-Modified-Since", xstrdup (strtime),
rel_value);
+ if (!disabled_header ("If-Modified-Since"))
+ request_set_header (req, "If-Modified-Since", xstrdup (strtime),
rel_value);
}
- if (hs->restval)
+ if (!disabled_header ("Range") && hs->restval)
request_set_header (req, "Range",
aprintf ("bytes=%s-",
number_to_static_string (hs->restval)),
rel_value);
+
SET_USER_AGENT (req);
- request_set_header (req, "Accept", "*/*", rel_none);
+
+ if (!disabled_header ("Accept"))
+ request_set_header (req, "Accept", "*/*", rel_none);
+
#ifdef HAVE_LIBZ
- if (opt.compression != compression_none)
+ if (!disabled_header ("Accept-Encoding") && opt.compression !=
compression_none)
request_set_header (req, "Accept-Encoding", "gzip", rel_none);
else
#endif
- request_set_header (req, "Accept-Encoding", "identity", rel_none);
+ if (!disabled_header ("Accept-Encoding"))
+ request_set_header (req, "Accept-Encoding", "identity", rel_none);
/* Find the username with priority */
if (u->user)
@@ -1966,17 +2003,19 @@ initialize_request (const struct url *u, struct
http_stat *hs, int *dt, struct u
};
int add_port = u->port != scheme_default_port (u->scheme);
int add_squares = strchr (u->host, ':') != NULL;
- request_set_header (req, "Host",
- aprintf (hfmt[add_port][add_squares], u->host,
u->port),
- rel_value);
+ if (!disabled_header ("Host"))
+ request_set_header (req, "Host",
+ aprintf (hfmt[add_port][add_squares], u->host,
u->port),
+ rel_value);
}
- if (inhibit_keep_alive)
+ if (!disabled_header ("Connection") && inhibit_keep_alive)
request_set_header (req, "Connection", "Close", rel_none);
else
{
- request_set_header (req, "Connection", "Keep-Alive", rel_none);
- if (proxy)
+ if (!disabled_header ("Connection"))
+ request_set_header (req, "Connection", "Keep-Alive", rel_none);
+ if (!disabled_header ("Proxy-Connection") && proxy)
request_set_header (req, "Proxy-Connection", "Keep-Alive", rel_none);
}
@@ -1985,8 +2024,9 @@ initialize_request (const struct url *u, struct http_stat
*hs, int *dt, struct u
if (opt.body_data || opt.body_file)
{
- request_set_header (req, "Content-Type",
- "application/x-www-form-urlencoded", rel_none);
+ if (!disabled_header ("Content-Type"))
+ request_set_header (req, "Content-Type",
+ "application/x-www-form-urlencoded", rel_none);
if (opt.body_data)
*body_data_size = strlen (opt.body_data);
@@ -2002,13 +2042,15 @@ initialize_request (const struct url *u, struct
http_stat *hs, int *dt, struct u
return NULL;
}
}
- request_set_header (req, "Content-Length",
- xstrdup (number_to_static_string
(*body_data_size)),
- rel_value);
+ if (!disabled_header ("Content-Length"))
+ request_set_header (req, "Content-Length",
+ xstrdup (number_to_static_string
(*body_data_size)),
+ rel_value);
}
- else if (c_strcasecmp (opt.method, "post") == 0
- || c_strcasecmp (opt.method, "put") == 0
- || c_strcasecmp (opt.method, "patch") == 0)
+ else if (!disabled_header ("Content-Length")
+ && (c_strcasecmp (opt.method, "post") == 0
+ || c_strcasecmp (opt.method, "put") == 0
+ || c_strcasecmp (opt.method, "patch") == 0))
request_set_header (req, "Content-Length", "0", rel_none);
}
return req;
@@ -2043,7 +2085,8 @@ initialize_proxy_configuration (const struct url *u,
struct request *req,
#ifdef HAVE_SSL
if (u->scheme != SCHEME_HTTPS)
#endif
- request_set_header (req, "Proxy-Authorization", *proxyauth, rel_value);
+ if (!disabled_header ("Proxy-Authorization"))
+ request_set_header (req, "Proxy-Authorization", *proxyauth, rel_value);
}
static uerr_t
@@ -2133,8 +2176,9 @@ establish_connection (const struct url *u, const struct
url **conn_ref,
CONNECT method to request passthrough. */
struct request *connreq = request_new ("CONNECT",
aprintf ("%s:%d", u->host, u->port));
+
SET_USER_AGENT (connreq);
- if (proxyauth)
+ if (!disabled_header ("Proxy-Authorization") && proxyauth)
{
request_set_header (connreq, "Proxy-Authorization",
*proxyauth, rel_value);
@@ -2143,9 +2187,10 @@ establish_connection (const struct url *u, const struct
url **conn_ref,
the regular request below. */
*proxyauth = NULL;
}
- request_set_header (connreq, "Host",
- aprintf ("%s:%d", u->host, u->port),
- rel_value);
+ if (!disabled_header ("Host"))
+ request_set_header (connreq, "Host",
+ aprintf ("%s:%d", u->host, u->port),
+ rel_value);
write_error = request_send (connreq, sock, 0);
request_free (&connreq);
@@ -2456,7 +2501,7 @@ check_auth (const struct url *u, char *user, char
*passwd, struct response *resp
auth_err = *auth_stat;
xfree (auth_stat);
xfree (pth);
- if (auth_err == RETROK)
+ if (!disabled_header ("Authorization") && auth_err == RETROK)
{
request_set_header (req, "Authorization", value, rel_value);
@@ -3248,9 +3293,10 @@ gethttp (const struct url *u, struct url *original_url,
struct http_stat *hs,
for the Digest authorization scheme.) */
if (opt.cookies)
- request_set_header (req, "Cookie",
- cookie_header (wget_cookie_jar,
- u->host, u->port, u->path,
+ if (!disabled_header ("Cookie"))
+ request_set_header (req, "Cookie",
+ cookie_header (wget_cookie_jar,
+ u->host, u->port, u->path,
#ifdef HAVE_SSL
u->scheme == SCHEME_HTTPS
#else
diff --git a/src/init.c b/src/init.c
index 9b6665a6..aaf99ba6 100644
--- a/src/init.c
+++ b/src/init.c
@@ -101,6 +101,7 @@ CMD_DECLARE (cmd_spec_compression);
#endif
CMD_DECLARE (cmd_spec_dirstruct);
CMD_DECLARE (cmd_spec_header);
+CMD_DECLARE (cmd_dis_header);
CMD_DECLARE (cmd_spec_warc_header);
CMD_DECLARE (cmd_spec_htmlify);
CMD_DECLARE (cmd_spec_mirror);
@@ -183,6 +184,7 @@ static const struct {
{ "deleteafter", &opt.delete_after, cmd_boolean },
{ "dirprefix", &opt.dir_prefix, cmd_directory },
{ "dirstruct", NULL, cmd_spec_dirstruct },
+ { "disableheader", NULL, cmd_dis_header},
{ "dnscache", &opt.dns_cache, cmd_boolean },
#ifdef HAVE_LIBCARES
{ "dnsservers", &opt.dns_servers, cmd_string },
@@ -398,6 +400,7 @@ defaults (void)
opt.metalink_index = -1;
#endif
+ opt.disabled_headers = NULL;
opt.cookies = true;
opt.verbose = -1;
opt.ntry = 20;
@@ -990,6 +993,7 @@ struct decode_item {
static bool decode_string (const char *, const struct decode_item *, int, int
*);
static bool simple_atof (const char *, const char *, double *);
+
#define CMP1(p, c0) (c_tolower((p)[0]) == (c0) && (p)[1] == '\0')
#define CMP2(p, c0, c1) (c_tolower((p)[0]) == (c0) \
@@ -1459,6 +1463,8 @@ cmd_cert_type (const char *com, const char *val, void
*place)
options specially. */
static bool check_user_specified_header (const char *);
+static bool check_user_disabled_header (const char *);
+static char ** vec_remove_header (char **, const char *);
#ifdef HAVE_LIBZ
static bool
@@ -1493,6 +1499,29 @@ cmd_spec_dirstruct (const char *com, const char *val,
void *place_ignored _GL_UN
return true;
}
+static bool
+cmd_dis_header (const char *com, const char *val, void *place_ignored
_GL_UNUSED)
+{
+ /* Empty value means reset the list of headers. */
+ if (*val == '\0')
+ {
+ free_vec (opt.disabled_headers);
+ opt.disabled_headers = NULL;
+ return true;
+ }
+
+ if (!check_user_disabled_header (val))
+ {
+ fprintf (stderr, _("%s: %s: Invalid header %s.\n"),
+ exec_name, com, quote (val));
+ return false;
+ }
+ /* Removes disabled headers from user defined headers */
+ opt.user_headers = vec_remove_header (opt.user_headers, val);
+ opt.disabled_headers = vec_append (opt.disabled_headers, val);
+ return true;
+}
+
static bool
cmd_spec_header (const char *com, const char *val, void *place_ignored
_GL_UNUSED)
{
@@ -1850,6 +1879,23 @@ simple_atof (const char *beg, const char *end, double
*dest)
contain a colon preceded by non-white-space characters and must not
contain newlines. */
+
+static bool
+check_user_disabled_header (const char* s)
+{
+ const char *p;
+
+ for (p = s; *p && !c_isspace (*p); p++)
+ ;
+
+ if (p == s)
+ return false;
+ /* The header MUST NOT contain newlines. */
+ if (strchr (s, '\n'))
+ return false;
+ return true;
+}
+
static bool
check_user_specified_header (const char *s)
{
@@ -1867,6 +1913,47 @@ check_user_specified_header (const char *s)
return true;
}
+/* Removes a header from a request headers vector */
+
+static char **
+vec_remove_header (char **vec, const char *str)
+{
+ char* s;
+ int i, cnt; /* count of vector elements */
+ size_t n;
+
+ if (vec != NULL)
+ {
+ for (cnt = 0; vec[cnt]; cnt++)
+ ;
+ /* remove all duplicates */
+ i = 0;
+ while (vec[i])
+ {
+ s = strchrnul (vec[i], ':');
+ n = (size_t) (s - vec[i]);
+ if (n == strlen (str) && 0 == strncmp (vec[i], str, n))
+ {
+ if (cnt == 1)
+ {
+ vec[i] = NULL;
+ return vec;
+ }
+ else
+ {
+ vec[i] = xstrdup (vec[cnt - 1]);
+ vec = xrealloc (vec, (cnt + 1) * sizeof (char *));
+ vec[cnt] = NULL;
+ --cnt;
+ }
+ }
+ else
+ ++i;
+ }
+ }
+ return vec;
+}
+
/* Decode VAL into a number, according to ITEMS. */
static bool
@@ -1977,6 +2064,7 @@ cleanup (void)
xfree (opt.http_passwd);
xfree (opt.dot_style);
free_vec (opt.user_headers);
+ free_vec (opt.disabled_headers);
free_vec (opt.warc_user_headers);
# ifdef HAVE_SSL
xfree (opt.cert_file);
diff --git a/src/main.c b/src/main.c
index 65b7f3f3..a73b39d9 100644
--- a/src/main.c
+++ b/src/main.c
@@ -304,6 +304,7 @@ static struct cmdline_option option_data[] =
{ "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 },
{ "directories", 0, OPT_BOOLEAN, "dirstruct", -1 },
{ "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 },
+ { "disable-header", 0, OPT_VALUE, "disableheader", -1 },
{ "dns-cache", 0, OPT_BOOLEAN, "dnscache", -1 },
#ifdef HAVE_LIBCARES
{ "dns-servers", 0, OPT_VALUE, "dnsservers", -1 },
@@ -544,6 +545,7 @@ init_switches (void)
identical to "--foo", except it has opposite meaning and
it doesn't allow an argument. */
longopt = &long_options[o++];
+
longopt->name = no_prefix (cmdopt->long_name);
longopt->has_arg = no_argument;
/* Mask the value so we'll be able to recognize that we're
@@ -792,6 +794,8 @@ HTTP options:\n"),
--ignore-length ignore 'Content-Length' header field\n"),
N_("\
--header=STRING insert STRING among the headers\n"),
+ N_("\
+ --disable-header=STRING disable STRING among the headers\n"),
#ifdef HAVE_LIBZ
N_("\
--compression=TYPE choose compression, one of auto, gzip and
none. (default: none)\n"),
diff --git a/src/options.h b/src/options.h
index 881e2b2e..5559c6fa 100644
--- a/src/options.h
+++ b/src/options.h
@@ -147,6 +147,7 @@ struct options
char *http_user; /* HTTP username. */
char *http_passwd; /* HTTP password. */
char **user_headers; /* User-defined header(s). */
+ char **disabled_headers; /* User-disabled header(s) */
bool http_keep_alive; /* whether we use keep-alive */
bool use_proxy; /* Do we use proxy? */
diff --git a/testenv/Makefile.am b/testenv/Makefile.am
index b5a39ad2..4b3e2d08 100644
--- a/testenv/Makefile.am
+++ b/testenv/Makefile.am
@@ -95,6 +95,9 @@ if HAVE_PYTHON3
Test-cookie-domain-mismatch.py \
Test-cookie-expires.py \
Test-cookie.py \
+ Test-disable-default-headers.py \
+ Test-disable-headers-after.py \
+ Test-disable-headers-before.py \
Test-Head.py \
Test-hsts.py \
Test--https.py \
diff --git a/testenv/README b/testenv/README
index 6580bc99..d2f38a67 100644
--- a/testenv/README
+++ b/testenv/README
@@ -182,6 +182,10 @@ This section lists the currently supported File Rules and
their structure.
* RejectHeader : This list of Headers must NEVER occur in a request. It
uses the same value format as ExpectHeader.
+ * RejectHeaderField : This list of Headers Fields must NOT appear in a
request.
+ The value for this key is a list of strings where each header field is
represented as:
+ |-->Header Field: <Header Field Name>
+
* SendHeader : This list of Headers will be sent in EVERY response to a
request for the respective file. It follows the same value format as
ExpectHeader. Additionally you can specify a list of strings as <Header
Data>
diff --git a/testenv/Test-disable-default-headers.py
b/testenv/Test-disable-default-headers.py
new file mode 100644
index 00000000..22ea54ad
--- /dev/null
+++ b/testenv/Test-disable-default-headers.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+ This is test ensures that the --disable-header option removes default
request
+ headers. There aren't any user defined header.
+"""
+############# File Definitions ###############################################
+file_content = """Les paroles de la bouche d'un homme sont des eaux profondes;
+La source de la sagesse est un torrent qui jaillit."""
+
+Headers = {
+ 'Authorization',
+ 'User-Agent',
+ 'Referer',
+ 'Cache-Control',
+ 'Pragma',
+ 'If-Modified-Since',
+ 'Range',
+ 'Accept',
+ 'Accept-Encoding',
+ 'Host',
+ 'Connection',
+ 'Proxy-Connection',
+ 'Content-Type',
+ 'Content-Length',
+ 'Proxy-Authorization',
+ 'Cookie',
+ 'MyHeader',
+}
+
+WGET_OPTIONS = ''
+WGET_URLS = [[]]
+Files = [[]]
+
+for index, header in enumerate(Headers, start=1):
+ File_rules = {
+ "RejectHeaderField" : {
+ header
+ }
+ }
+ file_name = "File" + str(index)
+ Files[0].append (WgetFile(file_name, file_content, rules=File_rules))
+ WGET_OPTIONS += ' --disable-header="' + header + '"'
+ WGET_URLS[0].append (file_name)
+
+Servers = [HTTP]
+
+ExpectedReturnCode = 0
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+ "ServerFiles" : Files
+}
+test_options = {
+ "WgetCommands" : WGET_OPTIONS,
+ "Urls" : WGET_URLS
+}
+post_test = {
+ "ExpectedRetcode" : ExpectedReturnCode
+}
+
+err = HTTPTest (
+ pre_hook=pre_test,
+ test_params=test_options,
+ post_hook=post_test,
+ protocols=Servers
+).begin ()
+
+exit (err)
diff --git a/testenv/Test-disable-headers-after.py
b/testenv/Test-disable-headers-after.py
new file mode 100644
index 00000000..44d03cb5
--- /dev/null
+++ b/testenv/Test-disable-headers-after.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+ This is test ensures that the --disable-header option removes user headers
+ from the HTTP request when it's placed after --header="header: value".
+"""
+############# File Definitions ###############################################
+file_content = """Les paroles de la bouche d'un homme sont des eaux profondes;
+La source de la sagesse est un torrent qui jaillit."""
+
+Headers = {
+ 'Authorization',
+ 'User-Agent',
+ 'Referer',
+ 'Cache-Control',
+ 'Pragma',
+ 'If-Modified-Since',
+ 'Range',
+ 'Accept',
+ 'Accept-Encoding',
+ 'Host',
+ 'Connection',
+ 'Proxy-Connection',
+ 'Content-Type',
+ 'Content-Length',
+ 'Proxy-Authorization',
+ 'Cookie',
+ 'MyHeader',
+}
+
+WGET_OPTIONS = ''
+WGET_URLS = [[]]
+Files = [[]]
+
+# Define user defined headers
+for header in Headers:
+ WGET_OPTIONS += ' --header="' + header + ': any"'
+
+for index, header in enumerate(Headers, start=1):
+ File_rules = {
+ "RejectHeader" : {
+ header : 'any'
+ }
+ }
+ file_name = "File" + str(index)
+ Files[0].append(WgetFile(file_name, file_content, rules=File_rules))
+ WGET_OPTIONS += ' --disable-header="' + header + '"'
+ WGET_URLS[0].append(file_name)
+
+Servers = [HTTP]
+
+ExpectedReturnCode = 0
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+ "ServerFiles" : Files
+}
+test_options = {
+ "WgetCommands" : WGET_OPTIONS,
+ "Urls" : WGET_URLS
+}
+post_test = {
+ "ExpectedRetcode" : ExpectedReturnCode
+}
+
+err = HTTPTest (
+ pre_hook=pre_test,
+ test_params=test_options,
+ post_hook=post_test,
+ protocols=Servers
+).begin ()
+
+exit (err)
diff --git a/testenv/Test-disable-headers-before.py
b/testenv/Test-disable-headers-before.py
new file mode 100644
index 00000000..4356b4c1
--- /dev/null
+++ b/testenv/Test-disable-headers-before.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from test.base_test import HTTP, HTTPS
+from misc.wget_file import WgetFile
+
+"""
+ This is test ensures that the --disable-header option doesn't remove user
headers
+ from the HTTP request when it's placed before --header="header: value".
+"""
+############# File Definitions ###############################################
+file_content = """Les paroles de la bouche d'un homme sont des eaux profondes;
+La source de la sagesse est un torrent qui jaillit."""
+
+Headers = {
+ 'Authorization',
+ 'User-Agent',
+ 'Referer',
+ 'Cache-Control',
+ 'Pragma',
+ 'If-Modified-Since',
+ 'Range',
+ 'Accept',
+ 'Accept-Encoding',
+ 'Host',
+ 'Connection',
+ 'Proxy-Connection',
+ 'Content-Type',
+ 'Content-Length',
+ 'Proxy-Authorization',
+ 'Cookie',
+ 'MyHeader',
+}
+
+WGET_OPTIONS = ''
+WGET_URLS = [[]]
+Files = [[]]
+
+for index, header in enumerate(Headers, start=1):
+ File_rules = {
+ "ExpectHeader" : {
+ header : 'any'
+ }
+ }
+ file_name = "File" + str(index)
+ Files[0].append (WgetFile(file_name, file_content, rules=File_rules))
+ WGET_OPTIONS += ' --disable-header="' + header + '"'
+ WGET_URLS[0].append (file_name)
+
+# Define user defined headers
+for header in Headers:
+ WGET_OPTIONS += ' --header="' + header + ': any"'
+
+Servers = [HTTP]
+
+ExpectedReturnCode = 0
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+ "ServerFiles" : Files
+}
+test_options = {
+ "WgetCommands" : WGET_OPTIONS,
+ "Urls" : WGET_URLS
+}
+post_test = {
+ "ExpectedRetcode" : ExpectedReturnCode
+}
+
+err = HTTPTest (
+ pre_hook=pre_test,
+ test_params=test_options,
+ post_hook=post_test,
+ protocols=Servers
+).begin ()
+
+exit (err)
diff --git a/testenv/conf/reject_header_field.py
b/testenv/conf/reject_header_field.py
new file mode 100644
index 00000000..e1009cdd
--- /dev/null
+++ b/testenv/conf/reject_header_field.py
@@ -0,0 +1,12 @@
+from conf import rule
+
+""" Rule: RejectHeaderField
+This is a server side rule which expects a string list of Header Fields
+which should be blacklisted by the server for a particular file's requests.
+"""
+
+
address@hidden()
+class RejectHeaderField:
+ def __init__(self, header_fields):
+ self.header_fields = header_fields
diff --git a/testenv/server/http/http_server.py
b/testenv/server/http/http_server.py
index 2cc82fb9..6f358335 100644
--- a/testenv/server/http/http_server.py
+++ b/testenv/server/http/http_server.py
@@ -370,6 +370,14 @@ class _Handler(BaseHTTPRequestHandler):
header_line)
raise ServerError("Header " + header_line + ' received')
+ def RejectHeaderField(self, header_fields_obj):
+ rej_header_fields = header_fields_obj.header_fields
+ for field in rej_header_fields:
+ if field in self.headers:
+ self.send_error(400, 'Blacklisted Header Field %s received' %
+ field)
+ raise ServerError('Header Field %s received' % field)
+
def __log_request(self, method):
req = method + " " + self.path
self.server.request_headers.append(req)
--
2.21.0