[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [PATCH] basenc: Paddingless input/output for base64url
|
From: |
Imre Rad |
|
Subject: |
Re: [PATCH] basenc: Paddingless input/output for base64url |
|
Date: |
Tue, 23 Aug 2022 12:42:39 +0200 |
Indeed; your suggestion is now included:
* src/basenc.c (ignore-padding):
Padding is optional for base64url encoding and many web
services produce and expect payload without padding.
New command line argument (--ignore-padding) for basenc
allows generating paddingless base64url outputs and
you can also use it for decoding without scary
warnings on stderr.
* tests/local.mk (reference to new test):
Reference to the new test.
---
src/basenc.c | 48 ++++++++++++++++----
tests/local.mk | 1 +
tests/misc/basenc-padding.sh | 86 ++++++++++++++++++++++++++++++++++++
3 files changed, 127 insertions(+), 8 deletions(-)
create mode 100755 tests/misc/basenc-padding.sh
diff --git a/src/basenc.c b/src/basenc.c
index 04857d59e..f11248e74 100644
--- a/src/basenc.c
+++ b/src/basenc.c
@@ -68,7 +68,8 @@ enum
BASE16_OPTION,
BASE2MSBF_OPTION,
BASE2LSBF_OPTION,
- Z85_OPTION
+ Z85_OPTION,
+ IGNORE_PADDING_OPTION
};
#endif
@@ -78,6 +79,7 @@ static struct option const long_options[] =
{"wrap", required_argument, 0, 'w'},
{"ignore-garbage", no_argument, 0, 'i'},
#if BASE_TYPE == 42
+ {"ignore-padding", no_argument, 0, IGNORE_PADDING_OPTION},
{"base64", no_argument, 0, BASE64_OPTION},
{"base64url", no_argument, 0, BASE64URL_OPTION},
{"base32", no_argument, 0, BASE32_OPTION},
@@ -146,6 +148,8 @@ Base%d encode or decode FILE, or standard input,
to standard output.\n\
"), stdout);
#if BASE_TYPE == 42
fputs (_("\
+ --ignore-padding base64url only: ignore missing padding at decoding,\n\
+ don't pad at encoding\n\
--z85 ascii85-like encoding (ZeroMQ spec:32/Z85);\n\
when encoding, input length must be a multiple of 4;\n\
when decoding, input length must be a multiple of 5\n\
@@ -335,7 +339,6 @@ base64url_decode_ctx_init_wrapper (struct
base_decode_context *ctx)
init_inbuf (ctx);
}
-
static bool
base64url_decode_ctx_wrapper (struct base_decode_context *ctx,
char const *restrict in, idx_t inlen,
@@ -368,7 +371,16 @@ base64url_decode_ctx_wrapper (struct
base_decode_context *ctx,
return b;
}
-
+static bool
+base64url_decode_ctx_wrapper_no_padding (struct base_decode_context *ctx,
+ char const *restrict in, idx_t inlen,
+ char *restrict out, idx_t *outlen)
+{
+ bool b = base64url_decode_ctx_wrapper(ctx, in, inlen, out, outlen);
+ if (!b && inlen == 0) // inlen 0 indicates the final round, see do_decode
+ b = true;
+ return b;
+}
static int
base32_length_wrapper (int len)
@@ -964,7 +976,7 @@ finish_and_exit (FILE *in, char const *infile)
}
static _Noreturn void
-do_encode (FILE *in, char const *infile, FILE *out, idx_t wrap_column)
+do_encode (FILE *in, char const *infile, FILE *out, idx_t
wrap_column, bool without_padding)
{
idx_t current_column = 0;
char *inbuf, *outbuf;
@@ -989,9 +1001,17 @@ do_encode (FILE *in, char const *infile, FILE
*out, idx_t wrap_column)
{
/* Process input one block at a time. Note that ENC_BLOCKSIZE
is sized so that no pad chars will appear in output. */
- base_encode (inbuf, sum, outbuf, BASE_LENGTH (sum));
+ int to_write = BASE_LENGTH (sum);
+ base_encode (inbuf, sum, outbuf, to_write);
+ if (without_padding)
+ {
+ while (to_write > 0 && *(outbuf+to_write-1) == '=')
+ {
+ --to_write;
+ }
+ }
- wrap_write (outbuf, BASE_LENGTH (sum), wrap_column,
+ wrap_write (outbuf, to_write, wrap_column,
¤t_column, out);
}
}
@@ -1084,6 +1104,13 @@ main (int argc, char **argv)
bool decode = false;
/* True if we should ignore non-base-alphabetic characters. */
bool ignore_garbage = false;
+
+ /* True if we should ignore padding (base64url only). */
+#if BASE_TYPE == 42
+ bool o_ignore_padding = false;
+#endif
+ bool ignore_padding = false;
+
/* Wrap encoded data around the 76th column, by default. */
idx_t wrap_column = 76;
@@ -1122,6 +1149,10 @@ main (int argc, char **argv)
break;
#if BASE_TYPE == 42
+ case IGNORE_PADDING_OPTION:
+ o_ignore_padding = true;
+ break;
+
case BASE64_OPTION:
case BASE64URL_OPTION:
case BASE32_OPTION:
@@ -1155,11 +1186,12 @@ main (int argc, char **argv)
break;
case BASE64URL_OPTION:
+ ignore_padding = o_ignore_padding;
base_length = base64_length_wrapper;
isbase = isbase64url;
base_encode = base64url_encode;
base_decode_ctx_init = base64url_decode_ctx_init_wrapper;
- base_decode_ctx = base64url_decode_ctx_wrapper;
+ base_decode_ctx = ignore_padding ?
base64url_decode_ctx_wrapper_no_padding :
base64url_decode_ctx_wrapper;
break;
case BASE32_OPTION:
@@ -1244,5 +1276,5 @@ main (int argc, char **argv)
if (decode)
do_decode (input_fh, infile, stdout, ignore_garbage);
else
- do_encode (input_fh, infile, stdout, wrap_column);
+ do_encode (input_fh, infile, stdout, wrap_column, ignore_padding);
}
diff --git a/tests/local.mk b/tests/local.mk
index 0496c2873..e3d9d85e2 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -728,6 +728,7 @@ all_tests = \
tests/touch/read-only.sh \
tests/touch/relative.sh \
tests/touch/trailing-slash.sh \
+ tests/misc/basenc-padding.sh \
$(all_root_tests)
# See tests/factor/create-test.sh.
diff --git a/tests/misc/basenc-padding.sh b/tests/misc/basenc-padding.sh
new file mode 100755
index 000000000..c90f22fa1
--- /dev/null
+++ b/tests/misc/basenc-padding.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+# make sure base64url works fine without paddings
+
+# Copyright (C) 2000-2022 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
+
+
+
+input='xs?>>>x'
+
+# should be fine with wrapped writes
+b64url_encoded_wo_p="$(echo "$input" | basenc --base64url --ignore-padding)"
+
+if grep -q "=" <<< "$b64url_encoded_wo_p"; then
+ echo "Paddings are not supposed to be present"
+ Exit 1
+fi
+
+
+# note the 2>&1's, stderr should be clean
+output="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d
--ignore-padding 2>&1)"
+
+if [ "$input" != "$output" ]; then
+ echo "Something is wrong without paddings (wrapped writes)"
+ Exit 1
+fi
+
+# decoding without --ignore-padding
+output_err="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d 2>&1)"
+if ! grep -q "invalid input" <<< "$output_err"; then
+ echo "Decoding without --ignore-padding should still complain"
+ Exit 1
+fi
+
+# decoding errors anywhere but the last round should still complain,
even with --ignore-padding
+known_broken="eHM_Pj4*eAo"
+output="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d
--ignore-padding 2>&1)"
+if ! grep -q "invalid input" <<< "$output_err"; then
+ echo "Invalid encoding anywhere but the last round should still
be rejected"
+ Exit 1
+fi
+
+# should be fine without wrapped writes
+b64url_encoded_wo_p="$(echo "$input" | basenc --base64url
--ignore-padding -w0)"
+
+# note the 2>&1's, stderr should be clean
+output="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d
--ignore-padding 2>&1)"
+
+if [ "$input" != "$output" ]; then
+ echo "Something is wrong without paddings (non-wrapped writes)"
+ Exit 1
+fi
+
+# should be ok with padding as well
+b64url_encoded_w_p="$(echo "$input"| basenc --base64url)"
+
+# note the 2>&1's, stderr should be clean
+output="$(echo "$b64url_encoded_w_p" | basenc --base64url -d 2>&1)"
+
+if [ "$input" != "$output" ]; then
+ echo "Something is wrong with paddings"
+ Exit 1
+fi
+
+if [ "$b64url_encoded_w_p" == "b64url_encoded_wo_p" ]; then
+ echo "Encoding with and without padding should look different"
+ Exit 1
+fi
+
+#echo Padding test success
+
+Exit 0
--
2.30.2
On Tue, Aug 23, 2022 at 11:02 AM Michael Cook <michael@waxrat.com> wrote:
>
> + int to_write = BASE_LENGTH (sum);
> + base_encode (inbuf, sum, outbuf, to_write);
> + if (without_padding)
> + {
> + while (*(outbuf+to_write-1) == '=')
> + {
> + --to_write;
> + }
> + }
>
> Probably should make sure `to_write` stays positive:
>
> while (to_write > 0 && outbuf[to_write - 1] == '=')
> {
> --to_write;
> }
>
>
> On Tue, Aug 23, 2022 at 4:46 AM Imre Rad <imrer@google.com> wrote:
>>
>> * src/basenc.c (ignore-padding):
>> Padding is optional for base64url encoding and many web
>> services produce and expect payload without padding.
>> New command line argument (--ignore-padding) for basenc
>> allows generating paddingless base64url outputs and
>> you can also use it for decoding without scary
>> warnings on stderr.
>> * tests/local.mk (reference to new test):
>> Reference to the new test.
>> * tests/misc/basenc-padding.sh (new test):
>> Tests covering the new feature.
>> ---
>> src/basenc.c | 48 ++++++++++++++++----
>> tests/local.mk | 1 +
>> tests/misc/basenc-padding.sh | 86 ++++++++++++++++++++++++++++++++++++
>> 3 files changed, 127 insertions(+), 8 deletions(-)
>> create mode 100755 tests/misc/basenc-padding.sh
>>
>> diff --git a/src/basenc.c b/src/basenc.c
>> index 04857d59e..9b020eccb 100644
>> --- a/src/basenc.c
>> +++ b/src/basenc.c
>> @@ -68,7 +68,8 @@ enum
>> BASE16_OPTION,
>> BASE2MSBF_OPTION,
>> BASE2LSBF_OPTION,
>> - Z85_OPTION
>> + Z85_OPTION,
>> + IGNORE_PADDING_OPTION
>> };
>> #endif
>>
>> @@ -78,6 +79,7 @@ static struct option const long_options[] =
>> {"wrap", required_argument, 0, 'w'},
>> {"ignore-garbage", no_argument, 0, 'i'},
>> #if BASE_TYPE == 42
>> + {"ignore-padding", no_argument, 0, IGNORE_PADDING_OPTION},
>> {"base64", no_argument, 0, BASE64_OPTION},
>> {"base64url", no_argument, 0, BASE64URL_OPTION},
>> {"base32", no_argument, 0, BASE32_OPTION},
>> @@ -146,6 +148,8 @@ Base%d encode or decode FILE, or standard input,
>> to standard output.\n\
>> "), stdout);
>> #if BASE_TYPE == 42
>> fputs (_("\
>> + --ignore-padding base64url only: ignore missing padding at
>> decoding,\n\
>> + don't pad at encoding\n\
>> --z85 ascii85-like encoding (ZeroMQ spec:32/Z85);\n\
>> when encoding, input length must be a multiple of
>> 4;\n\
>> when decoding, input length must be a multiple of
>> 5\n\
>> @@ -335,7 +339,6 @@ base64url_decode_ctx_init_wrapper (struct
>> base_decode_context *ctx)
>> init_inbuf (ctx);
>> }
>>
>> -
>> static bool
>> base64url_decode_ctx_wrapper (struct base_decode_context *ctx,
>> char const *restrict in, idx_t inlen,
>> @@ -368,7 +371,16 @@ base64url_decode_ctx_wrapper (struct
>> base_decode_context *ctx,
>> return b;
>> }
>>
>> -
>> +static bool
>> +base64url_decode_ctx_wrapper_no_padding (struct base_decode_context *ctx,
>> + char const *restrict in, idx_t inlen,
>> + char *restrict out, idx_t *outlen)
>> +{
>> + bool b = base64url_decode_ctx_wrapper(ctx, in, inlen, out, outlen);
>> + if (!b && inlen == 0) // inlen 0 indicates the final round, see
>> do_decode
>> + b = true;
>> + return b;
>> +}
>>
>> static int
>> base32_length_wrapper (int len)
>> @@ -964,7 +976,7 @@ finish_and_exit (FILE *in, char const *infile)
>> }
>>
>> static _Noreturn void
>> -do_encode (FILE *in, char const *infile, FILE *out, idx_t wrap_column)
>> +do_encode (FILE *in, char const *infile, FILE *out, idx_t
>> wrap_column, bool without_padding)
>> {
>> idx_t current_column = 0;
>> char *inbuf, *outbuf;
>> @@ -989,9 +1001,17 @@ do_encode (FILE *in, char const *infile, FILE
>> *out, idx_t wrap_column)
>> {
>> /* Process input one block at a time. Note that ENC_BLOCKSIZE
>> is sized so that no pad chars will appear in output. */
>> - base_encode (inbuf, sum, outbuf, BASE_LENGTH (sum));
>> + int to_write = BASE_LENGTH (sum);
>> + base_encode (inbuf, sum, outbuf, to_write);
>> + if (without_padding)
>> + {
>> + while (*(outbuf+to_write-1) == '=')
>> + {
>> + --to_write;
>> + }
>> + }
>>
>> - wrap_write (outbuf, BASE_LENGTH (sum), wrap_column,
>> + wrap_write (outbuf, to_write, wrap_column,
>> ¤t_column, out);
>> }
>> }
>> @@ -1084,6 +1104,13 @@ main (int argc, char **argv)
>> bool decode = false;
>> /* True if we should ignore non-base-alphabetic characters. */
>> bool ignore_garbage = false;
>> +
>> + /* True if we should ignore padding (base64url only). */
>> +#if BASE_TYPE == 42
>> + bool o_ignore_padding = false;
>> +#endif
>> + bool ignore_padding = false;
>> +
>> /* Wrap encoded data around the 76th column, by default. */
>> idx_t wrap_column = 76;
>>
>> @@ -1122,6 +1149,10 @@ main (int argc, char **argv)
>> break;
>>
>> #if BASE_TYPE == 42
>> + case IGNORE_PADDING_OPTION:
>> + o_ignore_padding = true;
>> + break;
>> +
>> case BASE64_OPTION:
>> case BASE64URL_OPTION:
>> case BASE32_OPTION:
>> @@ -1155,11 +1186,12 @@ main (int argc, char **argv)
>> break;
>>
>> case BASE64URL_OPTION:
>> + ignore_padding = o_ignore_padding;
>> base_length = base64_length_wrapper;
>> isbase = isbase64url;
>> base_encode = base64url_encode;
>> base_decode_ctx_init = base64url_decode_ctx_init_wrapper;
>> - base_decode_ctx = base64url_decode_ctx_wrapper;
>> + base_decode_ctx = ignore_padding ?
>> base64url_decode_ctx_wrapper_no_padding :
>> base64url_decode_ctx_wrapper;
>> break;
>>
>> case BASE32_OPTION:
>> @@ -1244,5 +1276,5 @@ main (int argc, char **argv)
>> if (decode)
>> do_decode (input_fh, infile, stdout, ignore_garbage);
>> else
>> - do_encode (input_fh, infile, stdout, wrap_column);
>> + do_encode (input_fh, infile, stdout, wrap_column, ignore_padding);
>> }
>> diff --git a/tests/local.mk b/tests/local.mk
>> index 0496c2873..e3d9d85e2 100644
>> --- a/tests/local.mk
>> +++ b/tests/local.mk
>> @@ -728,6 +728,7 @@ all_tests = \
>> tests/touch/read-only.sh \
>> tests/touch/relative.sh \
>> tests/touch/trailing-slash.sh \
>> + tests/misc/basenc-padding.sh \
>> $(all_root_tests)
>>
>> # See tests/factor/create-test.sh.
>> diff --git a/tests/misc/basenc-padding.sh b/tests/misc/basenc-padding.sh
>> new file mode 100755
>> index 000000000..c90f22fa1
>> --- /dev/null
>> +++ b/tests/misc/basenc-padding.sh
>> @@ -0,0 +1,86 @@
>> +#!/bin/sh
>> +# make sure base64url works fine without paddings
>> +
>> +# Copyright (C) 2000-2022 Free Software Foundation, Inc.
>> +
>> +# This program is free software: you can redistribute it and/or modify
>> +# it under the terms of the GNU General Public License as published by
>> +# the Free Software Foundation, either version 3 of the License, or
>> +# (at your option) any later version.
>> +
>> +# This program is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> +# GNU General Public License for more details.
>> +
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program. If not, see <https://www.gnu.org/licenses/>.
>> +
>> +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
>> +
>> +
>> +
>> +input='xs?>>>x'
>> +
>> +# should be fine with wrapped writes
>> +b64url_encoded_wo_p="$(echo "$input" | basenc --base64url --ignore-padding)"
>> +
>> +if grep -q "=" <<< "$b64url_encoded_wo_p"; then
>> + echo "Paddings are not supposed to be present"
>> + Exit 1
>> +fi
>> +
>> +
>> +# note the 2>&1's, stderr should be clean
>> +output="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d
>> --ignore-padding 2>&1)"
>> +
>> +if [ "$input" != "$output" ]; then
>> + echo "Something is wrong without paddings (wrapped writes)"
>> + Exit 1
>> +fi
>> +
>> +# decoding without --ignore-padding
>> +output_err="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d 2>&1)"
>> +if ! grep -q "invalid input" <<< "$output_err"; then
>> + echo "Decoding without --ignore-padding should still complain"
>> + Exit 1
>> +fi
>> +
>> +# decoding errors anywhere but the last round should still complain,
>> even with --ignore-padding
>> +known_broken="eHM_Pj4*eAo"
>> +output="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d
>> --ignore-padding 2>&1)"
>> +if ! grep -q "invalid input" <<< "$output_err"; then
>> + echo "Invalid encoding anywhere but the last round should still
>> be rejected"
>> + Exit 1
>> +fi
>> +
>> +# should be fine without wrapped writes
>> +b64url_encoded_wo_p="$(echo "$input" | basenc --base64url
>> --ignore-padding -w0)"
>> +
>> +# note the 2>&1's, stderr should be clean
>> +output="$(echo "$b64url_encoded_wo_p" | basenc --base64url -d
>> --ignore-padding 2>&1)"
>> +
>> +if [ "$input" != "$output" ]; then
>> + echo "Something is wrong without paddings (non-wrapped writes)"
>> + Exit 1
>> +fi
>> +
>> +# should be ok with padding as well
>> +b64url_encoded_w_p="$(echo "$input"| basenc --base64url)"
>> +
>> +# note the 2>&1's, stderr should be clean
>> +output="$(echo "$b64url_encoded_w_p" | basenc --base64url -d 2>&1)"
>> +
>> +if [ "$input" != "$output" ]; then
>> + echo "Something is wrong with paddings"
>> + Exit 1
>> +fi
>> +
>> +if [ "$b64url_encoded_w_p" == "b64url_encoded_wo_p" ]; then
>> + echo "Encoding with and without padding should look different"
>> + Exit 1
>> +fi
>> +
>> +#echo Padding test success
>> +
>> +Exit 0
>> --
>> 2.30.2
>>