[PATCH] decorate: combined sort of IPv4 and IPv6 addresses

[Erik Auerswald]

IPv4 and IPv6 addresses can be seen as IP addresses.  Logs from a
"dual-stack" application, e.g., a web server, may contain either an IPv4
or IPv6 address at a given position in each line.  Thus if one wants to
sort the log file on the IP address, both IPv4 and IPv6 addresses need
to be accepted as sort key and sorted consistently.

There are two common methods for accomodating IPv4 addresses in IPv4:
IPv4-Mapped addresses and the deprecated IPv4-Compatible addresses.
Both can be used to convert a given IPv4 address to an IPv6 address.
Both IPv4-Mapped and IPv4-Compatible IPv6 address ranges are reserved
by IANA and always represent IPv4 addresses in a dual stack enabled
application.  IPv4-Compatible addresses just add 96 leading zero bits to
the 32 bit IPv4 address to create a 128 bit IPv6 address.  This results in
an ambiguity for the unspecified address (all-zero in both IPv4 and IPv6)
and the IPv6 localhost address ::1 with the first host address of "this"
network in IPv4 (0.0.0.1).  IPv4-Mapped addresses avoid this ambiguity.

This commit adds two conversion methods to decorate: ipv6v4map and
ipv6v4comp.

Functionality like this was requested for sort from GNU Coreutils:
  https://lists.gnu.org/archive/html/coreutils/2011-06/msg00078.html
  https://lists.gnu.org/r/bug-coreutils/2015-06/msg00039.html

It was rejected for sort from GNU Coreutils:
  https://www.gnu.org/software/coreutils/rejected_requests.html#sort
  https://lists.gnu.org/r/bug-coreutils/2015-06/msg00041.html

Thus it seems appropriate for decorate(1).

* NEWS: Mention new decorate feature.
* src/decorate-functions.c: Implement new conversions.
* tests/decorate-errors.pl, tests/decorate-sort-tests.pl,
  tests/decorate-tests.pl: Add tests for new conversions.
---
 NEWS                         |  7 +++++
 src/decorate-functions.c     | 61 +++++++++++++++++++++++++++++++-----
 tests/decorate-errors.pl     | 24 ++++++++++++++
 tests/decorate-sort-tests.pl | 42 +++++++++++++++++++++++++
 tests/decorate-tests.pl      | 44 ++++++++++++++++++++++++++
 5 files changed, 171 insertions(+), 7 deletions(-)

diff --git a/NEWS b/NEWS
index ee831ad..15b0bfa 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,12 @@
 * Noteworthy changes in release ?.? (?-?-?) []
 
+** New Features
+
+  Decorate now supports sorting IP addresses of both versions 4 and 6
+  together. IPv4 addresses are logically converted to IPv6 addresses,
+  either as IPv4-Mapped (ipv6v4map) or IPv4-Compatible (ipv6v4comp)
+  addresses.
+
 ** Bug Fixes
 
   Datamash now passes the -z/--zero-terminated flag to the sort(1) child
diff --git a/src/decorate-functions.c b/src/decorate-functions.c
index c50387f..6b28492 100644
--- a/src/decorate-functions.c
+++ b/src/decorate-functions.c
@@ -178,14 +178,61 @@ decorate_ipv6 (const char* in)
 }
 
 
+static bool
+decorate_ipv6_ipv4 (const char* in, uint32_t mapping)
+{
+  struct in_addr adr4;
+  struct in6_addr adr6;
+  int s4, s6;
+
+  s4 = inet_pton (AF_INET, in, &adr4);
+  s6 = inet_pton (AF_INET6, in, &adr6);
+
+  if (s4 < 0 && s6 < 0)
+    die (SORT_FAILURE, errno, _("inet_pton failed for AF_INET and AF_INET6"));
+
+  if (!(s4 > 0 || s6 > 0))
+    {
+      error (0, 0, _("invalid IP address %s"), quote (in));
+      return false;
+    }
+
+  if (s6)
+    for (int i=0;i<16;++i)
+      printf ("%02X", adr6.s6_addr[i]);
+  else
+    printf ("%024X%08X", mapping, ntohl (adr4.s_addr));
+
+  return true;
+}
+
+
+bool
+decorate_ipv6_ipv4_mapped (const char* in)
+{
+  return decorate_ipv6_ipv4 (in, 0xFFFF);
+}
+
+
+bool
+decorate_ipv6_ipv4_compat (const char* in)
+{
+  return decorate_ipv6_ipv4 (in, 0);
+}
+
+
 
 struct conversions_t builtin_conversions[] = {
-  { "as-is",    "copy as-is", decorate_as_is },     /* for debugging */
-  { "roman",    "roman numerals", decorate_roman_numerals },
-  { "strlen",   "length (in bytes) of the specified field", decorate_strlen },
-  { "ipv4",     "dotted-decimal IPv4 addresses", decorate_ipv4_dot_decimal },
-  { "ipv6",     "IPv6 addresses", decorate_ipv6 },
-  { "ipv4inet", "number-and-dots IPv4 addresses (incl. octal, hex values)",
+  { "as-is",      "copy as-is", decorate_as_is },     /* for debugging */
+  { "roman",      "roman numerals", decorate_roman_numerals },
+  { "strlen",     "length (in bytes) of the specified field", decorate_strlen 
},
+  { "ipv4",       "dotted-decimal IPv4 addresses", decorate_ipv4_dot_decimal },
+  { "ipv6",       "IPv6 addresses", decorate_ipv6 },
+  { "ipv4inet",   "number-and-dots IPv4 addresses (incl. octal, hex values)",
     decorate_ipv4_inet_addr },
-  { NULL,       NULL, 0 }
+  { "ipv6v4map",  "IPv6 and IPv4 (as IPv4-Mapped IPv6) addresses",
+    decorate_ipv6_ipv4_mapped},
+  { "ipv6v4comp", "IPv6 and IPv4 (as IPv4-Compatible IPv6) addresses",
+    decorate_ipv6_ipv4_compat},
+  { NULL,         NULL, 0 }
 };
diff --git a/tests/decorate-errors.pl b/tests/decorate-errors.pl
index c8f4202..9fb979b 100644
--- a/tests/decorate-errors.pl
+++ b/tests/decorate-errors.pl
@@ -154,6 +154,30 @@ my @Tests =
  ['c5', '--decorate -k1,1:ipv6' , {IN_PIPE=>"FOO\n"}, {OUT => " "}, {EXIT=>2},
   {ERR=>"$prog: invalid IPv6 address 'FOO'\n" .
         "$prog: conversion failed in line 1\n" }],
+ ['c6', '--decorate -k1,1:ipv6v4map', {IN_PIPE=>"FOO\n"}, {OUT => " "},
+  {EXIT=>2},
+  {ERR=>"$prog: invalid IP address 'FOO'\n" .
+        "$prog: conversion failed in line 1\n" }],
+ ['c7', '--decorate -k1,1:ipv6v4comp', {IN_PIPE=>"FOO\n"}, {OUT => " "},
+  {EXIT=>2},
+  {ERR=>"$prog: invalid IP address 'FOO'\n" .
+        "$prog: conversion failed in line 1\n" }],
+ ['c8', '--decorate -k1,1:ipv6v4map', {IN_PIPE=>"0\n"}, {OUT => " "},
+  {EXIT=>2},
+  {ERR=>"$prog: invalid IP address '0'\n" .
+        "$prog: conversion failed in line 1\n" }],
+ ['c9', '--decorate -k1,1:ipv6v4comp', {IN_PIPE=>"0\n"}, {OUT => " "},
+  {EXIT=>2},
+  {ERR=>"$prog: invalid IP address '0'\n" .
+        "$prog: conversion failed in line 1\n" }],
+ ['c10', '--decorate -k1,1:ipv6v4map', {IN_PIPE=>"\n"}, {OUT => " "},
+  {EXIT=>2},
+  {ERR=>"$prog: invalid IP address ''\n" .
+        "$prog: conversion failed in line 1\n" }],
+ ['c11', '--decorate -k1,1:ipv6v4comp', {IN_PIPE=>"\n"}, {OUT => " "},
+  {EXIT=>2},
+  {ERR=>"$prog: invalid IP address ''\n" .
+        "$prog: conversion failed in line 1\n" }],
 
 
   # on a different architecture, would printf(%Lg) print something else?
diff --git a/tests/decorate-sort-tests.pl b/tests/decorate-sort-tests.pl
index bada9f8..88b75ec 100644
--- a/tests/decorate-sort-tests.pl
+++ b/tests/decorate-sort-tests.pl
@@ -177,6 +177,45 @@ D   192.168.17.8
 M   192.168.43.1
 EOF
 
+my $in2=<<'EOF';
+203.0.113.47
+192.0.2.33
+203.0.113.0
+192.0.2.3
+0.0.0.0
+::ffff:192.0.2.42
+2001:db8:6:5:4:3:2:1
+2001:Db8::
+::192.0.2.41
+::1
+EOF
+
+my $out2_ipv6v4map=<<'EOF';
+::1
+::192.0.2.41
+0.0.0.0
+192.0.2.3
+192.0.2.33
+::ffff:192.0.2.42
+203.0.113.0
+203.0.113.47
+2001:Db8::
+2001:db8:6:5:4:3:2:1
+EOF
+
+my $out2_ipv6v4comp=<<'EOF';
+0.0.0.0
+::1
+192.0.2.3
+192.0.2.33
+::192.0.2.41
+203.0.113.0
+203.0.113.47
+::ffff:192.0.2.42
+2001:Db8::
+2001:db8:6:5:4:3:2:1
+EOF
+
 
 my @Tests =
 (
@@ -195,6 +234,9 @@ my @Tests =
  ['s11', '-k2n,2 -k1,1:roman' , {IN_PIPE=>$in1},
   {OUT => $out1_dec_k2n_roman}],
 
+ ['s12', '-k1,1:ipv6v4map',  {IN_PIPE=>$in2}, {OUT => $out2_ipv6v4map}],
+ ['s13', '-k1,1:ipv6v4comp', {IN_PIPE=>$in2}, {OUT => $out2_ipv6v4comp}],
+
 
  # Sort with header lines
  ['sh1', '-H -k2,2r:ipv4', {IN_PIPE=>$in1}, {OUT=>$out1_dec_ipv4_rev_header1}],
diff --git a/tests/decorate-tests.pl b/tests/decorate-tests.pl
index 5131bf1..87a4a95 100755
--- a/tests/decorate-tests.pl
+++ b/tests/decorate-tests.pl
@@ -222,6 +222,47 @@ C0A81164 C   192.168.17.100
 C0A8110A L   192.168.17.10
 EOF
 
+my $in4=<<'EOF';
+203.0.113.47
+192.0.2.33
+203.0.113.0
+192.0.2.3
+0.0.0.0
+::ffff:192.0.2.42
+2001:db8:6:5:4:3:2:1
+2001:Db8::
+::192.0.2.41
+::1
+::
+EOF
+
+my $out4_ipv6v4map=<<'EOF';
+00000000000000000000FFFFCB00712F 203.0.113.47
+00000000000000000000FFFFC0000221 192.0.2.33
+00000000000000000000FFFFCB007100 203.0.113.0
+00000000000000000000FFFFC0000203 192.0.2.3
+00000000000000000000FFFF00000000 0.0.0.0
+00000000000000000000FFFFC000022A ::ffff:192.0.2.42
+20010DB8000600050004000300020001 2001:db8:6:5:4:3:2:1
+20010DB8000000000000000000000000 2001:Db8::
+000000000000000000000000C0000229 ::192.0.2.41
+00000000000000000000000000000001 ::1
+00000000000000000000000000000000 ::
+EOF
+
+my $out4_ipv6v4comp=<<'EOF';
+000000000000000000000000CB00712F 203.0.113.47
+000000000000000000000000C0000221 192.0.2.33
+000000000000000000000000CB007100 203.0.113.0
+000000000000000000000000C0000203 192.0.2.3
+00000000000000000000000000000000 0.0.0.0
+00000000000000000000FFFFC000022A ::ffff:192.0.2.42
+20010DB8000600050004000300020001 2001:db8:6:5:4:3:2:1
+20010DB8000000000000000000000000 2001:Db8::
+000000000000000000000000C0000229 ::192.0.2.41
+00000000000000000000000000000001 ::1
+00000000000000000000000000000000 ::
+EOF
 
 my @Tests =
 (
@@ -242,6 +283,9 @@ my @Tests =
   {OUT=>$out1_strlen3}],
  ['d11', '--decorate -k1,1:strlen -k2.2,2:strlen' ,  {IN_PIPE=>$in1},
   {OUT=>$out1_strlen4}],
+ ['d12', '--decorate -k1:ipv6v4map', {IN_PIPE=>$in4}, {OUT=>$out4_ipv6v4map}],
+ ['d13', '--decorate -k1:ipv6v4comp', {IN_PIPE=>$in4},
+  {OUT=>$out4_ipv6v4comp}],
 
  ## basic undecoration
  ['u1', '--undecorate 1' , {IN_PIPE=>$out1_dec_roman}, {OUT => $in1}],
-- 
2.17.1