[gnurl] 113/151: runtests: introduce --shallow to reduce huge torture tests

[gnunet]

This is an automated email from the git hooks/post-receive script.

ng0 pushed a commit to branch master
in repository gnurl.

commit 7c1bd035760faa5ad6d81b6280dffbdd7feb6302
Author: Daniel Stenberg <address@hidden>
AuthorDate: Tue Dec 10 14:29:48 2019 +0100

    runtests: introduce --shallow to reduce huge torture tests
    
    When set, shallow mode limits runtests -t to make no more than NUM fails
    per test case. If more are found, it will randomly discard entries until
    the number is right. The random seed can also be set.
    
    This is particularly useful when running MANY tests as then most torture
    failures will already fail the same functions over and over and make the
    total operation painfully tedious.
    
    Closes #4699
---
 tests/runtests.1  |  7 ++++++-
 tests/runtests.pl | 38 +++++++++++++++++++++++++++++++++++---
 2 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/tests/runtests.1 b/tests/runtests.1
index 18ddce483..bb36a8b44 100644
--- a/tests/runtests.1
+++ b/tests/runtests.1
@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <address@hidden>, et al.
+.\" * Copyright (C) 1998 - 2019, Daniel Stenberg, <address@hidden>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -84,6 +84,11 @@ Display run time statistics. (Requires Perl Time::HiRes 
module)
 Display full run time statistics. (Requires Perl Time::HiRes module)
 .IP "-s"
 Shorter output. Speaks less than default.
+.IP "--shallow=[num](,seed)"
+Used together with \fB-t\fP. This limits the number of tests to fail in
+torture mode to no more than 'num' per test case. If this reduces the amount,
+the given 'seed' will be used to randomly discard entries to fail until the
+amount is 'num'.
 .IP "-t[num]"
 Selects a \fBtorture\fP test for the given tests. This makes runtests.pl first
 run the tests once and count the number of memory allocations made. It then
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 961cdb753..6ee2cca20 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -321,6 +321,8 @@ my %runcert;      # cert file currently in use by an ssl 
running server
 my $torture;
 my $tortnum;
 my $tortalloc;
+my $shallow;
+my $shallowseed;
 
 #######################################################################
 # logmsg is our general message logging subroutine.
@@ -598,13 +600,34 @@ sub torture {
         return 0;
     }
 
-    logmsg " $count functions to make fail\n";
+    my @ttests = (1 .. $count);
+    if($shallow && ($shallow < $count)) {
+        my $discard = scalar(@ttests) - $shallow;
+        my $percent = sprintf("%.2f%%", $shallow * 100 / scalar(@ttests));;
+        logmsg " $count functions found, but only fail $shallow ($percent)\n";
+        while($discard) {
+            my $rm;
+            do {
+                # find a test to discard
+                $rm = rand(scalar(@ttests));
+            } while(!$ttests[$rm]);
+            $ttests[$rm] = undef;
+            $discard--;
+        }
+    }
+    else {
+        logmsg " $count functions to make fail\n";
+    }
 
-    for ( 1 .. $count ) {
+    for (@ttests) {
         my $limit = $_;
         my $fail;
         my $dumped_core;
 
+        if(!defined($limit)) {
+            # --shallow can undefine them
+            next;
+        }
         if($tortalloc && ($tortalloc != $limit)) {
             next;
         }
@@ -5022,6 +5045,14 @@ while(@ARGV) {
             $tortalloc = $1;
         }
     }
+    elsif($ARGV[0] =~ /--shallow=(\d+)(,|)(\d*)/) {
+        # Fail no more than this amount per tests when running
+        # torture.
+        my ($num, $seed)=($1,$3);
+        $shallow=$num;
+        $shallowseed=$seed?$seed:1234; # get a real seed later
+        srand($shallowseed); # make it predictable
+    }
     elsif($ARGV[0] eq "-a") {
         # continue anyway, even if a test fail
         $anyway=1;
@@ -5070,6 +5101,7 @@ while(@ARGV) {
         print <<EOHELP
 Usage: runtests.pl [options] [test selection(s)]
   -a       continue even if a test fails
+  -am      automake style output PASS/FAIL: [number] [name]
   -bN      use base port number N for test servers (default $base)
   -c path  use this curl executable
   -d       display server debug info
@@ -5085,7 +5117,7 @@ Usage: runtests.pl [options] [test selection(s)]
   -r       run time statistics
   -rf      full run time statistics
   -s       short output
-  -am      automake style output PASS/FAIL: [number] [name]
+  --shallow=[num](,seed) make the torture tests thinner
   -t[N]    torture (simulate function failures); N means fail Nth function
   -v       verbose output
   -vc path use this curl only to verify the existing servers

-- 
To stop receiving notification emails like this one, please contact
address@hidden.