[lmi-commits] [lmi] master 9c510ad 16/22: Measure elapsed time for MD5 data-file validation

[Greg Chicares]

branch: master
commit 9c510ad08bb0ec0ee3e30fe28ec3ff085f0b90e5
Author: Gregory W. Chicares <address@hidden>
Commit: Gregory W. Chicares <address@hidden>

    Measure elapsed time for MD5 data-file validation
    
    It is anticipated that this commit will soon be reverted. This
    instrumentation could have been placed on a throwaway branch, but it's
    more convenient to keep it on the main trunk.
    
    Measure how long it takes to validate MD5 files by two methods:
     - an external md5sum program, as in the past; and
     - internally, as now.
    Print timings to std::cout, so that only developers see them.
    
    When run with '--pyx=measure_md5' and without '--ash_nazg', ignore
    caching and thus revalidate before each PDF illustration is printed.
    Reason: now that validation is faster, it may be desirable to address
    this marked defect:
    
      // TODO ?? Known security hole: data files can be modified after they
      // have been validated.
    
    by inhibiting caching (at least when neither '--ash_nazg' nor '--mellon'
    is given--i.e., for users who aren't allowed to use the product editor).
    
    Here are timings on a machine that's probably faster than the average
    corporate laptop, for a one-line 'validated.md5':
    
      /opt/lmi/data[0]$cat validated.md5
      5fc68a795c9c60da1b32be989efc299a *expiry
    
    /opt/lmi/bin[0]$wine ./lmi_wx_shared --mellon \
      --data_path=/opt/lmi/data --pyx=measure_md5
    Assay: production 1 milliseconds
    Assay: external program 96 milliseconds
    
    ...and for a maximal 'validated.md5' prepared with the full list of
    files in 'fardel_checksummed_files', using every known product, and
    generating PDF illustrations several times (hence the repeated timings):
    
    /opt/lmi/bin[1]$wine ./lmi_wx_shared --mellon \
      --data_path=/opt/lmi/data --pyx=measure_md5
    Assay: production 87 milliseconds
    Assay: external program 245 milliseconds
    Assay: internal 97 milliseconds
    Assay: production 114 milliseconds
    Assay: external program 180 milliseconds
    Assay: internal 115 milliseconds
    Assay: production 114 milliseconds
    Assay: external program 181 milliseconds
    Assay: internal 115 milliseconds
    Assay: production 115 milliseconds
    Assay: external program 183 milliseconds
    Assay: internal 119 milliseconds
    
    Is the extra security worth the extra delay? Only measurements on a
    typical end-user machine can guide that decision. If it's considered
    worthwhile, then the speed can be improved by eliminating some files
    from 'fardel_checksummed_files':
     - 'lmi_md5sum.exe', which is no longer required in production
     - '*.dat *.ndx *.xst', which are already not human readable
    Effect of eliminating those files:
    
    /opt/lmi/bin[0]$wine ./lmi_wx_shared --mellon \
      --data_path=/opt/lmi/data --pyx=measure_md5
    Assay: production 76 milliseconds
    Assay: external program 201 milliseconds
    Assay: internal 79 milliseconds
    Assay: production 60 milliseconds
    Assay: external program 129 milliseconds
    Assay: internal 65 milliseconds
    Assay: production 59 milliseconds
    Assay: external program 176 milliseconds
    Assay: internal 78 milliseconds
    
    It would probably be helpful to cache 'validated.md5' as well.
---
 authenticity.cpp | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 objects.make     |  4 ++++
 2 files changed, 71 insertions(+)

diff --git a/authenticity.cpp b/authenticity.cpp
index 06b540f..46e3c24 100644
--- a/authenticity.cpp
+++ b/authenticity.cpp
@@ -25,11 +25,15 @@
 
 #include "alert.hpp"
 #include "calendar_date.hpp"
+#include "contains.hpp"
 #include "global_settings.hpp"
 #include "handle_exceptions.hpp"
 #include "md5.hpp"
 #include "md5sum.hpp"
 #include "path_utility.hpp"             // fs::path inserter
+#include "platform_dependent.hpp"       // chdir()
+#include "system_command.hpp"
+#include "timer.hpp"
 
 #include <boost/filesystem/fstream.hpp>
 #include <boost/filesystem/operations.hpp>
@@ -37,6 +41,7 @@
 #include <cstdio>                       // fclose(), fopen()
 #include <cstdlib>                      // exit(), EXIT_FAILURE
 #include <cstring>                      // memcpy()
+#include <iostream>                     // cout, endl
 #include <sstream>
 #include <stdexcept>
 #include <vector>
@@ -69,11 +74,14 @@ std::string Authenticity::Assay
     ,fs::path const&      data_path
     )
 {
+    Timer timer;
+
     // The cached date is valid unless it's the peremptorily-invalid
     // default value of JDN zero.
     if
         (  calendar_date(jdn_t(0)) != Instance().CachedDate_
         && candidate               == Instance().CachedDate_
+        && !contains(global_settings::instance().pyx(), "measure_md5")
         )
         {
         return "cached";
@@ -240,6 +248,65 @@ std::string Authenticity::Assay
         }
     // Cache the validated date.
     Instance().CachedDate_ = candidate;
+
+    std::cout << "Assay: production " << timer.stop().elapsed_msec_str() << 
std::endl;
+
+    // MD5 !! Revert "measure_md5" instrumentation soon.
+  if(contains(global_settings::instance().pyx(), "measure_md5"))
+    {
+    try
+        {
+        timer.restart();
+        fs::path original_path(fs::current_path());
+        if(0 != chdir(data_path.string().c_str()))
+            {
+            oss
+                << "Unable to change directory to '"
+                << data_path
+                << "'. Try reinstalling."
+                ;
+            return oss.str();
+            }
+        system_command("md5sum --check --status " + (data_path / 
md5sum_file()).string());
+        if(0 != chdir(original_path.string().c_str()))
+            {
+            oss
+                << "Unable to restore directory to '"
+                << original_path
+                << "'. Try reinstalling."
+                ;
+            return oss.str();
+            }
+        std::cout << "Assay: external program " << 
timer.stop().elapsed_msec_str() << std::endl;
+
+        timer.restart();
+        auto const sums = md5_read_checksum_file(data_path / md5sum_file());
+        for(auto const& s : sums)
+            {
+            auto const file_path = data_path / s.filename;
+            auto const md5 = md5_calculate_file_checksum
+                (data_path / s.filename
+                ,s.file_mode
+                );
+            if(md5 != s.md5sum)
+                {
+                    throw std::runtime_error
+                        ( "Integrity check failed for '"
+                        + s.filename.string()
+                        + "'"
+                        );
+                }
+            }
+        std::cout << "Assay: internal " << timer.stop().elapsed_msec_str() << 
std::endl;
+        }
+    catch(...)
+        {
+        report_exception();
+        oss << "Failure in time measurements.";
+        return oss.str();
+        }
+    }
+
     return "validated";
 }
 
diff --git a/objects.make b/objects.make
index 8e86a5e..d1231cc 100644
--- a/objects.make
+++ b/objects.make
@@ -530,6 +530,7 @@ assert_lmi_test$(EXEEXT): \
   $(common_test_objects) \
   assert_lmi_test.o \
 
+# MD5 !! Remove "timer.o" below.
 authenticity_test$(EXEEXT): \
   $(boost_filesystem_objects) \
   $(common_test_objects) \
@@ -544,6 +545,7 @@ authenticity_test$(EXEEXT): \
   path_utility.o \
   system_command.o \
   system_command_non_wx.o \
+  timer.o \
 
 bourn_cast_test$(EXEEXT): \
   $(common_test_objects) \
@@ -1100,6 +1102,7 @@ lmi_md5sum$(EXEEXT): \
   md5sum.o \
   md5sum_cli.o \
 
+# MD5 !! Remove "timer.o" below.
 generate_passkey$(EXEEXT): \
   $(boost_filesystem_objects) \
   $(main_auxiliary_common_objects) \
@@ -1114,6 +1117,7 @@ generate_passkey$(EXEEXT): \
   path_utility.o \
   system_command.o \
   system_command_non_wx.o \
+  timer.o \
 
 ihs_crc_comp$(EXEEXT): \
   $(main_auxiliary_common_objects) \