[GNUnet-SVN] [ascension] 21/45: fixed serial fetching and added serialization of zone

[gnunet]

This is an automated email from the git hooks/post-receive script.

rexxnor pushed a commit to branch master
in repository ascension.

commit 666f93fac2cece64606f66d1b457ae0a009cd59a
Author: rexxnor <address@hidden>
AuthorDate: Fri Oct 12 11:42:14 2018 +0200

    fixed serial fetching and added serialization of zone
---
 gnsmigrator/gnsmigrator.py | 91 ++++++++++++++++++++++++++++++----------------
 1 file changed, 60 insertions(+), 31 deletions(-)

diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
index f11c211..ba3819e 100644
--- a/gnsmigrator/gnsmigrator.py
+++ b/gnsmigrator/gnsmigrator.py
@@ -23,6 +23,8 @@ from ctypes import c_void_p
 from enum import Enum
 from dataclasses import dataclass
 import multiprocessing
+import gzip
+import pickle
 import queue
 import re
 import sys
@@ -395,12 +397,15 @@ class BaseMigrator():
         """
         Extracts the current serial from a given zone
         """
+        if zonename[-1] == '.':
+            zonename = zonename[:-1]
         serial = sp.check_output([GNUNET_GNS_COMMAND,
                                   '-t', 'SOA',
                                   '-u', '@.%s' % zonename])
-        soapattern = re.compile(r'.+\s(\d+,)\d+,+\d+,\d+,\d+')
-        if re.match(soapattern, serial):
-            soa_serial = re.match(soapattern, serial)
+        serial = serial.decode()
+        soapattern = re.compile(r'.+\s(\d+),\d+,+\d+,\d+,\d+', re.M)
+        if re.findall(soapattern, serial):
+            soa_serial = re.findall(soapattern, serial)[0]
         else:
             soa_serial = 0
         return soa_serial
@@ -427,11 +432,17 @@ class TLDMigrator(BaseMigrator):
         cls.zonegenerator = {}
 
     @classmethod
-    def initial_zone_transfer(cls):
+    def initial_zone_transfer(cls, serial):
         """
         Transfer and initialize the zone
         """
-        cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld)
+        if serial:
+            cls.zonegenerator = dns.query.xfr(cls.transferns,
+                                              cls.tld,
+                                              rdtype=dns.rdatatype.IXFR,
+                                              serial=serial)
+        else:
+            cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld,)
 
     @classmethod
     def bootstrap_zone(cls):
@@ -455,33 +466,49 @@ class TLDMigrator(BaseMigrator):
         Extract necessary information from Generator
         """
         zone = None
+        # And to unzip and unpickle:
         try:
-            for message in cls.zonegenerator:
-                origin = message.origin
-                rdclass = message.answer[0].rdclass
-                if zone is None:
-                    if relativize:
-                        origin = message.origin
-                    else:
-                        origin = message.answer[0].name
+            with gzip.open('%s.zone.gz' % cls.tld[:-1], 'rb') as zonefile:
+                zone = pickle.load(zonefile)
+        except Exception:
+            pass
+
+        currentserial = cls.get_current_serial(cls.tld)
+        zoneserial = None
+        if zone:
+            zoneserial = zone[0][2].split(' ')[2].split(',')[0]
+        if currentserial != zoneserial:
+            try:
+                for message in cls.zonegenerator:
+                    origin = message.origin
                     rdclass = message.answer[0].rdclass
-                    zone = zone_factory(origin, rdclass, relativize=relativize)
-                for rrset in message.answer:
-                    znode = zone.nodes.get(rrset.name)
-                    if not znode:
-                        znode = zone.node_factory()
-                        zone.nodes[rrset.name] = znode
-                    zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
-                                               rrset.covers, True)
-                    zrds.update_ttl(rrset.ttl)
-                    for record  in rrset:
-                        record.choose_relativity(zone.origin, relativize)
-                        zrds.add(record)
-            if check_origin:
-                zone.check_origin()
-            cls.zone = zone
-        except Exception as transferexception:
-            print("Error occured during Zone transfer: %s" % transferexception)
+                    if zone is None:
+                        if relativize:
+                            origin = message.origin
+                        else:
+                            origin = message.answer[0].name
+                        rdclass = message.answer[0].rdclass
+                        zone = zone_factory(origin, rdclass, 
relativize=relativize)
+                    for rrset in message.answer:
+                        znode = zone.nodes.get(rrset.name)
+                        if not znode:
+                            znode = zone.node_factory()
+                            zone.nodes[rrset.name] = znode
+                        zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
+                                                   rrset.covers, True)
+                        zrds.update_ttl(rrset.ttl)
+                        for record  in rrset:
+                            record.choose_relativity(zone.origin, relativize)
+                            zrds.add(record)
+                if check_origin:
+                    zone.check_origin()
+                cls.zone = zone
+            except Exception as transferexception:
+                print("Error occured during Zone transfer: %s" % 
transferexception)
+
+            # To zip and pickle:
+            with gzip.open('%s.zone.gz' % cls.tld[:-1], 'wb') as zonefile:
+                pickle.dump(zone, zonefile)
 
     @classmethod
     def multithreaded_add_records_to_gns(cls):
@@ -539,6 +566,7 @@ class TLDMigrator(BaseMigrator):
         for thread in threads:
             thread.join()
 
+
 def main():
     """
     Initializes object and handles arguments
@@ -560,7 +588,8 @@ def main():
 
     if tld and transferns:
         migrator = TLDMigrator(tld, transferns)
-        migrator.initial_zone_transfer()
+        serial = migrator.get_current_serial(tld)
+        migrator.initial_zone_transfer(serial)
         migrator.bootstrap_zone()
         migrator.mirror_zone()
         migrator.multithreaded_add_records_to_gns()

-- 
To stop receiving notification emails like this one, please contact
address@hidden