[GNUnet-SVN] [ascension] 15/45: created baseclass and separated small from big zones

[gnunet]

This is an automated email from the git hooks/post-receive script.

rexxnor pushed a commit to branch master
in repository ascension.

commit 900f93c431db4ef2455e159d9c98261083407196
Author: rexxnor <address@hidden>
AuthorDate: Mon Oct 8 12:46:54 2018 +0200

    created baseclass and separated small from big zones
---
 gnsmigrator/gnsmigrator.py | 257 ++++++++++++++++++++++++++++++++-------------
 1 file changed, 183 insertions(+), 74 deletions(-)

diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
index 5b4d2e7..4a366c8 100644
--- a/gnsmigrator/gnsmigrator.py
+++ b/gnsmigrator/gnsmigrator.py
@@ -2,24 +2,25 @@
 """GNS Migrator
 
 Usage:
-    gnsmigrator.py (-c <csv> | -f <txtfile>)
-    gnsmigrator.py (-c <csv> | -f <txtfile>) -r <resolver>
+    gnsmigrator.py -t <tld> -ns <transferns>
+    gnsmigrator.py -f <txtfile>
     gnsmigrator.py -h | --help
     gnsmigrator.py -v | --version
 
 Options:
-    <csv>           CSV File containing domains to transfer
+    <tld>           Top level domain to migrate
     <txtfile>       Text File containing domains to transfer
-    <resolver>      DNS Server that resolves missing domains
+    <transferns>    DNS Server that does the zone transfer
     -h --help       Show this screen.
     -v --version    Show version.
 """
 
 # imports
+import multiprocessing
+import queue
 import sys
-import time
 import subprocess
-import csv
+import threading
 import dns.query
 import dns.resolver
 import dns.zone
@@ -31,9 +32,9 @@ GNUNET_NAMESTORE_COMMAND = 'gnunet-namestore'
 GNUNET_GNS_COMMAND = 'gnunet-gns'
 GNUNET_ARM_COMMAND = 'gnunet-arm'
 
-class GNSMigrator():
+class BaseMigrator():
     """
-    Class that provides functionality to migrate zones
+    Base class for migration
     """
     @classmethod
     def __init__(cls, domainlist):
@@ -65,7 +66,46 @@ class GNSMigrator():
                 continue
             cls.zones[domain] = (zone, (master_answer[0].address,
                                         domain,
-                                        zone.get_rdataset('@', 
dns.rdatatype.SOA).ttl))
+                                        zone.get_rdataset('@', 
dns.rdatatype.SOA).ttl,
+                                        0))
+
+    @classmethod
+    def refresh_zone(cls, domain, zonetuple, dnsresolver):
+        """
+        Refresh the zone using IXFR and the previous serial as reference
+
+        :param domain: The domain to transfer and migrate
+        :param zonetuple: The necessary data tuple for the transfer
+        :param dnsresolver: Optional user specified resolver for subdomains
+        """
+        zone, xfrinfo = zonetuple
+        zonename = cls.get_lowest_domain_part(domain)
+        cls.add_records_to_gns(zonename, zone, domain, dnsresolver)
+        newzone = dns.zone.Zone(domain)
+
+        # Ugly way to get serial
+        if xfrinfo[3] == 0:
+            oldserial = 0
+        else:
+            oldserial = int(str(zone.get_rdataset('@', 
dns.rdatatype.SOA)).split(' ')[5])
+            xfrinfo[3] = 1
+
+        # A normal BIND9 returns a normal AXFR response with the entire zone
+        # if the serial is newer. This is why there is no real incremental
+        # zone transfer using bind. This makes the merger_zones function
+        # unnecessary. Furthermore this try except block updates only if
+        # there is a newer zone availible (according to serial). The IXFR
+        # returns only a SOA record with a new serial if it has not changed
+        try:
+            newzone = dns.zone.from_xfr(dns.query.xfr(xfrinfo[0],
+                                                      xfrinfo[1],
+                                                      
rdtype=dns.rdatatype.IXFR,
+                                                      serial=oldserial))
+            cls.zones[domain] = (newzone, (xfrinfo[0],
+                                           xfrinfo[1],
+                                           zone.get_rdataset('@', 
dns.rdatatype.SOA).ttl))
+        except dns.zone.NoNS:
+            print('the zone for domain %s was not updated' % domain)
 
     @classmethod
     def bootstrap_zones(cls):
@@ -76,6 +116,7 @@ class GNSMigrator():
             counter = 0
             # building list with arguments
             reverse_parsing = domain.split('.')[::-1]
+            reverse_parsing = list(filter(None, reverse_parsing))
             for domainpart in reverse_parsing:
                 pkey_lookup = subprocess.Popen([GNUNET_ZONE_CREATION_COMMAND,
                                                 '-d'],
@@ -190,7 +231,6 @@ class GNSMigrator():
                                             '-V', '%s.%s' % (dnsname_str, 
domain),
                                             '-e', '%ds' % ttl])
 
-
     @staticmethod
     def get_lowest_domain_part(domain):
         """
@@ -198,62 +238,136 @@ class GNSMigrator():
         """
         return domain.split('.')[0]
 
-    @staticmethod
-    def merge_zones(domain, fullzone, incrementalzone):
-        """
-        Merges full zone with incremental zone
-        """
-        # The library sucks so I do it with string operations
-        fullset = set(fullzone.to_text().decode().split('\n'))
-        incrementalset = set(incrementalzone.to_text().decode().split('\n'))
-        merged = '\n'.join(fullset.union(incrementalset))
-        mergedzone = dns.zone.from_text(merged.encode(), origin=domain)
-        return mergedzone
+class ZoneMigrator(BaseMigrator):
+    """
+    Class that migrates small zones efficiently
+    """
+    @classmethod
+    def __init__(cls, domainlist):
+        BaseMigrator.__init__(domainlist)
 
+class TLDMigrator(BaseMigrator):
+    """
+    Class that migrates big zones (TLDs) efficiently
+    """
+    @classmethod
+    def __init__(cls, tld, transferns):
+        BaseMigrator.__init__(tld)
+        cls.tld = tld
+        cls.transferns = transferns
+        cls.zone = None
+        cls.zonegenerator = None
 
     @classmethod
-    def refresh_zone(cls, domain, zonetuple, dnsresolver):
+    def initial_zone_transfer(cls):
         """
-        Refresh the zone using IXFR and the previous serial as reference
-
-        :param domain: The domain to transfer and migrate
-        :param zonetuple: The necessary data tuple for the transfer
-        :param dnsresolver: Optional user specified resolver for subdomains
+        Transfer and initialize the zone
         """
-        zone, xfrinfo = zonetuple
-        zonename = cls.get_lowest_domain_part(domain)
-        cls.add_records_to_gns(zonename, zone, domain, dnsresolver)
-        newzone = dns.zone.Zone(domain)
+        cls.zonegenerator = dns.query.xfr(cls.transferns, cls.tld)
 
-        # Ugly way to get serial
-        oldserial = int(str(zone.get_rdataset('@', dns.rdatatype.SOA)).split(' 
')[5])
+    @classmethod
+    def bootstrap_zone(cls):
+        """
+        Creates the zone in gnunet
+        """
+        reverse_parsing = cls.tld.split('.')[::-1]
+        reverse_parsing = list(filter(None, reverse_parsing))
+        for domainpart in reverse_parsing:
+            try:
+                subprocess.run([GNUNET_ZONE_CREATION_COMMAND,
+                                '-C', domainpart])
+            except subprocess.CalledProcessError:
+                print("Zone %s already exists!" % domainpart)
 
-        # A normal BIND9 returns a normal AXFR response with the entire zone
-        # if the serial is newer. This is why there is no real incremental
-        # zone transfer using bind. This makes the merger_zones function
-        # unnecessary. Furthermore this try except block updates only if
-        # there is a newer zone availible (according to serial). The IXFR
-        # returns only a SOA record with a new serial if it has not changed
+    @classmethod
+    def mirror_zone(cls, zone_factory=dns.zone.Zone, relativize=True, 
check_origin=True):
+        """
+        Extract necessary information from Generator
+        """
+        zone = None
         try:
-            newzone = dns.zone.from_xfr(dns.query.xfr(xfrinfo[0],
-                                                      xfrinfo[1],
-                                                      
rdtype=dns.rdatatype.IXFR,
-                                                      serial=oldserial))
-            cls.zones[domain] = (newzone, (xfrinfo[0],
-                                           xfrinfo[1],
-                                           zone.get_rdataset('@', 
dns.rdatatype.SOA).ttl))
-        except dns.zone.NoNS:
-            print('the zone for domain %s was not updated' % domain)
+            for message in cls.zonegenerator:
+                origin = message.origin
+                rdclass = message.answer[0].rdclass
+                if zone is None:
+                    if relativize:
+                        origin = message.origin
+                    else:
+                        origin = message.answer[0].name
+                    rdclass = message.answer[0].rdclass
+                    zone = zone_factory(origin, rdclass, relativize=relativize)
+                for rrset in message.answer:
+                    znode = zone.nodes.get(rrset.name)
+                    if not znode:
+                        znode = zone.node_factory()
+                        zone.nodes[rrset.name] = znode
+                    zrds = znode.find_rdataset(rrset.rdclass, rrset.rdtype,
+                                               rrset.covers, True)
+                    zrds.update_ttl(rrset.ttl)
+                    for record  in rrset:
+                        record.choose_relativity(zone.origin, relativize)
+                        zrds.add(record)
+            if check_origin:
+                zone.check_origin()
+            cls.zone = zone
+        except Exception as e:
+            print("Error occured during Zone transfer: %s" % e)
 
-        # Merge old and new zone
-        # updatedzone = cls.merge_zones(domain, zone, newzone)
+    @classmethod
+    def multithreaded_add_records_to_gns(cls):
+        """
+        Extracts records from zone and adds them to GNS
+        """
+        print("Starting to add records into GNS...")
+        zonename = cls.tld.split('.')[0]
+
+        # Defining FIFO Queue
+        taskqueue = queue.Queue(maxsize=200)
+        # Defining worker
+        def worker():
+            while True:
+                record = taskqueue.get()
+                if record is None:
+                    break
+                # execute thing to run on item
+                dnsname, ttl, authns = record
+                authns = str(authns)[:-1]
+                subprocess.run([GNUNET_NAMESTORE_COMMAND,
+                                '-z', zonename,
+                                '-a', '-n', str(dnsname),
+                                '-t', 'GNS2DNS',
+                                '-V', 'address@hidden' % (str(dnsname),
+                                                    zonename,
+                                                    str(authns)),
+                                '-e', '%ds' % int(ttl)])
+                taskqueue.task_done()
+
+        # Create threads
+        threads = []
+        for _ in range(multiprocessing.cpu_count()):
+            thread = threading.Thread(target=worker)
+            thread.start()
+            threads.append(thread)
+
+        # Give workers stuff to do
+        for record in cls.zone.iterate_rdatas(rdtype=dns.rdatatype.NS):
+            taskqueue.put(record)
+
+        # Block until all tasks are done
+        taskqueue.join()
+
+        # Stop workers
+        for _ in range(multiprocessing.cpu_count()):
+            taskqueue.put(None)
+        for thread in threads:
+            thread.join()
 
 def main():
     """
     Initializes object and handles arguments
     """
     # argument parsing from docstring definition
-    args = docopt.docopt(__doc__, version='GNS Migrator 0.1.1')
+    args = docopt.docopt(__doc__, version='GNS Migrator 0.1.3')
 
     # Checks if GNUnet services are running
     try:
@@ -261,35 +375,30 @@ def main():
     except subprocess.TimeoutExpired:
         print('GNUnet Services are not running!')
         print('Exiting...')
-        return 1
+        sys.exit(1)
 
     dnsresolver = args.get('<resolver>', None)
-
-    domainlist = []
-
-    if args.get('<csv>', None):
-        csvfile = args['<csv>']
-        with open(csvfile, 'r') as openedcsv:
-            linereader = csv.reader(openedcsv, delimiter=' ', quotechar='|')
-            for domain in linereader:
-                domainlist += domain
-
-    if args.get('<txtfile>', None):
+    tld = args.get('<tld>', None)
+    transferns = args.get('<transferns>', None)
+    txtfile = args.get('<txtfile>', None)
+
+    if tld and transferns:
+        migrator = TLDMigrator(tld, transferns)
+        migrator.initial_zone_transfer()
+        migrator.bootstrap_zone()
+        migrator.mirror_zone()
+        migrator.multithreaded_add_records_to_gns()
+    elif txtfile:
+        domainlist = []
         txtfile = args['<txtfile>']
         with open(txtfile, 'r') as openedtxt:
             for line in openedtxt:
                 domainlist.append(line.rstrip())
-
-
-    gnsmigrator = GNSMigrator(domainlist)
-    gnsmigrator.bootstrap_zones()
-    gnsmigrator.initial_zone_transfer()
-
-
-    # TODO add a daemon for doing this and notify when the TTL has expired
-    for domain, zonetuple in gnsmigrator.zones.items():
-        # Returns a value 0 if not changed and 1 if changed
-        gnsmigrator.refresh_zone(domain, zonetuple, dnsresolver)
+        zonemigrator = ZoneMigrator(domainlist)
+        zonemigrator.initial_zone_transfer()
+        zonemigrator.bootstrap_zones()
+        for domain, zonetuple in zonemigrator.zones.items():
+            zonemigrator.refresh_zone(domain, zonetuple, dnsresolver)
 
 if __name__ == '__main__':
     main()

-- 
To stop receiving notification emails like this one, please contact
address@hidden