[GNUnet-SVN] [ascension] 40/45: improvements, better case separation, more logging

[gnunet]

This is an automated email from the git hooks/post-receive script.

rexxnor pushed a commit to branch master
in repository ascension.

commit abc2641c8f0c5026a2754e3afed961bbf110dcae
Author: rexxnor <address@hidden>
AuthorDate: Fri Jan 11 10:43:45 2019 +0100

    improvements, better case separation, more logging
---
 ascension/ascension.py | 145 +++++++++++++++++++++++--------------------------
 1 file changed, 67 insertions(+), 78 deletions(-)

diff --git a/ascension/ascension.py b/ascension/ascension.py
index 44da1f8..116925a 100644
--- a/ascension/ascension.py
+++ b/ascension/ascension.py
@@ -48,6 +48,8 @@ class Ascender():
     @classmethod
     def __init__(cls, domain, transferns, port):
         cls.domain = domain
+        if cls.domain == '.':
+            cls.domain = cls.domain[:-1]
         cls.port = int(port)
         cls.soa = None
         cls.transferns = transferns
@@ -84,7 +86,7 @@ class Ascender():
             pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
                                     '-d'],
                                    stdout=sp.PIPE)
-            pkey_line = sp.Popen(['grep', domainpart],
+            pkey_line = sp.Popen(['grep', cls.domain],
                                  stdin=pkey_lookup.stdout,
                                  stdout=sp.PIPE)
             pkey_zone = sp.check_output(['cut', '-d',
@@ -106,7 +108,7 @@ class Ascender():
             pkey_lookup = sp.Popen([GNUNET_ZONE_CREATION_COMMAND,
                                     '-d'],
                                    stdout=sp.PIPE)
-            pkey_line = sp.Popen(['grep', domainpart],
+            pkey_line = sp.Popen(['grep', cls.domain],
                                  stdin=pkey_lookup.stdout,
                                  stdout=sp.PIPE)
             pkey_zone = sp.check_output(['cut', '-d', ' ', '-f3'],
@@ -117,12 +119,13 @@ class Ascender():
 
             # If it is TLD, don't add PKEY to higher zone
             if counter > 0:
-                result = sp.check_output([GNUNET_GNS_COMMAND,
+                result = sp.check_output([GNUNET_NAMESTORE_COMMAND,
+                                          '-D',
                                           '-t', 'PKEY',
-                                          '-u', '%s.%s' %
-                                          (domainpart,
-                                           reverse_parsing[counter - 1])])
-                if "No results." in result.decode():
+                                          '-z', '%s' % domainpart,
+                                          '-n', '%s' % reverse_parsing[counter 
- 1]
+                                         ])
+                if not "PKEY" in result.decode():
                     ret = sp.run([GNUNET_NAMESTORE_COMMAND,
                                   '-z', reverse_parsing[counter - 1],
                                   '-a', '-n', domainpart,
@@ -141,15 +144,14 @@ class Ascender():
 
         # Create the entire zone with name
         if counter != 0:
-            zonename = cls.domain[:-1]
             try:
                 ret = sp.run([GNUNET_ZONE_CREATION_COMMAND,
-                              '-C', zonename],
+                              '-C', cls.domain],
                              stdout=sp.DEVNULL,
                              stderr=sp.DEVNULL)
                 logging.info("executed command: %s", " ".join(ret.args))
             except sp.CalledProcessError:
-                logging.info("Zone %s already exists!", domainpart)
+                logging.info("Zone %s already exists!", cls.domain)
 
     @classmethod
     def get_current_serial(cls, domain, resolver=None):
@@ -190,25 +192,24 @@ class Ascender():
         Extract necessary information from Generator
         """
         currentserial = int(cls.get_current_serial(cls.domain, cls.transferns))
-        zoneserial = int(cls.get_zone_serial(cls.domain[:-1]))
+        zoneserial = int(cls.get_zone_serial())
         if zoneserial == 0:
+            logging.info("zone does not exist yet")
             cls.initial_zone_transfer()
             cls.zone = dns.zone.from_xfr(cls.zonegenerator)
             cls.soa = cls.get_zone_soa(cls.zone)
-            return
         elif zoneserial < currentserial:
+            logging.info("zone is out of date")
             cls.initial_zone_transfer(serial=zoneserial)
             cls.zone = dns.zone.from_xfr(cls.zonegenerator)
             cls.soa = cls.get_zone_soa(cls.zone)
-            return
         elif zoneserial == currentserial:
-            logging.warning("Nothing to do!")
-            sys.exit(0)
+            logging.info("zone is up to date")
         # should be unnecessary but AXFR SOA might not equal to direct SOA
         else:
-            logging.critical("SOA serial is bigger than zone serial?")
-            print(zoneserial, currentserial)
-            sys.exit(1)
+            # because it runs as a daemon, ignore this case but log it
+            logging.warning("SOA serial is bigger than zone serial?")
+            logging.warning("zone: %s, current: %s", zoneserial, currentserial)
 
     @classmethod
     def add_records_to_gns(cls, flags="n"):
@@ -216,7 +217,6 @@ class Ascender():
         Extracts records from zone and adds them to GNS
         """
         logging.info("Starting to add records into GNS...")
-        zonename = cls.domain[:-1]
 
         # Defining FIFO Queue
         taskqueue = queue.Queue(maxsize=5)
@@ -250,7 +250,6 @@ class Ascender():
                         # modify value to fit gns syntax
                         rdtype, value = cls.transform_to_gns_format(test,
                                                                     rdtype,
-                                                                    zonename,
                                                                     cls.domain,
                                                                     label)
                         # skip record if value is none
@@ -264,7 +263,7 @@ class Ascender():
 
                 # add recordline to gns
                 if len(recordline) > 1:
-                    cls.add_recordline_to_gns(recordline, zonename, label)
+                    cls.add_recordline_to_gns(recordline, cls.domain, label)
 
                 taskqueue.task_done()
 
@@ -272,30 +271,24 @@ class Ascender():
         thread = threading.Thread(target=worker)
         thread.start()
 
-        ## Add glue and rest of A/AAAA records to zone
-        #for gluerecord in cls.zone.iterate_rdatasets(rdtype=dns.rdatatype.A):
-        #    taskqueue.put(gluerecord)
-        #for gluerecord in 
cls.zone.iterate_rdatasets(rdtype=dns.rdatatype.AAAA):
-        #    taskqueue.put(gluerecord)
-        ## Add NS records to zone
-        #for nsrecord in cls.zone.iterate_rdatasets(rdtype=dns.rdatatype.NS):
-        #    taskqueue.put(nsrecord)
-        # Add remaining records to zone
         customrdataset = dict()
-        for remaining in cls.zone.iterate_rdatasets():
-            rdataset = remaining[1]
-            if customrdataset.get(str(remaining[0])) is None:
-                work = list()
-                work.append(rdataset)
-                customrdataset[str(remaining[0])] = work
-            else:
-                customrdataset[str(remaining[0])].append(rdataset)
-
-        for label in customrdataset.keys():
-            value = customrdataset.get(label)
-            if value is None:
-                continue
-            taskqueue.put((label, value))
+        try:
+            for remaining in cls.zone.iterate_rdatasets():
+                rdataset = remaining[1]
+                if customrdataset.get(str(remaining[0])) is None:
+                    work = list()
+                    work.append(rdataset)
+                    customrdataset[str(remaining[0])] = work
+                else:
+                    customrdataset[str(remaining[0])].append(rdataset)
+
+            for label, value in customrdataset.items():
+                if value is None:
+                    continue
+                taskqueue.put((label, value))
+        except AttributeError:
+            logging.info("skipping up to date zone")
+            return
 
         # Block until all tasks are done
         taskqueue.join()
@@ -306,7 +299,7 @@ class Ascender():
 
         # Add soa record to GNS once completed (updates the previous one)
         soa = cls.get_zone_soa(cls.zone)
-        cls.add_soa_record_to_gns(soa, zonename, cls.domain)
+        cls.add_soa_record_to_gns(soa, cls.domain)
         logging.info("All records have been added!")
 
     @staticmethod
@@ -328,26 +321,24 @@ class Ascender():
             logging.info("successfully added record with name %s", ' 
'.join(ret.args))
 
     @staticmethod
-    def transform_to_gns_format(record, rdtype, zonename, domain, label):
+    def transform_to_gns_format(record, rdtype, zonename, label):
         """
         Teansforms value of record to gnunet compatible format
         :param record: record to transform
         :param rdtype: record value to transform
         :param zonename: name of the zone to add to
-        :param domain: domain of the zone to add
-        :param domain: label under which the record is stored
+        :param label: label under which the record is stored
         """
         value = str(record)
         if rdtype == 'SOA':
             zonetuple = str(value).split(' ')
-            domain = str(".".join(domain.split('.')[:-1]))
             authns, owner, serial, refresh, retry, expiry, irefresh = zonetuple
             if authns[-1] == '.':
                 authns = authns[:-1]
             if owner[-1] == '.':
                 owner = owner[:-1]
             value = "rname=%s.%s mname=%s.%s %d,%d,%d,%d,%d" % (
-                authns, domain, owner, domain,
+                authns, zonename, owner, zonename,
                 int(serial), int(refresh), int(retry),
                 int(expiry), int(irefresh)
             )
@@ -363,12 +354,12 @@ class Ascender():
                 value = value[:-1]
             else:
                 value = "%s.%s" % (value, zonename)
-            if domain[-1] == ".":
-                domain = domain[:-1]
+            if zonename[-1] == ".":
+                zonename = zonename[:-1]
             if nameserver[-1] == ".":
                 dnsresolver = nameserver[:-1]
             else:
-                dnsresolver = "%s.%s" % (nameserver, domain)
+                dnsresolver = "%s.%s" % (nameserver, zonename)
             value = 'address@hidden' % (str(label), zonename, dnsresolver)
             logging.info("transformed %s record to GNS2DNS format", rdtype)
             rdtype = 'GNS2DNS'
@@ -406,17 +397,17 @@ class Ascender():
             logging.info("Did not transform record of type: %s", rdtype)
         return (rdtype, value)
 
-    @staticmethod
-    def get_zone_serial(zonename):
+    @classmethod
+    def get_zone_serial(cls):
         """
         Extracts the current serial from a given zone
         """
-        if zonename[-1] == '.':
-            zonename = zonename[:-1]
         try:
-            serial = sp.check_output([GNUNET_GNS_COMMAND,
+            serial = sp.check_output([GNUNET_NAMESTORE_COMMAND,
                                       '-t', 'SOA',
-                                      '-u', '@.%s' % zonename])
+                                      '-z', cls.domain,
+                                      '-n', '@',
+                                      '-D'])
             serial = serial.decode()
         except sp.CalledProcessError:
             serial = ""
@@ -428,17 +419,15 @@ class Ascender():
             soa_serial = 0
         return soa_serial
 
-    @staticmethod
-    def get_zone_refresh_time(zonename):
+    @classmethod
+    def get_zone_refresh_time(cls):
         """
         Extracts the current serial from a given zone
         """
-        if zonename[-1] == '.':
-            zonename = zonename[:-1]
         try:
             serial = sp.check_output([GNUNET_GNS_COMMAND,
                                       '-t', 'SOA',
-                                      '-u', '@.%s' % zonename])
+                                      '-u', '@.%s' % cls.domain])
             serial = serial.decode()
         except sp.CalledProcessError:
             serial = ""
@@ -450,17 +439,15 @@ class Ascender():
             refresh = 0
         return refresh
 
-    @staticmethod
-    def get_zone_retry_time(zonename):
+    @classmethod
+    def get_zone_retry_time(cls):
         """
         Extracts the current serial from a given zone
         """
-        if zonename[-1] == '.':
-            zonename = zonename[:-1]
         try:
             serial = sp.check_output([GNUNET_GNS_COMMAND,
                                       '-t', 'SOA',
-                                      '-u', '@.%s' % zonename])
+                                      '-u', '@.%s' % cls.domain])
             serial = serial.decode()
         except sp.CalledProcessError:
             serial = ""
@@ -487,7 +474,6 @@ class Ascender():
 
     @staticmethod
     def add_srv_record_to_gns(record, zonename):
-        # TODO Add support for SRV records
         """
         Adds a SRV record to GNS
         """
@@ -507,20 +493,20 @@ class Ascender():
                           '-t', rtype_str,
                           '-V', str(value),
                           '-e', '%ds' % ttl])
-            logging.info("executed command: %s", " ".join(ret.args))
+            logging.info("executed command: %s", debug)
             if ret.returncode != 0:
                 logging.warning("failed to add %s record %s",
                                 rtype_str, dnsname_str)
 
     @staticmethod
-    def add_soa_record_to_gns(record, zonename, domain):
+    def add_soa_record_to_gns(record, zonename):
         """
         Adds a SOA record to GNS
         """
         # needs to be added in any case
         _, ttl, rdata = record
         zonetuple = str(rdata).split(' ')
-        domain = str(".".join(domain.split('.')[:-1]))
+        zonename = str(".".join(zonename.split('.')[:-1]))
         authns, owner, serial, refresh, retry, expiry, irefresh = zonetuple
         if authns[-1] == '.':
             authns = authns[:-1]
@@ -531,7 +517,7 @@ class Ascender():
                       '-a', '-n', '@',
                       '-t', 'SOA',
                       '-V', "rname=%s.%s mname=%s.%s %d,%d,%d,%d,%d"
-                      % (authns, domain, owner, domain,
+                      % (authns, zonename, owner, zonename,
                          int(serial), int(refresh), int(retry),
                          int(expiry), int(irefresh)
                         ),
@@ -586,19 +572,22 @@ def main():
 
     # Event loop for actual daemon
     while 1:
-        serial = ascender.get_zone_serial(domain)
+        serial = ascender.get_zone_serial()
         ascender.initial_zone_transfer(serial)
         ascender.bootstrap_zone()
         ascender.mirror_zone()
         ascender.add_records_to_gns()
-        refresh = ascender.get_zone_refresh_time(domain)
-        retry = ascender.get_zone_retry_time(domain)
+        logging.info("Finished migrating of the zone %s", ascender.domain)
+        refresh = int(ascender.get_zone_refresh_time())
+        retry = int(ascender.get_zone_retry_time())
         if refresh == 0:
+            logging.info("unable to refresh zone, retrying in %ds", retry)
             time.sleep(retry)
         else:
+            logging.info("refreshing zone in %ds", refresh)
             time.sleep(refresh)
 
 if __name__ == '__main__':
-    PIDFILE = '/tmp/%s' % 'ascension'
+    PIDFILE = daemon.pidfile.TimeoutPIDLockFile
     DAEMON = Daemonize(app="ascension", pid=PIDFILE, action=main)
     DAEMON.start()

-- 
To stop receiving notification emails like this one, please contact
address@hidden