[Qemu-devel] [PATCH 08/36] qapi: add qapi2texi script

[marcandre . lureau]

From: Marc-André Lureau <address@hidden>

As the name suggests, the qapi2texi script converts JSON QAPI
description into a standalone texi file suitable for different target
formats.

It parses the following kind of blocks with some little variations:

  ##
  # = Section
  # == Subsection
  #
  # Some text foo with *emphasis*
  # 1. with a list
  # 2. like that
  #
  # And some code:
  # | <- do this
  # | -> get that
  #
  ##

  ##
  # @symbol
  #
  # Symbol body ditto ergo sum. Foo bar
  # baz ding.
  #
  # @arg: foo
  # @arg: #optional foo
  #
  # Returns: returns bla bla
  #
  #          Or bla blah
  #
  # Since: version
  # Notes: notes, comments can have
  #        - itemized list
  #        - like this
  #
  #        and continue...
  #
  # Example:
  #
  # -> { "execute": "quit" }
  # <- { "return": {} }
  #
  ##

Thanks to the json declaration, it's able to give extra information
about the type of arguments and return value expected.

Signed-off-by: Marc-André Lureau <address@hidden>
---
 scripts/qapi.py      |  88 +++++++++++++++-
 scripts/qapi2texi.py | 293 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 379 insertions(+), 2 deletions(-)
 create mode 100755 scripts/qapi2texi.py

diff --git a/scripts/qapi.py b/scripts/qapi.py
index 27894c1..2a9b6e5 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -112,6 +112,67 @@ class QAPIExprError(Exception):
             "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
 
 
+class QAPIDoc:
+    def __init__(self, comment):
+        self.symbol = None
+        self.comment = ""
+        self.args = OrderedDict()
+        self.meta = OrderedDict()
+        self.section = None
+
+        for line in comment.split('\n'):
+            sline = ' '.join(line.split())
+            split = sline.split(' ', 1)
+            key = split[0].rstrip(':')
+
+            if line.startswith(" @"):
+                key = key[1:]
+                sline = split[1] if len(split) > 1 else ""
+                if self.symbol is None:
+                    self.symbol = key
+                else:
+                    self.start_section(self.args, key)
+            elif self.symbol and \
+                    key in ("Since", "Returns", "Notes", "Note", "Example"):
+                sline = split[1] if len(split) > 1 else ""
+                line = None
+                self.start_section(self.meta, key)
+
+            if self.section and self.section[1] == "Example":
+                self.append_comment(line)
+            else:
+                self.append_comment(sline)
+
+        self.end_section()
+
+    def append_comment(self, line):
+        if line is None:
+            return
+        if self.section is not None:
+            if self.section[-1] == "" and line == "":
+                self.end_section()
+            else:
+                self.section.append(line)
+        elif self.comment == "":
+            self.comment = line
+        else:
+            self.comment += "\n" + line
+
+    def end_section(self):
+        if self.section is not None:
+            dic = self.section[0]
+            key = self.section[1]
+            doc = "\n".join(self.section[2:])
+            if key != "Example":
+                doc = doc.strip()
+            dic[key] = doc
+            self.section = None
+
+    def start_section(self, dic, key):
+        self.end_section()
+        self.section = [dic, key]  # .. remaining elems will be the doc
+
+
 class QAPISchemaParser(object):
 
     def __init__(self, fp, previously_included=[], incl_info=None):
@@ -127,11 +188,14 @@ class QAPISchemaParser(object):
         self.line = 1
         self.line_pos = 0
         self.exprs = []
+        self.comment = None
+        self.apidoc = incl_info['doc'] if incl_info else []
         self.accept()
 
         while self.tok is not None:
             expr_info = {'file': fname, 'line': self.line,
-                         'parent': self.incl_info}
+                         'parent': self.incl_info, 'doc': self.apidoc}
+            self.apidoc = []
             expr = self.get_expr(False)
             if isinstance(expr, dict) and "include" in expr:
                 if len(expr) != 1:
@@ -152,6 +216,8 @@ class QAPISchemaParser(object):
                     inf = inf['parent']
                 # skip multiple include of the same file
                 if incl_abs_fname in previously_included:
+                    expr_info['doc'].extend(self.apidoc)
+                    self.apidoc = expr_info['doc']
                     continue
                 try:
                     fobj = open(incl_abs_fname, 'r')
@@ -166,6 +232,12 @@ class QAPISchemaParser(object):
                              'info': expr_info}
                 self.exprs.append(expr_elem)
 
+    def append_doc(self):
+        if self.comment:
+            apidoc = QAPIDoc(self.comment)
+            self.apidoc.append(apidoc)
+            self.comment = None
+
     def accept(self):
         while True:
             self.tok = self.src[self.cursor]
@@ -174,8 +246,20 @@ class QAPISchemaParser(object):
             self.val = None
 
             if self.tok == '#':
-                self.cursor = self.src.find('\n', self.cursor)
+                end = self.src.find('\n', self.cursor)
+                line = self.src[self.cursor:end+1]
+                # start a comment section after ##
+                if line[0] == "#":
+                    if self.comment is None:
+                        self.comment = ""
+                # skip modeline
+                elif line.find("-*") == -1 and self.comment is not None:
+                    self.comment += line
+                if self.src[end] == "\n" and self.src[end+1] == "\n":
+                    self.append_doc()
+                self.cursor = end
             elif self.tok in ['{', '}', ':', ',', '[', ']']:
+                self.append_doc()
                 return
             elif self.tok == "'":
                 string = ''
diff --git a/scripts/qapi2texi.py b/scripts/qapi2texi.py
new file mode 100755
index 0000000..76ade1b
--- /dev/null
+++ b/scripts/qapi2texi.py
@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+# QAPI texi generator
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+"""This script produces the documentation of a qapi schema in texinfo format"""
+import re
+import sys
+
+from qapi import QAPISchemaParser, QAPISchemaError, check_exprs, QAPIExprError
+
+COMMAND_FMT = """
address@hidden {type} {{{ret}}} {name} @
+{{{args}}}
+
+{body}
+
address@hidden deftypefn
+
+""".format
+
+ENUM_FMT = """
address@hidden Enum {name}
+
+{body}
+
address@hidden deftp
+
+""".format
+
+STRUCT_FMT = """
address@hidden {type} {name} @
+{{{attrs}}}
+
+{body}
+
address@hidden deftp
+
+""".format
+
+EXAMPLE_FMT = """@example
+{code}
address@hidden example
+""".format
+
+
+def subst_emph(doc):
+    return re.sub(r'\*(\w*)\*', r'@emph{\1}', doc)
+
+
+def subst_vars(doc):
+    return re.sub(r'@(\w*)', r'@var{\1}', doc)
+
+
+def subst_braces(doc):
+    return doc.replace("{", "@{").replace("}", "@}")
+
+
+def texi_example(doc):
+    return EXAMPLE_FMT(code=subst_braces(doc).strip('\n'))
+
+
+def texi_comment(doc):
+    lines = []
+    doc = subst_vars(doc)
+    doc = subst_emph(doc)
+    inlist = False
+    lastempty = False
+    for line in doc.split('\n'):
+        empty = line == ""
+
+        if line.startswith("| "):
+            line = EXAMPLE_FMT(code=line[1:])
+        elif line.startswith("= "):
+            line = "@section " + line[1:]
+        elif line.startswith("== "):
+            line = "@subsection " + line[2:]
+        elif re.match("^([0-9]*[.) ]) ", line):
+            if not inlist:
+                lines.append("@enumerate")
+                inlist = "enumerate"
+            line = line[line.find(" ")+1:]
+            lines.append("@item")
+        elif re.match("^[o*-] ", line):
+            if not inlist:
+                lines.append("@itemize %s" % {'o': "@bullet",
+                                              '*': "@minus",
+                                              '-': ""}[line[0]])
+                inlist = "itemize"
+            lines.append("@item")
+            line = line[2:]
+        elif lastempty and inlist:
+            lines.append("@end %s\n" % inlist)
+            inlist = False
+
+        lastempty = empty
+        lines.append(line)
+
+    if inlist:
+        lines.append("@end %s\n" % inlist)
+    return "\n".join(lines)
+
+
+def texi_args(expr):
+    data = expr["data"] if "data" in expr else {}
+    if isinstance(data, str):
+        args = data
+    else:
+        args = []
+        for name, typ in data.iteritems():
+            # optional arg
+            if name.startswith("*"):
+                name = name[1:]
+                args.append("['%s': @var{%s}]" % (name, typ))
+            # regular arg
+            else:
+                args.append("'%s': @var{%s}" % (name, typ))
+        args = ", ".join(args)
+    return args
+
+
+def texi_body(doc, arg="@var"):
+    body = "@table %s\n" % arg
+    for arg, desc in doc.args.iteritems():
+        if desc.startswith("#optional"):
+            desc = desc[10:]
+            arg += "*"
+        body += "@item %s\n%s\n" % (arg, texi_comment(desc))
+    body += "@end table\n"
+    body += texi_comment(doc.comment)
+
+    for k in ("Returns", "Note", "Notes", "Since", "Example"):
+        if k not in doc.meta:
+            continue
+        func = texi_comment if k != "Example" else texi_example
+        body += "address@hidden address@hidden quotation" % \
+                (k, func(doc.meta[k]))
+    return body
+
+
+def texi_alternate(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Alternate",
+                      name=doc.symbol,
+                      attrs="[ " + args + " ]",
+                      body=body)
+
+
+def texi_union(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Union",
+                      name=doc.symbol,
+                      attrs="[ " + args + " ]",
+                      body=body)
+
+
+def texi_enum(_, doc):
+    body = texi_body(doc, "@samp")
+    return ENUM_FMT(name=doc.symbol,
+                    body=body)
+
+
+def texi_struct(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Struct",
+                      name=doc.symbol,
+                      attrs="@{ " + args + " @}",
+                      body=body)
+
+
+def texi_command(expr, doc):
+    args = texi_args(expr)
+    ret = expr["returns"] if "returns" in expr else ""
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Command",
+                       name=doc.symbol,
+                       ret=ret,
+                       args="(" + args + ")",
+                       body=body)
+
+
+def texi_event(expr, doc):
+    args = texi_args(expr)
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Event",
+                       name=doc.symbol,
+                       ret="",
+                       args="(" + args + ")",
+                       body=body)
+
+
+def parse_schema(fname):
+    try:
+        schema = QAPISchemaParser(open(fname, "r"))
+        check_exprs(schema.exprs)
+        return schema.exprs
+    except (QAPISchemaError, QAPIExprError), err:
+        print >>sys.stderr, err
+        exit(1)
+
+def main(argv):
+    if len(argv) != 5:
+        print >>sys.stderr, "%s: need exactly 4 arguments" % argv[0]
+        sys.exit(1)
+
+    exprs = parse_schema(argv[4])
+
+    print r"""
+\input texinfo
address@hidden {filename}
address@hidden en
address@hidden 0
address@hidden 0
+
address@hidden {title}
+
address@hidden
address@hidden
+* QEMU: (qemu-doc).    {title}
address@hidden direntry
address@hidden ifinfo
+
address@hidden
address@hidden {title} {version}
address@hidden titlepage
+
address@hidden
address@hidden Top
address@hidden
+
+This is the API reference for QEMU {version}.
+
address@hidden
+* API Reference::
+* Commands and Events Index::
+* Data Types Index::
address@hidden menu
+
address@hidden ifnottex
+
address@hidden
+
address@hidden API Reference
address@hidden API Reference
+
address@hidden man begin DESCRIPTION
+""".format(title=argv[1], version=argv[2], filename=argv[3])
+
+    for cmd in exprs:
+        try:
+            expr = cmd['expr']
+            docs = cmd['info']['doc']
+
+            (kind, _) = expr.items()[0]
+
+            for doc in docs[0:-1]:
+                print texi_body(doc)
+
+            texi = {"command": texi_command,
+                    "struct": texi_struct,
+                    "enum": texi_enum,
+                    "union": texi_union,
+                    "alternate": texi_alternate,
+                    "event": texi_event}
+            try:
+                print texi[kind](expr, docs[-1])
+            except KeyError:
+                raise ValueError("Unknown expression kind '%s'" % kind)
+        except:
+            print >>sys.stderr, "error at @%s" % cmd
+            raise
+
+    print """
address@hidden man end
+
address@hidden man begin SEEALSO
+The HTML documentation of QEMU for more precise information.
address@hidden man end
+
address@hidden Commands and Events Index
address@hidden Commands and Events Index
address@hidden fn
address@hidden Data Types Index
address@hidden Data Types Index
address@hidden tp
address@hidden
+"""
+
+if __name__ == "__main__":
+    main(sys.argv)
-- 
2.4.3