From a23fa7edc9cad4b344fb597cf17be875b7ebda86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= Date: Tue, 10 May 2022 15:00:06 +0200 Subject: [PATCH 1/7] Add Sphinx extension to help with ARM maintenance and cross-linking The extension provides a "Sphinx domain factory". Each new Sphinx domain defines a namespace for configuration statements so named.conf and rndc.conf do not clash. Currently the Sphinx domains are instantiated twice and resuling domains are named "namedconf" and "rndcconf". This commit adds a single new directive: .. statement:: max-cache-size It is namespaced like this: .. namedconf:statement:: max-cache-size This directive generates a new anchor for configuration statement and it can be referenced like :any:`max-cache-size` (if the identifier is unique), or more specific :namedconf:ref:`max-cache-size`. It is based on Sphinx "tutorial" extension "recipe". Beware, some details in Sphinx docs are not up-to-date, it's better to read Sphinx and docutil sources. --- doc/arm/Makefile.am | 3 + doc/arm/_ext/iscconf.py | 176 ++++++++++++++++++++++++++++++++++++++ doc/arm/_ext/namedconf.py | 22 +++++ doc/arm/_ext/rndcconf.py | 22 +++++ doc/arm/conf.py | 11 ++- 5 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 doc/arm/_ext/iscconf.py create mode 100644 doc/arm/_ext/namedconf.py create mode 100644 doc/arm/_ext/rndcconf.py diff --git a/doc/arm/Makefile.am b/doc/arm/Makefile.am index 798d6ba839..d4aa22b767 100644 --- a/doc/arm/Makefile.am +++ b/doc/arm/Makefile.am @@ -61,6 +61,9 @@ EXTRA_DIST = \ troubleshooting.inc.rst \ tsig.inc.rst \ zones.inc.rst \ + _ext/iscconf.py \ + _ext/namedconf.py \ + _ext/rndcconf.py \ _static/custom.css \ ../dnssec-guide \ ../misc/acl.grammar.rst \ diff --git a/doc/arm/_ext/iscconf.py b/doc/arm/_ext/iscconf.py new file mode 100644 index 0000000000..8608e03cea --- /dev/null +++ b/doc/arm/_ext/iscconf.py @@ -0,0 +1,176 @@ +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +""" +Sphinx domains for ISC configuration files. + +Use setup() to install new Sphinx domains for ISC configuration files. + +This extension is based on combination of two Sphinx extension tutorials: +https://www.sphinx-doc.org/en/master/development/tutorials/todo.html +https://www.sphinx-doc.org/en/master/development/tutorials/recipe.html +""" + +from sphinx import addnodes +from sphinx.directives import ObjectDescription +from sphinx.domains import Domain +from sphinx.roles import XRefRole +from sphinx.util.nodes import make_refnode + + +# pylint: disable=too-many-statements +def domain_factory(domainname, domainlabel): + """ + Return parametrized Sphinx domain object. + @param domainname Name used when referencing domain in .rst: e.g. namedconf + @param confname Humand-readable name for texts, e.g. named.conf + """ + + class ISCConfDomain(Domain): + """ + Custom Sphinx domain for ISC config. + Provides .. statement:: directive to define config statement. + :ref:`statementname` works as usual. + + See https://www.sphinx-doc.org/en/master/extdev/domainapi.html + """ + + class StatementDirective(ObjectDescription): + """ + A custom directive that describes a statement, + e.g. max-cache-size. + """ + + has_content = True + required_arguments = 1 + option_spec = {} + + def handle_signature(self, sig, signode): + signode += addnodes.desc_name(text=sig) + return sig + + def add_target_and_index(self, _name_cls, sig, signode): + signode["ids"].append(domainname + "-statement-" + sig) + + iscconf = self.env.get_domain(domainname) + iscconf.add_statement(sig) + + name = domainname + label = domainlabel + + directives = { + "statement": StatementDirective, + } + + roles = {"ref": XRefRole(warn_dangling=True)} + initial_data = { + "statements": [], # object list for Sphinx API + } + + indices = {} # no custom indicies + + def get_objects(self): + """Sphinx API: iterable of object descriptions""" + for obj in self.data["statements"]: + yield obj + + # pylint: disable=too-many-arguments + def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): + """ + Sphinx API: + Resolve the pending_xref *node* with the given typ and target. + """ + match = [ + (docname, anchor) + for name, sig, typ, docname, anchor, _prio in self.get_objects() + if sig == target + ] + + if len(match) == 0: + return None + todocname = match[0][0] + targ = match[0][1] + + refnode = make_refnode( + builder, fromdocname, todocname, targ, contnode, targ + ) + return refnode + + def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode): + """ + Sphinx API: + Raising NotImplementedError uses fall-back bassed on resolve_xref. + """ + raise NotImplementedError + + def add_statement(self, signature): + """ + Add a new statement to the domain data structures. + No visible effect. + """ + name = "{}.{}.{}".format(domainname, "statement", signature) + anchor = "{}-statement-{}".format(domainname, signature) + + # Sphinx API: name, dispname, type, docname, anchor, priority + self.data["statements"].append( + ( + name, + signature, + domainlabel + " statement", + self.env.docname, + anchor, + 1, + ) + ) + + def clear_doc(self, docname): + """ + Sphinx API: like env-purge-doc event, but in a domain. + + Remove traces of a document in the domain-specific inventories. + """ + self.data["statements"] = list( + obj for obj in self.data["statements"] if obj[3] != docname + ) + + def merge_domaindata(self, docnames, otherdata): + """Sphinx API: Merge in data regarding *docnames* from a different + domaindata inventory (coming from a subprocess in parallel builds). + + @param otherdata is self.data equivalent from another process + + Beware: As of Sphinx 4.5.0, this is called multiple times in a row + with the same data and has to guard against duplicites. It seems + that all existing domains in Sphinx distribution have todo like + "deal with duplicates" but do nothing about them, so we just follow + the suite.""" + self.data["statements"] = list( + set(self.data["statements"] + otherdata["statements"]) + ) + + return ISCConfDomain + + +def setup(app, domainname, confname): + """ + Install new parametrized Sphinx domain. + """ + + Conf = domain_factory(domainname, confname) + app.add_domain(Conf) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/doc/arm/_ext/namedconf.py b/doc/arm/_ext/namedconf.py new file mode 100644 index 0000000000..40dc070a0d --- /dev/null +++ b/doc/arm/_ext/namedconf.py @@ -0,0 +1,22 @@ +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +""" +Sphinx domain "namedconf". See iscconf.py for details. + +""" +import iscconf + + +def setup(app): + return iscconf.setup(app, "namedconf", "named.conf") diff --git a/doc/arm/_ext/rndcconf.py b/doc/arm/_ext/rndcconf.py new file mode 100644 index 0000000000..2a7d2cdf42 --- /dev/null +++ b/doc/arm/_ext/rndcconf.py @@ -0,0 +1,22 @@ +############################################################################ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. +############################################################################ + +""" +Sphinx domain "rndcconf". See iscconf.py for details. +""" + +import iscconf + + +def setup(app): + return iscconf.setup(app, "rndcconf", "rndc.conf") diff --git a/doc/arm/conf.py b/doc/arm/conf.py index 84061baae2..578591e463 100644 --- a/doc/arm/conf.py +++ b/doc/arm/conf.py @@ -13,7 +13,9 @@ # flake8: noqa: E501 +from pathlib import Path import re +import sys from typing import List, Tuple @@ -99,12 +101,9 @@ def setup(app): # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. +# documentation root, make it absolute. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - +sys.path.append(str(Path(__file__).resolve().parent / "_ext")) # -- Project information ----------------------------------------------------- @@ -135,7 +134,7 @@ release = version # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ["namedconf", "rndcconf"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From b12606cebe700ce65cd245d2991a4c06eb5260ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= Date: Tue, 10 May 2022 11:09:23 +0200 Subject: [PATCH 2/7] Extend .. statement:: directive with optional values New and currently unused values can be provided using this syntax: .. statement:: max-cache-size :tags: resolver, cache :short: Short description The domain stores them in its internal structures for further use. --- doc/arm/_ext/iscconf.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/doc/arm/_ext/iscconf.py b/doc/arm/_ext/iscconf.py index 8608e03cea..c1ce46783a 100644 --- a/doc/arm/_ext/iscconf.py +++ b/doc/arm/_ext/iscconf.py @@ -53,7 +53,12 @@ def domain_factory(domainname, domainlabel): has_content = True required_arguments = 1 - option_spec = {} + # currently both options are unused + option_spec = { + "tags": directives.unchanged_required, + # one-sentece description for use in summary tables, in the future + "short": directives.unchanged_required, + } def handle_signature(self, sig, signode): signode += addnodes.desc_name(text=sig) @@ -61,9 +66,11 @@ def domain_factory(domainname, domainlabel): def add_target_and_index(self, _name_cls, sig, signode): signode["ids"].append(domainname + "-statement-" + sig) + tags = [x.strip() for x in self.options.get("tags", "").split(",")] + short = self.options.get("short") iscconf = self.env.get_domain(domainname) - iscconf.add_statement(sig) + iscconf.add_statement(sig, tags, short) name = domainname label = domainlabel @@ -75,6 +82,8 @@ def domain_factory(domainname, domainlabel): roles = {"ref": XRefRole(warn_dangling=True)} initial_data = { "statements": [], # object list for Sphinx API + # our own metadata: name -> {"tags": [list of tags], "short": "short desc"} + "statements_extra": {}, } indices = {} # no custom indicies @@ -113,7 +122,7 @@ def domain_factory(domainname, domainlabel): """ raise NotImplementedError - def add_statement(self, signature): + def add_statement(self, signature, tags, short): """ Add a new statement to the domain data structures. No visible effect. @@ -121,6 +130,7 @@ def domain_factory(domainname, domainlabel): name = "{}.{}.{}".format(domainname, "statement", signature) anchor = "{}-statement-{}".format(domainname, signature) + self.data["statements_extra"][name] = {"tags": tags, "short": short} # Sphinx API: name, dispname, type, docname, anchor, priority self.data["statements"].append( ( @@ -139,6 +149,11 @@ def domain_factory(domainname, domainlabel): Remove traces of a document in the domain-specific inventories. """ + # use name->doc mapping from Sphinx metadata + for name, _, _, cur_docname, _, _ in self.data["statements"]: + if cur_docname == docname: + if name in self.data["statements_extra"]: + del self.data["statements_extra"][name] self.data["statements"] = list( obj for obj in self.data["statements"] if obj[3] != docname ) @@ -157,6 +172,7 @@ def domain_factory(domainname, domainlabel): self.data["statements"] = list( set(self.data["statements"] + otherdata["statements"]) ) + self.data["statements_extra"].update(otherdata["statements_extra"]) return ISCConfDomain From 976aef030aa2ff6574c14340df43b39393bf920b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= Date: Tue, 10 May 2022 14:50:34 +0200 Subject: [PATCH 3/7] Add table generator into Sphinx config extension New directive .. statementlist:: generates table of statements in a the given domain (named.conf or rndc.conf). The table contains link to definition, short description, and also list of tags. Short description and tags have to be provided by user using optional parameters. E.g.: .. statement:: max-cache-size :tags: resolver, cache :short: Short description .. statementlist:: is currently not parametrized. This modification is based on Sphinx "tutorial" extension "TODO". The main trick is to use placeholder node for .. statementlist:: and replace it with table at later stage, when all source files were processed and all cross-references can be resolved. Beware, some details in Sphinx docs are not up-to-date, it's better to read Sphinx and docutil sources. --- doc/arm/_ext/iscconf.py | 135 ++++++++++++++++++++++++++++++++++++-- doc/arm/_ext/namedconf.py | 11 +++- doc/arm/_ext/rndcconf.py | 11 +++- 3 files changed, 149 insertions(+), 8 deletions(-) diff --git a/doc/arm/_ext/iscconf.py b/doc/arm/_ext/iscconf.py index c1ce46783a..d1ec5702cb 100644 --- a/doc/arm/_ext/iscconf.py +++ b/doc/arm/_ext/iscconf.py @@ -21,25 +21,43 @@ https://www.sphinx-doc.org/en/master/development/tutorials/todo.html https://www.sphinx-doc.org/en/master/development/tutorials/recipe.html """ +from collections import namedtuple + +from docutils.parsers.rst import directives +from docutils import nodes + from sphinx import addnodes from sphinx.directives import ObjectDescription from sphinx.domains import Domain from sphinx.roles import XRefRole from sphinx.util.nodes import make_refnode +from sphinx.util.docutils import SphinxDirective # pylint: disable=too-many-statements -def domain_factory(domainname, domainlabel): +def domain_factory(domainname, domainlabel, todolist): """ Return parametrized Sphinx domain object. @param domainname Name used when referencing domain in .rst: e.g. namedconf @param confname Humand-readable name for texts, e.g. named.conf + @param todolist A placeholder object which must be pickable. + See StatementListDirective. """ + class StatementListDirective(SphinxDirective): + """A custom directive to generate list of statements. + It only installs placeholder which is later replaced by + process_statementlist_nodes() callback. + """ + + def run(self): + return [todolist("")] + class ISCConfDomain(Domain): """ Custom Sphinx domain for ISC config. - Provides .. statement:: directive to define config statement. + Provides .. statement:: directive to define config statement and + .. statementlist:: to generate summary tables. :ref:`statementname` works as usual. See https://www.sphinx-doc.org/en/master/extdev/domainapi.html @@ -53,10 +71,9 @@ def domain_factory(domainname, domainlabel): has_content = True required_arguments = 1 - # currently both options are unused option_spec = { "tags": directives.unchanged_required, - # one-sentece description for use in summary tables, in the future + # one-sentece description for use in summary tables "short": directives.unchanged_required, } @@ -77,6 +94,7 @@ def domain_factory(domainname, domainlabel): directives = { "statement": StatementDirective, + "statementlist": StatementListDirective, } roles = {"ref": XRefRole(warn_dangling=True)} @@ -174,16 +192,121 @@ def domain_factory(domainname, domainlabel): ) self.data["statements_extra"].update(otherdata["statements_extra"]) + @classmethod + def process_statementlist_nodes(cls, app, doctree, fromdocname): + """ + Replace todolist objects (placed into document using + .. statementlist::) with automatically generated table + of statements. + """ + env = app.builder.env + iscconf = env.get_domain(cls.name) + + table_header = [ + TableColumn("ref", "Statement name"), + TableColumn("short", "Short desc"), + TableColumn("tags", "Tags"), + ] + table_b = DictToDocutilsTableBuilder(table_header) + table_b.append_iterable(iscconf.list_all(fromdocname)) + table = table_b.get_docutils() + for node in doctree.traverse(todolist): + node.replace_self(table) + + def list_all(self, fromdocname): + for statement in self.data["statements"]: + name, sig, _const, _doc, _anchor, _prio = statement + extra = self.data["statements_extra"][name] + short = extra["short"] + tags = ", ".join(extra["tags"]) + + refpara = nodes.inline() + refpara += self.resolve_xref( + self.env, + fromdocname, + self.env.app.builder, + None, + sig, + None, + nodes.Text(sig), + ) + + yield { + "fullname": name, + "ref": refpara, + "short": short, + "tags": tags, + } + return ISCConfDomain -def setup(app, domainname, confname): +# source dict key: human description +TableColumn = namedtuple("TableColumn", ["dictkey", "description"]) + + +class DictToDocutilsTableBuilder: + """generate docutils table""" + + def __init__(self, header): + """@param header: [ordered list of TableColumn]s""" + self.header = header + self.table = nodes.table() + self.table["classes"] += ["colwidths-auto"] + self.returned = False + # inner nodes of the table + self.tgroup = nodes.tgroup(cols=len(self.header)) + for _ in range(len(self.header)): + # ignored because of colwidths-auto, but must be present + colspec = nodes.colspec(colwidth=1) + self.tgroup.append(colspec) + self.table += self.tgroup + self._gen_header() + + self.tbody = nodes.tbody() + self.tgroup += self.tbody + + def _gen_header(self): + thead = nodes.thead() + + row = nodes.row() + for column in self.header: + entry = nodes.entry() + entry += nodes.Text(column.description) + row += entry + + thead.append(row) + self.tgroup += thead + + def append_iterable(self, objects): + """Append rows for each object (dict), ir order. + Extract column values from keys listed in self.header.""" + for obj in objects: + row = nodes.row() + for column in self.header: + entry = nodes.entry() + value = obj[column.dictkey] + if isinstance(value, str): + value = nodes.Text(value) + entry += value + row += entry + self.tbody.append(row) + + def get_docutils(self): + # guard against table reuse - that's most likely an error + assert not self.returned + self.returned = True + return self.table + + +def setup(app, domainname, confname, docutilsplaceholder): """ Install new parametrized Sphinx domain. """ - Conf = domain_factory(domainname, confname) + Conf = domain_factory(domainname, confname, docutilsplaceholder) app.add_domain(Conf) + app.connect("doctree-resolved", Conf.process_statementlist_nodes) return { "version": "0.1", diff --git a/doc/arm/_ext/namedconf.py b/doc/arm/_ext/namedconf.py index 40dc070a0d..2011d5a118 100644 --- a/doc/arm/_ext/namedconf.py +++ b/doc/arm/_ext/namedconf.py @@ -15,8 +15,17 @@ Sphinx domain "namedconf". See iscconf.py for details. """ +from docutils import nodes + import iscconf +class ToBeReplacedStatementList(nodes.General, nodes.Element): + """ + Placeholder, does nothing, but must be picklable + (= cannot be in generated class). + """ + + def setup(app): - return iscconf.setup(app, "namedconf", "named.conf") + return iscconf.setup(app, "namedconf", "named.conf", ToBeReplacedStatementList) diff --git a/doc/arm/_ext/rndcconf.py b/doc/arm/_ext/rndcconf.py index 2a7d2cdf42..bb9dbba065 100644 --- a/doc/arm/_ext/rndcconf.py +++ b/doc/arm/_ext/rndcconf.py @@ -15,8 +15,17 @@ Sphinx domain "rndcconf". See iscconf.py for details. """ +from docutils import nodes + import iscconf +class ToBeReplacedStatementList(nodes.General, nodes.Element): + """ + Placeholder, does nothing, but must be picklable + (= cannot be in a generated class). + """ + + def setup(app): - return iscconf.setup(app, "rndcconf", "rndc.conf") + return iscconf.setup(app, "rndcconf", "rndc.conf", ToBeReplacedStatementList) From 475f7a9603b76fe93962b4ef027f98b6d3eecc86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= Date: Tue, 10 May 2022 16:00:57 +0200 Subject: [PATCH 4/7] Render optional statement metadata in the ARM Optional values :short: and :tags: are now rendered right after the statement heading. --- doc/arm/_ext/iscconf.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/doc/arm/_ext/iscconf.py b/doc/arm/_ext/iscconf.py index d1ec5702cb..4a09ee023d 100644 --- a/doc/arm/_ext/iscconf.py +++ b/doc/arm/_ext/iscconf.py @@ -34,6 +34,17 @@ from sphinx.util.nodes import make_refnode from sphinx.util.docutils import SphinxDirective +def split_csv(argument, required): + argument = argument or "" + outlist = list(filter(len, (s.strip() for s in argument.split(",")))) + if required and not outlist: + raise ValueError( + "a non-empty list required; provide at least one value or remove" + " this option" + ) + return outlist + + # pylint: disable=too-many-statements def domain_factory(domainname, domainlabel, todolist): """ @@ -72,7 +83,7 @@ def domain_factory(domainname, domainlabel, todolist): has_content = True required_arguments = 1 option_spec = { - "tags": directives.unchanged_required, + "tags": lambda arg: split_csv(arg, required=False), # one-sentece description for use in summary tables "short": directives.unchanged_required, } @@ -83,11 +94,27 @@ def domain_factory(domainname, domainlabel, todolist): def add_target_and_index(self, _name_cls, sig, signode): signode["ids"].append(domainname + "-statement-" + sig) - tags = [x.strip() for x in self.options.get("tags", "").split(",")] - short = self.options.get("short") iscconf = self.env.get_domain(domainname) - iscconf.add_statement(sig, tags, short) + iscconf.add_statement(sig, self.isc_tags, self.isc_short) + + @property + def isc_tags(self): + return self.options.get("tags", []) + + @property + def isc_short(self): + return self.options.get("short", "") + + def transform_content(self, contentnode: addnodes.desc_content) -> None: + """autogenerate content from structured data""" + if self.isc_short: + contentnode.insert(0, nodes.paragraph(text=self.isc_short)) + if self.isc_tags: + tags = nodes.paragraph() + tags += nodes.strong(text="Tags: ") + tags += nodes.Text(", ".join(self.isc_tags)) + contentnode.insert(0, tags) name = domainname label = domainlabel From 2f2aa1d21cf5b0872eaaaedda96a476eaabb8f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= Date: Thu, 12 May 2022 09:20:46 +0200 Subject: [PATCH 5/7] Refactor and unite internal data structures for iscconf Sphinx extension It turns out it is easier to regenerate Sphinx-mandated structure in get_objects than to maintain two separate data structures. I should have realized that before. --- doc/arm/_ext/iscconf.py | 88 +++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/doc/arm/_ext/iscconf.py b/doc/arm/_ext/iscconf.py index 4a09ee023d..4b12c091e4 100644 --- a/doc/arm/_ext/iscconf.py +++ b/doc/arm/_ext/iscconf.py @@ -126,17 +126,29 @@ def domain_factory(domainname, domainlabel, todolist): roles = {"ref": XRefRole(warn_dangling=True)} initial_data = { - "statements": [], # object list for Sphinx API - # our own metadata: name -> {"tags": [list of tags], "short": "short desc"} - "statements_extra": {}, + # name -> {"tags": [list of tags], ...}; see add_statement() + "statements": {}, } indices = {} # no custom indicies def get_objects(self): - """Sphinx API: iterable of object descriptions""" - for obj in self.data["statements"]: - yield obj + """ + Sphinx API: + Iterable of Sphinx object descriptions (tuples defined in the API). + """ + for obj in self.data["statements"].values(): + yield tuple( + obj[key] + for key in [ + "fullname", + "signature", + "label", + "docname", + "anchor", + "priority", + ] + ) # pylint: disable=too-many-arguments def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): @@ -175,18 +187,17 @@ def domain_factory(domainname, domainlabel, todolist): name = "{}.{}.{}".format(domainname, "statement", signature) anchor = "{}-statement-{}".format(domainname, signature) - self.data["statements_extra"][name] = {"tags": tags, "short": short} - # Sphinx API: name, dispname, type, docname, anchor, priority - self.data["statements"].append( - ( - name, - signature, - domainlabel + " statement", - self.env.docname, - anchor, - 1, - ) - ) + self.data["statements"][name] = { + "tags": tags, + "short": short, + # Sphinx API + "fullname": name, # internal name + "signature": signature, # display name + "label": domainlabel + " statement", # description for index + "docname": self.env.docname, + "anchor": anchor, + "priority": 1, # search priority + } def clear_doc(self, docname): """ @@ -194,13 +205,12 @@ def domain_factory(domainname, domainlabel, todolist): Remove traces of a document in the domain-specific inventories. """ - # use name->doc mapping from Sphinx metadata - for name, _, _, cur_docname, _, _ in self.data["statements"]: - if cur_docname == docname: - if name in self.data["statements_extra"]: - del self.data["statements_extra"][name] - self.data["statements"] = list( - obj for obj in self.data["statements"] if obj[3] != docname + self.data["statements"] = dict( + { + key: obj + for key, obj in self.data["statements"].items() + if obj["docname"] != docname + } ) def merge_domaindata(self, docnames, otherdata): @@ -214,10 +224,7 @@ def domain_factory(domainname, domainlabel, todolist): that all existing domains in Sphinx distribution have todo like "deal with duplicates" but do nothing about them, so we just follow the suite.""" - self.data["statements"] = list( - set(self.data["statements"] + otherdata["statements"]) - ) - self.data["statements_extra"].update(otherdata["statements_extra"]) + self.data["statements"].update(otherdata["statements"]) @classmethod def process_statementlist_nodes(cls, app, doctree, fromdocname): @@ -232,7 +239,7 @@ def domain_factory(domainname, domainlabel, todolist): table_header = [ TableColumn("ref", "Statement name"), TableColumn("short", "Short desc"), - TableColumn("tags", "Tags"), + TableColumn("tags_txt", "Tags"), ] table_b = DictToDocutilsTableBuilder(table_header) table_b.append_iterable(iscconf.list_all(fromdocname)) @@ -241,11 +248,8 @@ def domain_factory(domainname, domainlabel, todolist): node.replace_self(table) def list_all(self, fromdocname): - for statement in self.data["statements"]: - name, sig, _const, _doc, _anchor, _prio = statement - extra = self.data["statements_extra"][name] - short = extra["short"] - tags = ", ".join(extra["tags"]) + for statement in self.data["statements"].values(): + tags_txt = ", ".join(statement["tags"]) refpara = nodes.inline() refpara += self.resolve_xref( @@ -253,17 +257,15 @@ def domain_factory(domainname, domainlabel, todolist): fromdocname, self.env.app.builder, None, - sig, + statement["signature"], None, - nodes.Text(sig), + nodes.Text(statement["signature"]), ) - yield { - "fullname": name, - "ref": refpara, - "short": short, - "tags": tags, - } + copy = statement.copy() + copy["ref"] = refpara + copy["tags_txt"] = tags_txt + yield copy return ISCConfDomain From ff577462f9b7154a70fc9449ee34ab4a2b7097bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= Date: Thu, 12 May 2022 09:51:41 +0200 Subject: [PATCH 6/7] Warn about duplicate .. statement:: definitions --- doc/arm/_ext/iscconf.py | 42 ++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/doc/arm/_ext/iscconf.py b/doc/arm/_ext/iscconf.py index 4b12c091e4..d73b1d12ea 100644 --- a/doc/arm/_ext/iscconf.py +++ b/doc/arm/_ext/iscconf.py @@ -30,8 +30,12 @@ from sphinx import addnodes from sphinx.directives import ObjectDescription from sphinx.domains import Domain from sphinx.roles import XRefRole -from sphinx.util.nodes import make_refnode +from sphinx.util import logging from sphinx.util.docutils import SphinxDirective +from sphinx.util.nodes import make_refnode + + +logger = logging.getLogger(__name__) def split_csv(argument, required): @@ -96,7 +100,7 @@ def domain_factory(domainname, domainlabel, todolist): signode["ids"].append(domainname + "-statement-" + sig) iscconf = self.env.get_domain(domainname) - iscconf.add_statement(sig, self.isc_tags, self.isc_short) + iscconf.add_statement(sig, self.isc_tags, self.isc_short, self.lineno) @property def isc_tags(self): @@ -179,7 +183,18 @@ def domain_factory(domainname, domainlabel, todolist): """ raise NotImplementedError - def add_statement(self, signature, tags, short): + @staticmethod + def log_statement_overlap(new, old): + assert new["fullname"] == old["fullname"] + logger.warning( + "duplicite detected! %s previously defined at %s:%d", + new["fullname"], + old["filename"], + old["lineno"], + location=(new["docname"], new["lineno"]), + ) + + def add_statement(self, signature, tags, short, lineno): """ Add a new statement to the domain data structures. No visible effect. @@ -187,9 +202,11 @@ def domain_factory(domainname, domainlabel, todolist): name = "{}.{}.{}".format(domainname, "statement", signature) anchor = "{}-statement-{}".format(domainname, signature) - self.data["statements"][name] = { + new = { "tags": tags, "short": short, + "filename": self.env.doc2path(self.env.docname), + "lineno": lineno, # Sphinx API "fullname": name, # internal name "signature": signature, # display name @@ -199,6 +216,10 @@ def domain_factory(domainname, domainlabel, todolist): "priority": 1, # search priority } + if name in self.data["statements"]: + self.log_statement_overlap(new, self.data["statements"][name]) + self.data["statements"][name] = new + def clear_doc(self, docname): """ Sphinx API: like env-purge-doc event, but in a domain. @@ -218,13 +239,12 @@ def domain_factory(domainname, domainlabel, todolist): domaindata inventory (coming from a subprocess in parallel builds). @param otherdata is self.data equivalent from another process - - Beware: As of Sphinx 4.5.0, this is called multiple times in a row - with the same data and has to guard against duplicites. It seems - that all existing domains in Sphinx distribution have todo like - "deal with duplicates" but do nothing about them, so we just follow - the suite.""" - self.data["statements"].update(otherdata["statements"]) + """ + old = self.data["statements"] + new = otherdata["statements"] + for name in set(old).intersection(set(new)): + self.log_statement_overlap(new[name], old[name]) + old.update(new) @classmethod def process_statementlist_nodes(cls, app, doctree, fromdocname): From 33931c97faaa0f728b4194b8077825e75c351e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pa=C4=8Dek?= Date: Fri, 13 May 2022 16:36:28 +0200 Subject: [PATCH 7/7] Add tag filter to .. statementlist:: RST directive Introduce a new syntax: .. namedconf:statementlist:: :filter_tags: acl, resolver The resulting table contains only items tagged as acl OR resolver. --- doc/arm/_ext/iscconf.py | 54 ++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/doc/arm/_ext/iscconf.py b/doc/arm/_ext/iscconf.py index d73b1d12ea..122673839d 100644 --- a/doc/arm/_ext/iscconf.py +++ b/doc/arm/_ext/iscconf.py @@ -65,8 +65,12 @@ def domain_factory(domainname, domainlabel, todolist): process_statementlist_nodes() callback. """ + option_spec = {"filter_tags": lambda arg: split_csv(arg, required=True)} + def run(self): - return [todolist("")] + placeholder = todolist("") + placeholder["isc_filter_tags"] = set(self.options.get("filter_tags", [])) + return [placeholder] class ISCConfDomain(Domain): """ @@ -104,7 +108,7 @@ def domain_factory(domainname, domainlabel, todolist): @property def isc_tags(self): - return self.options.get("tags", []) + return set(self.options.get("tags", [])) @property def isc_short(self): @@ -253,19 +257,47 @@ def domain_factory(domainname, domainlabel, todolist): .. statementlist::) with automatically generated table of statements. """ + + def gen_replacement_table(acceptable_tags): + table_header = [ + TableColumn("ref", "Statement"), + TableColumn("short", "Description"), + ] + table_b = DictToDocutilsTableBuilder(table_header) + table_b.append_iterable( + sorted( + iscconf.list_all(fromdocname), + key=lambda x: x["fullname"], + ) + ) + tag_header = [] + + if len(acceptable_tags) != 1: + # tags column only if tag filter is not applied + tag_header = [ + TableColumn("tags_txt", "Tags"), + ] + table_b = DictToDocutilsTableBuilder(table_header + tag_header) + table_b.append_iterable( + sorted( + filter( + lambda item: ( + not acceptable_tags + or item["tags"].intersection(acceptable_tags) + ), + iscconf.list_all(fromdocname), + ), + key=lambda x: x["fullname"], + ) + ) + return table_b.get_docutils() + env = app.builder.env iscconf = env.get_domain(cls.name) - table_header = [ - TableColumn("ref", "Statement name"), - TableColumn("short", "Short desc"), - TableColumn("tags_txt", "Tags"), - ] - table_b = DictToDocutilsTableBuilder(table_header) - table_b.append_iterable(iscconf.list_all(fromdocname)) - table = table_b.get_docutils() for node in doctree.traverse(todolist): - node.replace_self(table) + acceptable_tags = node["isc_filter_tags"] + node.replace_self(gen_replacement_table(acceptable_tags)) def list_all(self, fromdocname): for statement in self.data["statements"].values():