Generate documentation for platform flows

.github/workflows/Pipeline.yml

@@ -79,6 +79,7 @@ jobs:
       - name: 📓 Build the documentation
         run: |
           . ./docs/env/conda/bin/activate f4pga-docs
+          make -C docs gen_flow_rst
           make -C docs html
 
       - name: '📤 Upload artifact: Sphinx HTML'

docs/Makefile

@@ -49,5 +49,9 @@ latexpdf:
 	make -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
+gen_flow_rst:
+	mkdir -p f4pga/generated/flows
+	python3 gen_flow_docs.py f4pga/generated/flows
+
 %:
 	$(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) $(BUILDDIR)/$@

docs/f4pga/Usage.md → docs/f4pga/flows/index.md

@@ -362,6 +362,20 @@ In the example above file `counter.v` has been modified and is now marked as "**
 This causes a bunch of other dependencies to be rebuilt ("**R**").
 `build_dir` and `xdc` were already present, so they are marked as "**O**".
 
+# Flow documentation
+
+Here you can find generated documentation for various platforms:
+
+```{toctree}
+:maxdepth: 1
+
+../generated/flows/xc7a50t
+../generated/flows/xc7a100t
+../generated/flows/xc7a200t
+../generated/flows/eos-s3
+../generated/flows/ice40
+```
+
 ## Common targets and values
 
 Targets and values are named with some conventions.

docs/flow.rst.jinja2

@@ -0,0 +1,53 @@
+{{ platform_name }} flow
+{% for c in platform_name -%}#{% endfor %}#####
+
+.. NOTE::
+  This document has been generated. It's purpose is to provide an overview over usage of the flow
+  for a group of devices which are considered to be one "platform". See ``f4pga/platforms.yml``
+  to see the current platforms.
+  You can query information about individual stages using ``--stageinfo`` option for ``build``
+  subcommand. 
+
+Available chips
+===============
+
+{% for chip in sup_chips -%}
+* {{ chip }}
+{% endfor %}
+
+Required inputs
+===============
+
+The following dependencies have to be provided by the user:
+
+.. table::
+
+  +-{% for i in range(inputs_len_0) -%}{{ "-" }}{% endfor %}-+-{% for i in range(inputs_len_1) -%}{{ "-" }}{% endfor %}-+
+  | Name{% for i in range(inputs_len_0 - 4) -%}{{ " " }}{% endfor %} | Is required to execute entire flow?{% for i in range(inputs_len_1 - 35) -%}{{ " " }}{% endfor %} |
+  +={% for i in range(inputs_len_0) -%}{{ "=" }}{% endfor %}=+={% for i in range(inputs_len_1) -%}{{ "=" }}{% endfor %}=+
+{% for user_dep_name, usr_dep_req in user_deps.items() -%}
+  {{ "  " }}| {{ user_dep_name }}{% for i in range(inputs_len_0 - user_dep_name|length) -%}{{ " " }}{% endfor %} | {{ usr_dep_req }}{% for i in range(inputs_len_1 - usr_dep_req|length) -%}{{ " " }}{% endfor %} |
+  +-{% for i in range(inputs_len_0) -%}{{ "-" }}{% endfor %}-+-{% for i in range(inputs_len_1) -%}{{ "-" }}{% endfor %}-+
+{% endfor %}
+{{ "  " }}
+
+Stages within this flow
+=======================
+
+{% for stage in stages -%}
+* {{ stage }}
+{% endfor %}
+
+Targets
+=======
+
+.. table::
+
+  +-{% for i in range(targets_len_0) -%}{{ "-" }}{% endfor %}-+------------+-{% for i in range(targets_len_2) -%}{{ "-" }}{% endfor %}-+
+  | Name{% for i in range(targets_len_0 - 4) -%}{{ " " }}{% endfor %} | On-demand? | Description{% for i in range(targets_len_2 - 11) -%}{{ " " }}{% endfor %} |
+  +={% for i in range(targets_len_0) -%}{{ "=" }}{% endfor %}=+============+={% for i in range(targets_len_2) -%}{{ "=" }}{% endfor %}=+
+{% for target_name, target_spec in targets.items() -%}
+  {{ "  " }}| {{ target_name }}{% for i in range(targets_len_0 - target_name|length) -%}{{ " " }}{% endfor %} | {% if target_spec.qualifier == "demand" -%}yes{% else %}no {% endif %}        | {{ target_spec.meta }}{% for i in range(targets_len_2 - target_spec.meta|length) -%}{{ " " }}{% endfor %} |
+  +-{% for i in range(targets_len_0) -%}{{ "-" }}{% endfor %}-+------------+-{% for i in range(targets_len_2) -%}{{ "-" }}{% endfor %}-+
+{% endfor %}
+{{ "  " }}

docs/gen_flow_docs.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 F4PGA Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# XXX: Hack to make f4pga think that we are in an environment set-up for eos-s3.
+# eos-s3 has been chosen because it does not require prjxray-config which is used when
+# configuring variables available for xc7.
+import os
+os.environ["FPGA_FAM"] = "eos-s3"
+
+import yaml
+from f4pga.flows.commands import setup_resolution_env
+import jinja2
+from f4pga.flows.common import (
+    scan_modules as f4pga_scan_modules,
+    ROOT as f4pga_ROOT,
+    F4PGAException,
+)
+from f4pga.flows.flow_config import FlowDefinition as f4pga_FlowDefinition
+from f4pga.flows.commands import setup_resolution_env as f4pga_setup_resolution_env
+from f4pga.flows.inspector import get_stages_info_dict as f4pga_get_stages_info_dict
+from argparse import ArgumentParser
+from pathlib import Path
+import sys
+
+class FailedFlowGenException(Exception):
+    platform: str
+
+    def __init__(self, platform: str):
+        self.platform = platform
+
+class Inspector:
+    part_db: dict
+    platforms: dict
+
+    def __init__(self):
+        with open(f4pga_ROOT / "part_db.yml", "r") as f:
+            self.part_db = yaml.safe_load(f.read())
+        with (f4pga_ROOT / "platforms.yml").open("r") as f:
+            self.platforms = yaml.safe_load(f.read())
+
+    def get_platforms(self):
+        return self.part_db.keys()
+
+    def get_platform_flow_info(self, platform: str):
+        platform_def = self.platforms.get(platform)
+        if platform_def is None:
+            raise F4PGAException(
+                message=f"Flow definition for platform <{platform}> cannot be found!"
+            )
+
+        r_env = f4pga_setup_resolution_env()
+
+        # XXX: We don't care for the exact part name, but it might be required to initilize
+        # stages. There's an assumption being made here, that it won't affect I/O.
+        part_name = self.part_db[platform][0]
+        r_env.add_values({"part_name": part_name.lower()})
+
+        return f4pga_get_stages_info_dict(
+            f4pga_FlowDefinition(self.platforms[platform], r_env)
+        )
+
+class FlowDocGenerator:
+    inspector: Inspector
+    template: jinja2.Template
+
+    def __init__(self, inspector: Inspector, template_path: str = "flow.rst.jinja2"):
+        self.inspector = inspector
+
+        with open(template_path, "r") as f:
+            template_src = f.read()
+
+        self.template = jinja2.Template(template_src)
+
+    def get_user_deps(self, io):
+        all_takes = set()
+        all_produces = set()
+        for stage_io in io.values():
+            all_takes.update(stage_io["takes"].keys())
+            all_produces.update(stage_io["produces"].keys())
+
+        non_producible_takes = all_takes.difference(all_produces)
+
+        user_deps = dict((take, "n/a") for take in non_producible_takes)
+        user_dep_r_users = dict((take, []) for take in non_producible_takes)
+
+        for stage, stage_io in io.items():
+            for take, take_spec in stage_io["takes"].items():
+                if take not in user_deps.keys():
+                    continue
+
+                q = take_spec["qualifier"]
+                if q == "req":
+                    if user_deps[take] == "n/a":
+                        user_deps[take] = "yes"
+                    elif user_deps[take] == "yes":
+                        user_deps[take] = "yes"
+                    elif user_deps[take] == "no":
+                        user_deps[take] = f"required by {stage}"
+                    elif "required by" in user_deps[take]:
+                        user_deps[take] += f", {stage}"
+                    user_dep_r_users[take].append(stage)
+                if q == "maybe":
+                    if user_deps[take] == "yes":
+                        user_deps[take] = f"required by {','.join(user_dep_r_users[take])}"
+                    if "required by" not in user_deps[take]:
+                        user_deps[take] = "no"
+
+        return user_deps
+
+    def get_targets(self, io):
+        all_produces = {}
+        for stage_io in io.values():
+            all_produces.update(stage_io["produces"].items())
+
+        for target_spec in all_produces.values():
+            if target_spec.get("meta") is None:
+                target_spec["meta"] = "*(No description)*"
+            else:
+                target_spec["meta"] = target_spec["meta"].replace("\n", " ") 
+
+        return all_produces
+
+    def generate_doc_for_platform(self, platform_name: str) -> str:
+        try:
+            io = self.inspector.get_platform_flow_info(platform_name)
+        except:
+            raise FailedFlowGenException(platform_name)
+
+        stages = list(io.keys())
+        sup_chips = self.inspector.part_db[platform_name]
+
+        targets = self.get_targets(io)
+        user_deps = self.get_user_deps(io)
+
+        return self.template.render(
+            platform_name=platform_name,
+            stages=stages,
+            sup_chips=sup_chips,
+            user_deps=user_deps,
+            targets=targets,
+            # XXX: Handling dumb ASCII-art requirements for tables.
+            inputs_len_0=max([len(name) for name in user_deps.keys()] + [len("Name")]),
+            inputs_len_1=max([len(v) for v in user_deps.values()] + [len("Is required to execute entire flow?")]),
+            targets_len_0=max([len(name) for name in targets.keys()] + [len("Name")]),
+            targets_len_2=max([len(v["meta"]) for v in targets.values()] + [len("Description")])
+        )
+
+
+def main():
+    argparser = ArgumentParser()
+    argparser.add_argument("output_dir")
+    args = argparser.parse_args()
+
+    output_dir = Path(args.output_dir)
+
+    f4pga_scan_modules()
+
+    inspector = Inspector()
+    doc_gen = FlowDocGenerator(inspector)
+    for platform_name in inspector.get_platforms():
+        try:
+            doc = doc_gen.generate_doc_for_platform(platform_name)
+        except FailedFlowGenException as e:
+            print(
+                f"WARNING: Failed to create a flow for platform <{e.platform}>",
+                file=sys.stderr
+            )
+            continue
+        with open(output_dir / f"{platform_name}.rst", "w") as f:
+            f.write(doc)
+
+
+if __name__ == "__main__":
+    main()

docs/index.rst

@@ -63,7 +63,7 @@ Table of Contents
   :maxdepth: 2
 
   f4pga/index
-  f4pga/Usage
+  f4pga/flows/index
   f4pga/modules/index
   f4pga/DevNotes
   f4pga/Deprecated

docs/requirements.txt

@@ -5,3 +5,5 @@ sphinxcontrib-bibtex
 https://github.com/f4pga/sphinx_f4pga_theme/archive/f4pga.zip#sphinx-f4pga-theme
 https://github.com/SymbiFlow/sphinx-verilog-domain/archive/master.zip#sphinx-verilog-domain
 tabulate
+jinja2
+../f4pga

f4pga/flows/commands.py

@@ -195,7 +195,7 @@ def make_flow_config(project_flow_cfg: ProjectFlowConfig, part_name: str) -> Flo
     r_env = setup_resolution_env()
     r_env.add_values({"part_name": part_name.lower()})
 
-    scan_modules(str(ROOT))
+    scan_modules()
 
     with (ROOT / "platforms.yml").open("r") as rfptr:
         platforms = yaml_load(rfptr, yaml_loader)

f4pga/flows/common.py

@@ -27,6 +27,7 @@
 
 from f4pga.context import FPGA_FAM, F4PGA_SHARE_DIR
 
+ROOT = Path(__file__).resolve().parent
 
 bin_dir_path = str(Path(sys_argv[0]).resolve().parent.parent)
 share_dir_path = str(F4PGA_SHARE_DIR)
@@ -67,12 +68,11 @@ def with_qualifier(name: str, q: str) -> str:
 _sfbuild_module_collection_name_to_path = {}
 
 
-def scan_modules(mypath: str):
+def scan_modules():
     global _sfbuild_module_collection_name_to_path
-    sfbuild_home = mypath
     _sfbuild_module_collection_name_to_path = {
-        re_match("(.*)_modules$", moddir).groups()[0]: str(Path(sfbuild_home) / moddir)
-        for moddir in [dir for dir in os_listdir(sfbuild_home) if re_match(".*_modules$", dir)]
+        re_match("(.*)_modules$", moddir).groups()[0]: str(ROOT / moddir)
+        for moddir in [dir for dir in os_listdir(str(ROOT)) if re_match(".*_modules$", dir)]
     }
 
 

f4pga/flows/inspector.py

@@ -18,10 +18,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from colorama import Style
+from f4pga.flows.flow_config import FlowDefinition
+from pathlib import Path
 
 from f4pga.flows.module import Module
 from f4pga.flows.common import decompose_depname
 
+ROOT = Path(__file__).resolve().parent
+
 
 def _get_if_qualifier(deplist: "list[str]", qualifier: str):
     for dep_name in deplist:
@@ -59,3 +63,34 @@ def get_module_info(module: Module) -> str:
     r += _list_if_qualifier(module.produces, "maybe", indent=4)
 
     return r
+
+
+def _make_io_dict(io: str, metas: "None | dict[str]" = None) -> "dict[str, str]":
+    name, q = decompose_depname(io)
+    d = {"qualifier": q}
+    if metas is not None:
+        meta = metas.get(name)
+        if meta is not None:
+            d["meta"] = meta
+    return name, d
+
+
+def _make_io_dict_dict(ios: "list[str]", metas: "None | dict[str]" = None) -> "dict[str, str]":
+    return dict(_make_io_dict(io, metas) for io in ios)
+
+
+def _get_module_info_dict(module: Module) -> "dict[str, dict[str, str]]":
+    return {
+        "takes": _make_io_dict_dict(module.takes),
+        "produces": _make_io_dict_dict(
+            module.produces, metas=module.prod_meta if hasattr(module, "prod_meta") else None
+        ),
+        "values": _make_io_dict_dict(module.produces),
+    }
+
+
+def get_stages_info_dict(flow_definition: FlowDefinition) -> "dict[str, dict[str, dict[str, str]]]":
+    d = {}
+    for stage_name, stage in flow_definition.stages.items():
+        d[stage_name] = _get_module_info_dict(stage.module)
+    return d