Sphinx Multiversioning

.github/workflows/build_and_publish_docs.yml

@@ -21,6 +21,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: fetch gh-pages branch
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+        run: |
+          git checkout gh-pages
+          git checkout ${{ env.BRANCH_NAME }}
 
       - name: set up python 3.9
         uses: actions/setup-python@v5
@@ -45,9 +54,6 @@ jobs:
         run: |
           python -m poetry run poe docs
 
-      - name: remove jekyll theming
-        run: touch docs/build/.nojekyll
-
       - name: save branch name without slashes
         if: ${{ github.ref != 'refs/heads/master' }}
         env:
@@ -58,17 +64,37 @@ jobs:
           echo BRANCH_NAME=${BRANCH_NAME} >> $GITHUB_ENV
 
       - name: save artifact
-        if: ${{ github.ref != 'refs/heads/master' }}
         uses: actions/upload-artifact@v4
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         with:
           name: ${{ format('github-pages-for-branch-{0}', env.BRANCH_NAME) }}
           path: docs/build/
           retention-days: 3
 
+      - name: save branch and checkout to gh-pages
+        run: |
+          git add -f .
+          git stash
+          git checkout gh-pages
+
+      - name: delete previous docs build for this branch
+        uses: EndBug/[email protected]
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+        with:
+          remove: '-r --ignore-unmatch ./${{ env.BRANCH_NAME }}'
+
+      - name: checkout to original branch and reapply changes
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+        run: |
+          git checkout ${{ env.BRANCH_NAME }}
+          git stash pop
+
       - name: deploy website
-        if: ${{ github.ref == 'refs/heads/master' }}
         uses: JamesIves/github-pages-deploy-action@v4
         with:
           branch: gh-pages
           folder: docs/build/
-          single-commit: True
+          clean: False

docs/source/_static/switcher.json

@@ -0,0 +1,12 @@
+[
+    {
+        "name": "0.8.0(stable)",
+        "version": "v0.8.0",
+        "url": "https://deeppavlov.github.io/dialog_flow_framework/master/index.html"
+    },
+    {
+        "name": "latest(dev)",
+        "version": "dev",
+        "url": "https://deeppavlov.github.io/dialog_flow_framework/dev/index.html"
+    }
+]

docs/source/builder.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+import enum
+import os
+import sys
+import shutil
+import importlib.util
+from logging import getLogger
+from pathlib import Path, PurePath
+from subprocess import CalledProcessError
+from typing import TYPE_CHECKING, Any, Iterable
+
+from sphinx_polyversion.builder import Builder, BuildError
+from sphinx_polyversion.environment import Environment
+from sphinx_polyversion.json import GLOBAL_ENCODER, JSONable
+
+if TYPE_CHECKING:
+    import json
+
+import scripts.doc
+from sphinx_polyversion.sphinx import CommandBuilder, Placeholder
+
+
+class DffSphinxBuilder(CommandBuilder):
+    def __init__(
+        self,
+        source: str | PurePath,
+        args: Iterable[str] = [],
+        encoder: json.JSONEncoder | None = None,
+        pre_cmd: Iterable[str | Placeholder] | None = None,
+        post_cmd: Iterable[str | Placeholder] | None = None,
+    ) -> None:
+        cmd: Iterable[str | Placeholder] = [
+            "sphinx-build",
+            "--color",
+            *args,
+            Placeholder.SOURCE_DIR,
+            Placeholder.OUTPUT_DIR,
+        ]
+        super().__init__(
+            source,
+            cmd,
+            encoder=encoder,
+            pre_cmd=pre_cmd,
+            post_cmd=post_cmd,
+        )
+        self.args = args
+
+    async def build(
+        self, environment: Environment, output_dir: Path, data: JSONable
+    ) -> None:
+        """
+        Build and render a documentation.
+
+        This method runs the command the instance was created with.
+        The metadata will be passed to the subprocess encoded as json
+        using the `POLYVERSION_DATA` environment variable.
+
+        Parameters
+        ----------
+        environment : Environment
+            The environment to use for building.
+        output_dir : Path
+            The output directory to build to.
+        data : JSONable
+            The metadata to use for building.
+        """
+        self.logger.info("Building...")
+        source_dir = str(environment.path.absolute() / self.source)
+
+        def replace(v: Any) -> str:
+            if v == Placeholder.OUTPUT_DIR:
+                return str(output_dir)
+            if v == Placeholder.SOURCE_DIR:
+                return source_dir
+            return str(v)
+
+        env = os.environ.copy()
+        env["POLYVERSION_DATA"] = self.encoder.encode(data)
+
+        cmd = tuple(map(replace, self.cmd))
+
+        # create output directory
+        output_dir.mkdir(exist_ok=True, parents=True)
+
+        # Importing version-dependent module setup.py
+        root_dir = environment.path.absolute()
+        os.system("ls" + str(source_dir))
+        spec = importlib.util.spec_from_file_location("setup", str(source_dir) + "/setup.py")
+        setup_module = importlib.util.module_from_spec(spec)
+        sys.modules["setup"] = setup_module
+        spec.loader.exec_module(setup_module)
+
+        # doing DFF funcs before doc building
+        scripts.doc.dff_funcs(str(root_dir))
+        setup_module.setup(str(root_dir), str(output_dir))
+
+        # Replacing old conf.py file with the newest one
+        # This shouldn't be there in builders for older versions.
+        newer_conf_path = (os.getcwd() + "/docs/source/conf.py")
+        older_conf_path = str(source_dir) + "/conf.py"
+        shutil.copyfile(newer_conf_path, older_conf_path)
+
+        # Removing Jekyll theming
+        open(str(output_dir) + '/.nojekyll', 'a')
+
+        # pre hook
+        if self.pre_cmd:
+            out, err, rc = await environment.run(*map(replace, self.pre_cmd), env=env)
+            if rc:
+                raise BuildError from CalledProcessError(rc, " ".join(cmd), out, err)
+
+        # build command
+        out, err, rc = await environment.run(*cmd, env=env)
+
+        self.logger.debug("Installation output:\n %s", out)
+        if rc:
+            raise BuildError from CalledProcessError(rc, " ".join(cmd), out, err)
+
+        # post hook
+        if self.post_cmd:
+            out, err, rc = await environment.run(*map(replace, self.post_cmd), env=env)
+            if rc:
+                raise BuildError from CalledProcessError(rc, " ".join(cmd), out, err)
+

docs/source/conf.py

@@ -7,12 +7,14 @@
 
 sys.path.append(os.path.abspath("."))
 from utils.notebook import py_percent_to_notebook  # noqa: E402
-from utils.generate_tutorials import generate_tutorial_links_for_notebook_creation  # noqa: E402
-from utils.link_misc_files import link_misc_files  # noqa: E402
-from utils.regenerate_apiref import regenerate_apiref  # noqa: E402
+from sphinx_polyversion import load
+from sphinx_polyversion.git import GitRef
 
 # -- Project information -----------------------------------------------------
 
+data = load(globals())  # adds variables `current` and `revisions`
+current: GitRef = data['current']
+
 _distribution_metadata = importlib.metadata.metadata('dff')
 
 project = _distribution_metadata["Name"]
@@ -57,7 +59,6 @@
 
 add_module_names = False
 
-
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -111,6 +112,17 @@
     "css/custom.css",
 ]
 
+# Version switcher url
+switcher_url = "https://zerglev.github.io/dialog_flow_framework/master/_static/switcher.json"
+# Could make a better url. Could it be stored in dev? Should it be passed to older versions somehow? This should be addressed before release.
+
+# Checking for dev before passing version to switcher
+if current[0] == "dev":
+    version_data = "dev"
+    # Need to use metadata to show the warning banner for unstable version.
+else:
+    version_data = version
+
 # Theme options
 html_theme_options = {
     "header_links_before_dropdown": 5,
@@ -139,9 +151,15 @@
         },
     ],
     "secondary_sidebar_items": ["page-toc", "source-links", "example-links"],
+    "switcher": {
+        "json_url": switcher_url,
+        "version_match" : version_data,
+    },
+    "navbar_persistent": ["search-button.html", "theme-switcher.html"],
+    "navbar_end": ["version-switcher.html", "navbar-icon-links.html"],
+    "show_version_warning_banner": True,
 }
 
-
 favicons = [
     {"href": "images/logo-dff.svg"},
 ]
@@ -154,48 +172,3 @@
     "member-order": "bysource",
     "exclude-members": "_abc_impl, model_fields",
 }
-
-
-def setup(_):
-    link_misc_files(
-        [
-            "utils/db_benchmark/benchmark_schema.json",
-            "utils/db_benchmark/benchmark_streamlit.py",
-        ]
-    )
-    generate_tutorial_links_for_notebook_creation(
-        [
-            ("tutorials.context_storages", "Context Storages"),
-            (
-                "tutorials.messengers",
-                "Interfaces",
-                [
-                    ("telegram", "Telegram"),
-                    ("web_api_interface", "Web API"),
-                ],
-            ),
-            ("tutorials.pipeline", "Pipeline"),
-            (
-                "tutorials.script",
-                "Script",
-                [
-                    ("core", "Core"),
-                    ("responses", "Responses"),
-                ],
-            ),
-            ("tutorials.utils", "Utils"),
-            ("tutorials.stats", "Stats"),
-        ]
-    )
-    regenerate_apiref(
-        [
-            ("dff.context_storages", "Context Storages"),
-            ("dff.messengers", "Messenger Interfaces"),
-            ("dff.pipeline", "Pipeline"),
-            ("dff.script", "Script"),
-            ("dff.stats", "Stats"),
-            ("dff.utils.testing", "Testing Utils"),
-            ("dff.utils.turn_caching", "Caching"),
-            ("dff.utils.db_benchmark", "DB Benchmark"),
-        ]
-    )

docs/source/poly.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+from datetime import datetime
+from functools import partial
+from sphinx_polyversion import *
+from sphinx_polyversion.git import *
+from sphinx_polyversion.git import closest_tag
+from sphinx_polyversion.pyvenv import Poetry
+from docs.source.builder import DffSphinxBuilder
+
+#: Regex matching the branches to build docs for
+# BRANCH_REGEX = r"((?!master).)*"
+# Put all branches here except master, so docs can be built for any branch
+# if the workflow is launched from it.
+BRANCH_REGEX = r".*"
+
+
+#: Regex matching the tags to build docs for
+TAG_REGEX = r"(v0.8.0)"
+# That was just 0.8.0, should change that to auto tags at some point.
+# Could just check always what version is being released, but I'm not sure how to do that yet.
+# For now this means someone has to change this on every single release to whatever the release tag will be. Potentially, if a mistake is made, this can be changed in dev and launched from there, just don't forget to fetch 'master' branch.
+
+#: Output dir relative to project root
+OUTPUT_DIR = "docs/build"
+
+#: Source directory
+SOURCE_DIR = "docs/source"
+
+#: Arguments to pass to `poetry install`
+POETRY_ARGS = "--with tutorials,docs --all-extras --no-ansi --no-interaction".split()
+
+#: Arguments to pass to `sphinx-build`
+SPHINX_ARGS = "-b html -W --keep-going -v".split()
+
+#: Mock data used for building local version
+MOCK_DATA = {
+    "revisions": [
+        GitRef("dev", "", "", GitRefType.BRANCH, datetime.fromtimestamp(0)),
+        GitRef("master", "", "", GitRefType.BRANCH, datetime.fromtimestamp(1)),
+    ],
+    "current": GitRef("local", "", "", GitRefType.BRANCH, datetime.fromtimestamp(2)),
+}
+MOCK = False
+
+# Load overrides read from commandline to global scope
+apply_overrides(globals())
+
+# Determine repository root directory
+root = Git.root(Path(__file__).parent)
+src = Path(SOURCE_DIR)
+
+# Setup driver and run it
+DefaultDriver(
+    root,
+    OUTPUT_DIR,
+    vcs=Git(
+        branch_regex=BRANCH_REGEX,
+        tag_regex=TAG_REGEX,
+        buffer_size=1 * 10**9,  # 1 GB
+        predicate=file_predicate([src]), # exclude refs without source dir
+    ),
+    builder=DffSphinxBuilder(src, args=SPHINX_ARGS),
+    env=Poetry.factory(args=POETRY_ARGS),
+    selector=partial(closest_tag, root),
+    template_dir=root / src / "templates",
+    static_dir=root / src / "static",
+    mock=MOCK_DATA,
+).run(MOCK)