[manuf] Make orchestrator resolve paths using runfiles library

This simplifies use of the packaged orchestrator script. Instead of
having to extract the packaged runfiles manually, the rules_python
machinery extracts it into a temp dir and the library handles path
resolution.

Some hackery was required to preserve the scheme used previously for
referring to the main repo vs external repos.

Signed-off-by: Noah Moroze <[email protected]>

sw/host/provisioning/orchestrator/README.md

@@ -61,34 +61,14 @@ cp ${REPO_TOP}/bazel-bin/sw/host/provisioning/orchestrator/src/orchestrator.zip
 
 export ORCHESTRATOR_ZIP="${ORCHESTRATOR_RUN_DIR}/orchestrator.zip"
 
-# Extract runfile folders from orchestrator package.
-unzip ${ORCHESTRATOR_ZIP} \
-  "runfiles/lowrisc_opentitan/*" \
-  "runfiles/openocd/*" \
-  "runfiles/provisioning_exts/*"
-
-# All external dependencies are mapped under
-# runfiles/lowrisc_opentitan/external.
-mkdir -p  runfiles/lowrisc_opentitan/external
-
-ln -fs $(pwd)/runfiles/openocd runfiles/lowrisc_opentitan/external
-
-# The following is needed if you are using the provisioning extensions
-# infrastructure.
-PROVISIONING_EXT_RUNFILES=$(pwd)/runfiles/provisioning_exts
-[ -d "${PROVISION_EXT_RUNFILES}" ] &&   \
-  ln -fs "${PROVISIONING_EXT_RUNFILES}" \
-    runfiles/lowrisc_opentitan/external/provisioning_exts
-
 # Run tool. The path to the --sku-config parameter is relative to the
-# runfiles-dir.
+# workspace root.
 export FPGA_TARGET=hyper310
 python3 ${ORCHESTRATOR_ZIP} \
   --sku-config=sw/host/provisioning/orchestrator/configs/skus/emulation.hjson \
   --test-unlock-token="0x11111111_11111111_11111111_11111111" \
   --test-exit-token="0x22222222_22222222_22222222_22222222" \
   --fpga=${FPGA_TARGET} \
   --non-interactive \
-  --runfiles-dir=$(pwd)/runfiles/lowrisc_opentitan \
-  --db-path=$(pwd)/provisioning.sqlite
+  --db-path=provisioning.sqlite
 ```

sw/host/provisioning/orchestrator/src/BUILD

@@ -54,7 +54,10 @@ py_library(
     name = "util",
     srcs = ["util.py"],
     imports = ["."],
-    deps = [requirement("hjson")],
+    deps = [
+        requirement("hjson"),
+        "@rules_python//python/runfiles",
+    ],
 )
 
 py_library(

sw/host/provisioning/orchestrator/src/ca_config.py

@@ -6,6 +6,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 
+from util import must_resolve_runfile
+
 
 @dataclass
 class CaConfig:
@@ -18,9 +20,9 @@ class CaConfig:
 
     def __post_init__(self):
         # Update certificate and key members to Path objs if necessary.
-        self.certificate = Path(self.certificate)
+        self.certificate = Path(must_resolve_runfile(self.certificate))
         if self.key_type == "Raw":
-            self.key = Path(self.key)
+            self.key = Path(must_resolve_runfile(self.key))
         self.validate()
 
     def validate(self) -> None:

sw/host/provisioning/orchestrator/src/orchestrator.py

@@ -5,7 +5,6 @@
 
 import argparse
 import logging
-import os
 import shlex
 import subprocess
 import sys
@@ -17,7 +16,7 @@
 from device_id import DeviceId, DeviceIdentificationNumber
 from ot_dut import OtDut
 from sku_config import SkuConfig
-from util import confirm, parse_hexstring_to_int
+from util import confirm, parse_hexstring_to_int, must_resolve_runfile
 
 
 def get_user_confirmation(
@@ -104,11 +103,6 @@ def main(args_in):
         default=False,
         help="Skip all non-required user confirmations.",
     )
-    parser.add_argument(
-        "--runfiles-dir",
-        type=str,
-        help="Runfiles directory to use for provisioning.",
-    )
     parser.add_argument(
         "--log-dir",
         default="logs",
@@ -130,13 +124,10 @@ def main(args_in):
     if not args.cp_only and args.db_path is None:
         parser.error("--db-path is required when --cp-only is not provided")
 
-    # All relative paths are relative to the runfiles directory.
-    if args.runfiles_dir:
-        os.chdir(args.runfiles_dir)
-
     # Load and validate a SKU configuration file.
+    sku_config_path = must_resolve_runfile(args.sku_config)
     sku_config_args = {}
-    with open(args.sku_config, "r") as fp:
+    with open(sku_config_path, "r") as fp:
         sku_config_args = hjson.load(fp)
     sku_config = SkuConfig(**sku_config_args)
 

sw/host/provisioning/orchestrator/src/ot_dut.py

@@ -14,7 +14,7 @@
 
 from device_id import DeviceId
 from sku_config import SkuConfig
-from util import confirm, format_hex, run
+from util import confirm, format_hex, run, must_resolve_runfile
 
 # FPGA bitstream.
 _FPGA_UNIVERSAL_SPLICE_BITSTREAM = "hw/bitstream/universal/splice.bit"
@@ -100,27 +100,33 @@ def run_cp(self) -> None:
         host_flags = _BASE_PROVISIONING_FLAGS
         device_elf = _CP_DEVICE_ELF
         print(f"device_elf: {device_elf}")
+
+        openocd_bin = must_resolve_runfile(_OPENOCD_BIN)
+        openocd_cfg = must_resolve_runfile(_OPENOCD_ADAPTER_CONFIG)
         if self.fpga:
             # Set host flags and device binary for FPGA DUT.
             host_flags = host_flags.format(target=self.fpga,
-                                           openocd_bin=_OPENOCD_BIN,
-                                           openocd_cfg=_OPENOCD_ADAPTER_CONFIG)
+                                           openocd_bin=openocd_bin,
+                                           openocd_cfg=openocd_cfg)
             host_flags += " --clear-bitstream"
-            host_flags += f" --bitstream={_FPGA_UNIVERSAL_SPLICE_BITSTREAM}"
+            bitstream = must_resolve_runfile(_FPGA_UNIVERSAL_SPLICE_BITSTREAM)
+            host_flags += f" --bitstream={bitstream}"
             device_elf = device_elf.format(
                 base_dir=self._base_dev_dir(),
                 target=f"fpga_{self.fpga}_rom_with_fake_keys")
         else:
             # Set host flags and device binary for Silicon DUT.
             host_flags = host_flags.format(target="teacup",
-                                           openocd_bin=_OPENOCD_BIN,
-                                           openocd_cfg=_OPENOCD_ADAPTER_CONFIG)
+                                           openocd_bin=openocd_bin,
+                                           openocd_cfg=openocd_cfg)
             host_flags += " --disable-dft-on-reset"
             device_elf = device_elf.format(base_dir=self._base_dev_dir(),
                                            target="silicon_creator")
+        device_elf = must_resolve_runfile(device_elf)
 
         # Assemble CP command.
-        cmd = f"""{_CP_HOST_BIN} \
+        cp_host_bin = must_resolve_runfile(_CP_HOST_BIN)
+        cmd = f"""{cp_host_bin} \
         --rcfile= \
         --logging=info \
         {host_flags} \
@@ -171,6 +177,9 @@ def run_ft(self) -> None:
 
         # Set cmd args and device binaries.
         host_bin = _FT_HOST_BIN.format(sku=self.sku_config.name)
+        host_bin = must_resolve_runfile(host_bin)
+        openocd_bin = must_resolve_runfile(_OPENOCD_BIN)
+        openocd_cfg = must_resolve_runfile(_OPENOCD_ADAPTER_CONFIG)
         host_flags = _BASE_PROVISIONING_FLAGS
         individ_elf = _FT_INDIVID_DEVICE_ELF
         # Emulation perso bins are signed online with fake keys, and therefore
@@ -184,8 +193,8 @@ def run_ft(self) -> None:
             # No need to load another bitstream, we will take over where CP
             # stage above left off.
             host_flags = host_flags.format(target=self.fpga,
-                                           openocd_bin=_OPENOCD_BIN,
-                                           openocd_cfg=_OPENOCD_ADAPTER_CONFIG)
+                                           openocd_bin=openocd_bin,
+                                           openocd_cfg=openocd_cfg)
             individ_elf = individ_elf.format(
                 base_dir=self._base_dev_dir(),
                 sku=self.sku_config.name,
@@ -201,8 +210,8 @@ def run_ft(self) -> None:
         else:
             # Set host flags and device binaries for Silicon DUT.
             host_flags = host_flags.format(target="teacup",
-                                           openocd_bin=_OPENOCD_BIN,
-                                           openocd_cfg=_OPENOCD_ADAPTER_CONFIG)
+                                           openocd_bin=openocd_bin,
+                                           openocd_cfg=openocd_cfg)
             host_flags += " --disable-dft-on-reset"
             individ_elf = individ_elf.format(base_dir=self._base_dev_dir(),
                                              sku=self.sku_config.name,
@@ -214,6 +223,10 @@ def run_ft(self) -> None:
                                                  sku=self.sku_config.name,
                                                  target="silicon_creator")
 
+        individ_elf = must_resolve_runfile(individ_elf)
+        perso_bin = must_resolve_runfile(perso_bin)
+        fw_bundle_bin = must_resolve_runfile(fw_bundle_bin)
+
         # Write CA configs to a JSON tmpfile.
         ca_config_dict = {
             "dice": self.sku_config.dice_ca.to_dict_entry(),

sw/host/provisioning/orchestrator/src/sku_config.py

@@ -10,6 +10,7 @@
 import hjson
 
 from ca_config import CaConfig
+from util import must_resolve_runfile
 
 _PRODUCT_IDS_HJSON = "sw/host/provisioning/orchestrator/data/products.hjson"
 _PACKAGE_IDS_HJSON = "sw/host/provisioning/orchestrator/data/packages/earlgrey_a1.hjson"
@@ -35,13 +36,15 @@ def __post_init__(self):
             self.ext_ca = CaConfig(name="ext_ca", **self.ext_ca)
 
         # Load product IDs database.
+        product_ids_hjson = must_resolve_runfile(_PRODUCT_IDS_HJSON)
         self._product_ids = None
-        with open(_PRODUCT_IDS_HJSON, "r") as fp:
+        with open(product_ids_hjson, "r") as fp:
             self._product_ids = hjson.load(fp)
 
         # Load package IDs database.
+        package_ids_hjson = must_resolve_runfile(_PACKAGE_IDS_HJSON)
         self._package_ids = None
-        with open(_PACKAGE_IDS_HJSON, "r") as fp:
+        with open(package_ids_hjson, "r") as fp:
             self._package_ids = hjson.load(fp)
 
         # Validate inputs.
@@ -56,18 +59,24 @@ def __post_init__(self):
         if self.package in self._package_ids:
             self.package_id = int(self._package_ids[self.package], 16)
 
+        if self.token_encrypt_key:
+            self.token_encrypt_key = must_resolve_runfile(
+                self.token_encrypt_key)
+
     @staticmethod
     def from_ids(product_id: int, si_creator_id: int,
                  package_id: int) -> "SkuConfig":
         """Creates a SKU configuration object from product, SiliconCreator, and package IDs."""
         # Load product IDs database.
+        product_ids_hjson = must_resolve_runfile(_PRODUCT_IDS_HJSON)
         product_ids = None
-        with open(_PRODUCT_IDS_HJSON, "r") as fp:
+        with open(product_ids_hjson, "r") as fp:
             product_ids = hjson.load(fp)
 
         # Load package IDs database.
+        package_ids_hjson = must_resolve_runfile(_PACKAGE_IDS_HJSON)
         package_ids = None
-        with open(_PACKAGE_IDS_HJSON, "r") as fp:
+        with open(package_ids_hjson, "r") as fp:
             package_ids = hjson.load(fp)
 
         # Create SKU configuration object.

sw/host/provisioning/orchestrator/src/util.py

@@ -3,9 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import os
 import shlex
 import subprocess
 
+from python.runfiles import Runfiles
+
 
 def parse_hexstring_to_int(x):
     """Accepts hexstrings with and without the 0x."""
@@ -64,3 +67,36 @@ def run(cmd, stdout_logfile, stderr_logfile):
     out_tee.stdin.close()
     err_tee.stdin.close()
     return res
+
+
+_runfiles = Runfiles.Create()
+
+
+def must_resolve_runfile(path):
+    """Resolves path to runfile.
+
+    Relative paths specified as external/<repo>/... will be resolved relative to
+    the external repository @<repo>. Otherwise, relative paths will be resolved
+    relative to the main workspace. Absolute paths are returned as-is.
+
+    Raises a ValueError if the path does not exist on the filesystem.
+    """
+
+    # orchestrator.py assumes the "old" style of runfiles tree, where paths to
+    # files within the main workspace do not include the repo name and external
+    # deps prepend external/.
+    #
+    # The old scheme does not work within a zipped py_binary, so this logic is a
+    # hack to fix up the supplied path.
+    #
+    # See https://docs.google.com/document/d/1skNx5o-8k5-YXUAyEETvr39eKoh9fecJbGUquPh5iy8/edit.
+    REPO = "lowrisc_opentitan"
+    if path.startswith("external/"):
+        corrected_path = path[len("external/"):]
+    else:
+        corrected_path = os.path.join(REPO, path)
+
+    resolved = _runfiles.Rlocation(corrected_path)
+    if resolved is None or not os.path.exists(resolved):
+        raise ValueError(f"Could not find runfile: {path}")
+    return resolved

sw/host/provisioning/orchestrator/tests/BUILD

@@ -21,6 +21,7 @@ py_test(
     data = [
         "//sw/device/silicon_creator/manuf/keys/fake:dice_ca.pem",
         "//sw/device/silicon_creator/manuf/keys/fake:ext_ca.pem",
+        "//sw/device/silicon_creator/manuf/keys/fake:rma_unlock_enc_rsa3072.pub.der",
         "//sw/device/silicon_creator/manuf/keys/fake:sk.pkcs8.der",
         "//sw/host/provisioning/orchestrator/configs/skus:emulation.hjson",
     ],
@@ -31,3 +32,15 @@ py_test(
         "//sw/host/provisioning/orchestrator/src:util",
     ],
 )
+
+py_test(
+    name = "util_test",
+    srcs = ["util_test.py"],
+    data = [
+        "//sw/device/silicon_creator/manuf/keys/fake:dice_ca.pem",
+        "//third_party/openocd:jtag_cmsis_dap_adapter_cfg",
+    ],
+    deps = [
+        "//sw/host/provisioning/orchestrator/src:util",
+    ],
+)

sw/host/provisioning/orchestrator/tests/util_test.py

@@ -0,0 +1,34 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import unittest
+
+from util import must_resolve_runfile
+
+
+class TestRunfileMustResolve(unittest.TestCase):
+
+    def test_main_workspace_path(self):
+        resolved = must_resolve_runfile(
+            "sw/device/silicon_creator/manuf/keys/fake/dice_ca.pem")
+        self.assertTrue(os.path.exists(resolved))
+
+    def test_external_path(self):
+        resolved = must_resolve_runfile(
+            "external/openocd/tcl/interface/cmsis-dap.cfg")
+        self.assertTrue(os.path.exists(resolved))
+
+    def test_abs_path(self):
+        path = os.path.abspath(__file__)
+        resolved = must_resolve_runfile(path)
+        self.assertEqual(path, resolved)
+
+    def test_non_existent(self):
+        with self.assertRaises(ValueError):
+            must_resolve_runfile("file-does-not-exist")
+
+
+if __name__ == '__main__':
+    unittest.main()