feat: add support for json config files (#55)

* feat: add support for json config files

* feat: make json the preferred config format

* feat: allow yaml config format

* feat!: remove compatibility tweaks for tt07's config.tcl

---------

Co-authored-by: htfab <mpw@htamas.net>

config_utils.py

@@ -0,0 +1,102 @@
+import json
+import logging
+import os
+import re
+import tkinter
+from collections.abc import Iterable
+
+import yaml
+
+
+class ConfigFileError(Exception):
+    pass
+
+
+def read_tcl_config(file: str, design_dir: str | None = None):
+    logging.warning(
+        "Support for TCL configuration files is deprecated and will be removed"
+    )
+    if design_dir is None:
+        design_dir = os.path.dirname(file)
+    design_dir = os.path.realpath(design_dir)
+    tcl_code = open(file).read()
+    interp = tkinter.Tcl()
+    interp.eval("array unset ::env")
+    interp.setvar("env(DESIGN_DIR)", design_dir)
+    config = {}
+    env_rx = re.compile(r"(?:\:\:)?env\((\w+)\)")
+
+    def py_set(key: str, value: str | None = None):
+        if match := env_rx.fullmatch(key):
+            if value is not None:
+                value = value.replace(design_dir, "dir::")
+                value = value.replace("dir::/", "dir::")
+                config[match.group(1)] = value
+
+    py_set_name = interp.register(py_set)
+    interp.call("rename", py_set_name, "_py_set")
+    interp.call("rename", "set", "_orig_set")
+    interp.eval("proc set args { _py_set {*}$args; tailcall _orig_set {*}$args; }")
+    interp.eval(tcl_code)
+    return config
+
+
+def read_json_config(file: str):
+    return json.load(open(file))
+
+
+def read_yaml_config(file: str):
+    return yaml.safe_load(open(file))
+
+
+def read_config(basename: str, formats: Iterable[str], design_dir: str | None = None):
+    for fmt in formats:
+        file = f"{basename}.{fmt}"
+        if os.path.exists(file):
+            if fmt == "tcl":
+                return read_tcl_config(file, design_dir)
+            elif fmt == "json":
+                return read_json_config(file)
+            elif fmt == "yaml":
+                return read_yaml_config(file)
+            else:
+                raise ConfigFileError(f"Unexpected configuration file format: {fmt}")
+    raise ConfigFileError(
+        f"Could not file configuration file {basename}.{{{'|'.join(formats)}}}"
+    )
+
+
+def write_tcl_config(config: dict, file: str):
+    logging.warning(
+        "Support for TCL configuration files is deprecated and will be removed"
+    )
+    with open(file, "w") as f:
+        for key, value in config.items():
+            if type(value) in (list, tuple):
+                value = " ".join(value)
+            if type(value) == str:
+                value = value.replace("dir::", "$::env(DESIGN_DIR)/")
+            print(f'set ::env({key}) "{value}"', file=f)
+
+
+def write_json_config(config: dict, file: str):
+    with open(file, "w") as f:
+        json.dump(config, f, indent=2)
+
+
+def write_yaml_config(config: dict, file: str):
+    with open(file, "w") as f:
+        yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False)
+
+
+def write_config(config: dict, basename: str, formats: Iterable[str]):
+    for fmt in formats:
+        file = f"{basename}.{fmt}"
+        if fmt == "tcl":
+            write_tcl_config(config, file)
+        elif fmt == "json":
+            write_json_config(config, file)
+        elif fmt == "yaml":
+            write_yaml_config(config, file)
+        else:
+            raise ConfigFileError("Unexpected configuration file format: {fmt}")

project.py

@@ -17,6 +17,7 @@ from git.repo import Repo
 
 import git_utils
 from cells import Cell, load_cells
+from config_utils import read_config, write_config
 from markdown_utils import limit_markdown_headings
 from project_info import ProjectInfo, ProjectYamlError
 
@@ -447,26 +448,19 @@ class Project:
             self.fetch_wokwi_files()
         self.check_ports()
         logging.info("creating include file")
-        filename = "user_config.tcl"
-        with open(os.path.join(self.src_dir, filename), "w") as fh:
-            fh.write("set ::env(DESIGN_NAME) {}\n".format(self.info.top_module))
-            fh.write('set ::env(VERILOG_FILES) "\\\n')
-            for line, source in enumerate(self.sources):
-                fh.write("    $::env(DESIGN_DIR)/" + source)
-                if line != len(self.sources) - 1:
-                    fh.write(" \\\n")
-            fh.write('"\n\n')
         tiles = self.info.tiles
         die_area = tile_sizes[tiles]
-            fh.write(f"# Project area: {tiles} tiles\n")
+        config = {
-            fh.write(f'set ::env(DIE_AREA) "{die_area}"\n')
+            "DESIGN_NAME": self.info.top_module,
-            fh.write(
+            "VERILOG_FILES": [f"dir::{src}" for src in self.sources],
-                f'set ::env(FP_DEF_TEMPLATE) "$::env(DESIGN_DIR)/../tt/def/tt_block_{tiles}_pg.def"\n'
+            "DIE_AREA": die_area,
-            )
+            "FP_DEF_TEMPLATE": f"dir::../tt/def/tt_block_{tiles}_pg.def",
+        }
+        write_config(config, os.path.join(self.src_dir, "user_config"), ("json",))
 
     def golden_harden(self):
         logging.info(f"hardening {self}")
-        shutil.copyfile("golden_config.tcl", os.path.join(self.src_dir, "config.tcl"))
+        shutil.copyfile("golden_config.json", os.path.join(self.src_dir, "config.json"))
         self.harden()
 
     def harden(self):
@@ -478,18 +472,19 @@ class Project:
         tt_version = self.get_tt_tools_version()
         workflow_url = self.get_workflow_url()
 
+        config = read_config("src/config", ("yaml", "json", "tcl"))
+        user_config = read_config("src/user_config", ("json",))
+        config.update(user_config)
+        write_config(config, "src/config_merged", ("json",))
+
         if self.args.openlane2:
-            # workaround for wrong pin sizes in lef
-            with open("src/config_patched.tcl", "w") as f:
-                f.write(open("src/config.tcl").read())
-                f.write("set ::env(MAGIC_WRITE_LEF_PINONLY) 1\n")
             shutil.rmtree("runs/wokwi", ignore_errors=True)
             os.makedirs("runs/wokwi", exist_ok=True)
             progress = "--hide-progress-bar" if "CI" in os.environ else ""
-            harden_cmd = f"python -m openlane --dockerized --run-tag wokwi --force-run-dir runs/wokwi {progress} src/config_patched.tcl"
+            harden_cmd = f"python -m openlane --dockerized --run-tag wokwi --force-run-dir runs/wokwi {progress} src/config_merged.json"
         else:
             # requires PDK, PDK_ROOT, OPENLANE_ROOT & OPENLANE_IMAGE_NAME to be set in local environment
-            harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK=$PDK -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -tag wokwi"'
+            harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK=$PDK -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -config_file /work/src/config_merged.json -tag wokwi"'
         logging.debug(harden_cmd)
         env = os.environ.copy()
         p = subprocess.run(harden_cmd, shell=True, env=env)

tt_tool.py

@@ -93,7 +93,7 @@ if __name__ == "__main__":
     # configure
     parser.add_argument(
         "--create-user-config",
-        help="create the user_config.tcl file with top module and source files",
+        help="create the user_config.json file with top module and source files",
         action="store_const",
         const=True,
     )