From 45633298e88bdc736bc12d31f35adb546f6e2a9f Mon Sep 17 00:00:00 2001 From: Uri Shaked Date: Sat, 23 Dec 2023 22:15:31 +0200 Subject: [PATCH] fix(docs): update code to support new info.yaml format --- config.py | 2 + configure.py | 40 +++------------ docs/doc_template.md | 31 ++++------- docs/hugo_index_template.md | 8 +-- docs/hugo_template.md | 33 +++++------- documentation.py | 100 ++++++++---------------------------- markdown_utils.py | 14 +++-- project.py | 18 +++---- project_info.py | 13 +++-- 9 files changed, 79 insertions(+), 180 deletions(-) diff --git a/config.py b/config.py index 6ded8ec..c1cfd1d 100644 --- a/config.py +++ b/config.py @@ -8,3 +8,5 @@ class Config(TypedDict): name: str project_dir: str start_date: str + end_date: str + openframe: bool diff --git a/configure.py b/configure.py index 5a5d0e1..d4fac64 100755 --- a/configure.py +++ b/configure.py @@ -92,7 +92,7 @@ class Projects: logging.info(f"loaded {len(self.projects)} projects") - def assert_unique(self, check): + def assert_unique(self, check: List[str]): duplicates = [ item for item, count in collections.Counter(check).items() if count > 1 ] @@ -112,7 +112,6 @@ class Projects: min_util = 100.0 max_util_project = None languages: Dict[str, int] = {} - tags: List[str] = [] for project in self.projects: try: @@ -122,6 +121,9 @@ class Projects: except KeyError: continue + if project.is_chip_rom(): + continue + delt = datetime.timedelta( hours=dt.hour, minutes=dt.minute, seconds=dt.second ) @@ -164,8 +166,7 @@ class Projects: num_cells = project.get_cell_count_from_synth() total_physical_cells += num_cells - yaml_data = project.get_project_doc_yaml() - lang = yaml_data["language"].lower() + lang = project.info.language if lang in languages: languages[lang] += 1 else: @@ -183,11 +184,6 @@ class Projects: if util < min_util: min_util = util - try: - tags += yaml_data["tag"].split(",") - except KeyError: - pass - logging.info(f"build time for all projects {total_seconds / 3600} hrs") logging.info(f"total wire length {total_wire_length} um") logging.info(f"total cells {total_physical_cells}") @@ -196,23 +192,6 @@ class Projects: logging.info(f"max util {max_util} for project {max_util_project}") logging.info(f"min util {min_util}") logging.info(f"languages {languages}") - tags = [x.strip().lower() for x in tags] - tags = list(filter(lambda a: a != "", tags)) - - def count_items(lst: List[str]): - freq: Dict[str, int] = {} - for item in lst: - if item not in freq: - freq[item] = 1 - else: - freq[item] += 1 - - sorted_items = sorted(freq.items(), key=lambda x: x[1], reverse=True) - for item, count in sorted_items[0:9]: - logging.info(f"{item:10}: {count}") - - logging.info("top 10 tags") - count_items(tags) if __name__ == "__main__": @@ -315,7 +294,7 @@ if __name__ == "__main__": else: modules_yaml_name = "modules.yaml" - docs = Docs(config, projects.projects, args=args) + docs = Docs(config, projects.projects) shuttle = ShuttleConfig(config, projects.projects, modules_yaml_name) rom = ROMFile(config) @@ -343,13 +322,10 @@ if __name__ == "__main__": if args.update_image: docs.update_image() - if args.dump_json: - docs.dump_json() - if args.dump_markdown: shuttle.configure_mux() - docs.dump_markdown() + docs.write_datasheet(args.dump_markdown, args.dump_pdf) if args.build_hugo_content: shuttle.configure_mux() - docs.build_hugo_content() + docs.build_hugo_content(args.build_hugo_content) diff --git a/docs/doc_template.md b/docs/doc_template.md index 6fc489f..c3270d7 100644 --- a/docs/doc_template.md +++ b/docs/doc_template.md @@ -1,7 +1,5 @@ ## {title} [{mux_address}] -{picture_link} - * Author: {author} * Description: {description} * [GitHub repository](__git_url__) @@ -9,25 +7,18 @@ * Mux address: {mux_address} * [Extra docs](__doc_link__) * Clock: {clock_hz} Hz -* External hardware: {external_hw} -### How it works - -{how_it_works} - -### How to test - -{how_to_test} +{user_docs} ### Pinout -| # | Input | Output | Bidirectional | -|---|--------------|--------------| -------------------| -| 0 | {inputs[0]} | {outputs[0]} | {bidirectional[0]} | -| 1 | {inputs[1]} | {outputs[1]} | {bidirectional[1]} | -| 2 | {inputs[2]} | {outputs[2]} | {bidirectional[2]} | -| 3 | {inputs[3]} | {outputs[3]} | {bidirectional[3]} | -| 4 | {inputs[4]} | {outputs[4]} | {bidirectional[4]} | -| 5 | {inputs[5]} | {outputs[5]} | {bidirectional[5]} | -| 6 | {inputs[6]} | {outputs[6]} | {bidirectional[6]} | -| 7 | {inputs[7]} | {outputs[7]} | {bidirectional[7]} | +| # | Input | Output | Bidirectional | +| - | -------------- | -------------- | --------------- | +| 0 | {pinout.ui[0]} | {pinout.uo[0]} | {pinout.uio[0]} | +| 1 | {pinout.ui[1]} | {pinout.uo[1]} | {pinout.uio[1]} | +| 2 | {pinout.ui[2]} | {pinout.uo[2]} | {pinout.uio[2]} | +| 3 | {pinout.ui[3]} | {pinout.uo[3]} | {pinout.uio[3]} | +| 4 | {pinout.ui[4]} | {pinout.uo[4]} | {pinout.uio[4]} | +| 5 | {pinout.ui[5]} | {pinout.uo[5]} | {pinout.uio[5]} | +| 6 | {pinout.ui[6]} | {pinout.uo[6]} | {pinout.uio[6]} | +| 7 | {pinout.ui[7]} | {pinout.uo[7]} | {pinout.uio[7]} | diff --git a/docs/hugo_index_template.md b/docs/hugo_index_template.md index ec78b54..5633876 100644 --- a/docs/hugo_index_template.md +++ b/docs/hugo_index_template.md @@ -1,10 +1,10 @@ --- -title: "Tiny Tapeout 05" +title: "{{shuttle_name}}" weight: 50 -description: 174 designs, closed 4th November 2023 +description: {{project_count}} designs, closed {{end_date}} --- -{{% toc %}} +{{{{% toc %}}}} # Design details @@ -22,5 +22,5 @@ description: 174 designs, closed 4th November 2023 # Chip map -{{< shuttle-map "tt05" >}} +{{{{< shuttle-map "{shuttle_id}" >}}}} diff --git a/docs/hugo_template.md b/docs/hugo_template.md index 944cfed..78da27c 100644 --- a/docs/hugo_template.md +++ b/docs/hugo_template.md @@ -13,31 +13,22 @@ weight: {weight} * {project_type} project * [Extra docs]({doc_link}) * Clock: {clock_hz} Hz -* External hardware: {external_hw} -{picture_link} - -### How it works - -{how_it_works} - -### How to test - -{how_to_test} +{user_docs} ### IO -| # | Input | Output | Bidirectional | -|---|--------------|--------------| -------------------| -| 0 | {inputs[0]} | {outputs[0]} | {bidirectional[0]} | -| 1 | {inputs[1]} | {outputs[1]} | {bidirectional[1]} | -| 2 | {inputs[2]} | {outputs[2]} | {bidirectional[2]} | -| 3 | {inputs[3]} | {outputs[3]} | {bidirectional[3]} | -| 4 | {inputs[4]} | {outputs[4]} | {bidirectional[4]} | -| 5 | {inputs[5]} | {outputs[5]} | {bidirectional[5]} | -| 6 | {inputs[6]} | {outputs[6]} | {bidirectional[6]} | -| 7 | {inputs[7]} | {outputs[7]} | {bidirectional[7]} | +| # | Input | Output | Bidirectional | +| - | -------------- | -------------- | --------------- | +| 0 | {pinout.ui[0]} | {pinout.uo[0]} | {pinout.uio[0]} | +| 1 | {pinout.ui[1]} | {pinout.uo[1]} | {pinout.uio[1]} | +| 2 | {pinout.ui[2]} | {pinout.uo[2]} | {pinout.uio[2]} | +| 3 | {pinout.ui[3]} | {pinout.uo[3]} | {pinout.uio[3]} | +| 4 | {pinout.ui[4]} | {pinout.uo[4]} | {pinout.uio[4]} | +| 5 | {pinout.ui[5]} | {pinout.uo[5]} | {pinout.uio[5]} | +| 6 | {pinout.ui[6]} | {pinout.uo[6]} | {pinout.uio[6]} | +| 7 | {pinout.ui[7]} | {pinout.uo[7]} | {pinout.uio[7]} | ### Chip location -{{{{< shuttle-map "tt05" "{mux_address}" >}}}} +{{{{< shuttle-map "{shuttle_id}" "{mux_address}" >}}}} diff --git a/documentation.py b/documentation.py index d6f9c1a..57b0a9f 100644 --- a/documentation.py +++ b/documentation.py @@ -1,28 +1,22 @@ -import json import logging import os import shutil import subprocess -from typing import List +from typing import List, Optional import frontmatter # type: ignore import git # type: ignore from config import Config from git_utils import get_first_remote -from markdown_utils import latex_centered_image, rewrite_image_paths +from markdown_utils import rewrite_image_paths from project import Project -class DocsArgs: - dump_json: str - - class Docs: - def __init__(self, config: Config, projects: List[Project], args: DocsArgs): + def __init__(self, config: Config, projects: List[Project]): self.config = config self.projects = projects - self.args = args self.script_dir = os.path.dirname(os.path.realpath(__file__)) # stuff related to docs @@ -52,17 +46,6 @@ class Docs: logging.info(cmd) os.system(cmd) - # create a json file of all the project info, this is then used by tinytapeout.com to show projects - def dump_json(self): - designs = [] - for project in self.projects: - design = project.get_yaml() - designs.append(design) - - with open(self.args.dump_json, "w") as fh: - fh.write(json.dumps(designs, indent=4)) - logging.info(f"wrote json to {self.args.dump_json}") - def load_doc_template(self, name: str) -> str: root = os.path.join(self.script_dir, "docs") doc_path = os.path.join(root, name) @@ -76,14 +59,14 @@ class Docs: else: return doc.content + "\n" - def dump_markdown(self): + def write_datasheet(self, markdown_file: str, pdf_file: Optional[str] = None): doc_header = self.load_doc_template("doc_header.md") doc_template = self.load_doc_template("doc_template.md") doc_pinout = self.load_doc_template("PINOUT.md") doc_info = self.load_doc_template("../../tt-multiplexer/docs/INFO.md") doc_credits = self.load_doc_template("CREDITS.md") - with open(self.args.dump_markdown, "w") as fh: + with open(markdown_file, "w") as fh: repo = git.Repo(".") fh.write( doc_header.format(name=self.config["name"], repo=get_first_remote(repo)) @@ -92,30 +75,16 @@ class Docs: self.projects.sort(key=lambda x: x.mux_address) for project in self.projects: - yaml_data = project.get_project_doc_yaml() - + yaml_data = project.get_project_docs_dict() yaml_data["mux_address"] = project.mux_address logging.info(f"building datasheet for {project}") - # handle pictures - yaml_data["picture_link"] = "" - if yaml_data["picture"]: - extension = os.path.splitext(yaml_data["picture"])[1] - picture_path = os.path.join( - project.local_dir, f"picture{extension}" - ) - if os.path.exists(picture_path): - yaml_data["picture_link"] = latex_centered_image(picture_path) - else: - logging.warning(f"picture {picture_path} not found, skipping") # ensure there are no LaTeX escape sequences in various fields, and that optional fields are set for key in [ "author", "description", - "how_it_works", - "how_to_test", - "external_hw", + "user_docs", "clock_hz", "git_url", "doc_link", @@ -127,12 +96,6 @@ class Docs: else: yaml_data[key] = "" - # many people remove unused pins in input / output / bidirectional - for key in ["inputs", "outputs", "bidirectional"]: - if not yaml_data[key]: - yaml_data[key] = [] - yaml_data[key].extend((8 - len(yaml_data[key])) * ["n/a"]) - # now build the doc & print it try: doc = ( @@ -152,17 +115,16 @@ class Docs: fh.write("\n\\clearpage\n") fh.write(doc_credits) - logging.info(f"wrote markdown to {self.args.dump_markdown}") + logging.info(f"wrote markdown to {markdown_file}") - if self.args.dump_pdf: - pdf_cmd = f"pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {self.args.dump_markdown} -o {self.args.dump_pdf}" + if pdf_file is not None: + pdf_cmd = f"pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {markdown_file} -o {pdf_file}" logging.info(pdf_cmd) p = subprocess.run(pdf_cmd, shell=True) if p.returncode != 0: logging.error("pdf command failed") - def build_hugo_content(self) -> None: - hugo_root = self.args.build_hugo_content + def build_hugo_content(self, hugo_root: str) -> None: hugo_images = os.path.join(hugo_root, "images") shutil.rmtree(hugo_root) os.makedirs(hugo_root) @@ -185,8 +147,14 @@ class Docs: # index page logging.info("building pages") + shuttle_info = { + "shuttle_name": self.config["name"], + "shuttle_id": self.config["id"], + "project_count": len(self.projects), + "end_date": self.config["end_date"], + } with open(os.path.join(hugo_root, "_index.md"), "w") as fh: - fh.write(index_template) + fh.write(index_template.format(**shuttle_info)) fh.write("# All projects\n") fh.write("| Index | Title | Author |\n") fh.write("| ----- | ----- | -------|\n") @@ -199,41 +167,13 @@ class Docs: project_image_dir = os.path.join(project_dir, "images") os.makedirs(project_dir) os.makedirs(project_image_dir) - yaml_data = project.get_project_doc_yaml() + yaml_data = project.get_project_docs_dict() yaml_data["mux_address"] = project.mux_address - if '""' in yaml_data["title"]: - yaml_data["title"] = yaml_data["title"].replace('""', "") - yaml_data["index"] = project.index yaml_data["weight"] = project.index + 1 yaml_data["git_action"] = project.get_workflow_url_when_submitted() - for key in "external_hw", "clock_hz": - if key not in yaml_data: - yaml_data[key] = "" + yaml_data["shuttle_id"] = self.config["id"] - # many people remove unused pins in input / output / bidirectional - for key in ["inputs", "outputs", "bidirectional"]: - yaml_data[key].extend((8 - len(yaml_data[key])) * ["n/a"]) - - yaml_data["picture_link"] = "" - if yaml_data["picture"]: - extension = os.path.splitext(yaml_data["picture"])[1] - picture_path = os.path.join( - project.local_dir, f"picture{extension}" - ) - picture_basename = os.path.basename(picture_path) - try: - shutil.copyfile( - picture_path, - os.path.join(project_image_dir, picture_basename), - ) - yaml_data[ - "picture_link" - ] = f"![picture](images/{picture_basename})" - logging.warning(f"picture found {picture_path}") - except FileNotFoundError: - logging.warning(f"picture not found {picture_path}") - yaml_data["picture_link"] = "Image path is broken" doc = doc_template.format(**yaml_data) with open(os.path.join(project_dir, "_index.md"), "w") as pfh: pfh.write(doc) diff --git a/markdown_utils.py b/markdown_utils.py index a33e100..5359127 100644 --- a/markdown_utils.py +++ b/markdown_utils.py @@ -1,4 +1,5 @@ import os +from typing import Any, Dict import mistune from mistune.renderers.markdown import MarkdownRenderer @@ -8,9 +9,12 @@ class HeadingsRenderer(MarkdownRenderer): def __init__(self, min_level: int): super().__init__() self.min_level = min_level + self.initial_level = float("inf") - def heading(self, token, state): - token["attrs"]["level"] += self.min_level - 1 + def heading(self, token: Dict[str, Any], state: Any): + if self.initial_level is float("inf"): + self.initial_level = token["attrs"]["level"] + token["attrs"]["level"] += self.min_level - max(1, self.initial_level) return super().heading(token, state) @@ -34,9 +38,3 @@ class ImagePathRewriterRenderer(MarkdownRenderer): def rewrite_image_paths(source: str, prefix: str) -> str: markdown = mistune.create_markdown(renderer=ImagePathRewriterRenderer(prefix)) return markdown(source) - - -def latex_centered_image(src: str): - return "\n".join( - [r"\begin{center}", rf"\includegraphics[]{{{src}}}", r"\end{center}"] - ) diff --git a/project.py b/project.py index b9cfb57..b66ba18 100644 --- a/project.py +++ b/project.py @@ -217,20 +217,16 @@ class Project: else: return "HDL" - def get_project_doc_yaml(self): - # fstring dict support is limited to one level deep, so put the git url and wokwi url in the docs key - docs = self.yaml["documentation"] + def get_project_docs_dict(self): + docs = self.info.__dict__.copy() docs["project_type"] = self.get_project_type_string() docs["git_url"] = self.git_url - # "How it works" and "How to test" may include markdown headings - make sure they don't break the ToC - docs["how_it_works"] = limit_markdown_headings( - docs["how_it_works"], min_level=4 - ) - docs["how_to_test"] = limit_markdown_headings(docs["how_to_test"], min_level=4) + with open(os.path.join(self.local_dir, "docs/info.md")) as fh: + docs["user_docs"] = limit_markdown_headings(fh.read(), min_level=3) return docs def get_wokwi_url(self): - return f"https://wokwi.com/projects/{self.wokwi_id}" + return f"https://wokwi.com/projects/{self.info.wokwi_id}" # top module name is defined in one of the source files, which one? def find_top_verilog(self): @@ -538,7 +534,7 @@ class Project: logging.error("pdf command failed") # Read and return top-level GDS data from the final GDS file, using gdstk: - def get_final_gds_top_cells(self) -> str: + def get_final_gds_top_cells(self): gds = glob.glob( os.path.join(self.local_dir, "runs/wokwi/results/final/gds/*gds") ) @@ -639,7 +635,7 @@ class Project: # Compress with pngquant: final_png = "gds_render.png" - if self.yaml["project"]["tiles"] == "8x2": + if self.info.tiles == "8x2": quality = "0-10" # Compress more for 8x2 tiles. else: quality = "0-30" diff --git a/project_info.py b/project_info.py index 1a1aef3..6e108d8 100644 --- a/project_info.py +++ b/project_info.py @@ -26,6 +26,9 @@ class PinoutSection: class ProjectInfo: + top_module: str + source_files: List[str] + def __init__(self, yaml_data: Dict[str, Any]): # Validate Version yaml_version = yaml_data.get("yaml_version") @@ -95,7 +98,7 @@ class ProjectInfo: raise ProjectYamlError( "Top module must start with 'tt_um_' (e.g. tt_um_my_project)" ) - self.top_module: str = top_module + self.top_module = top_module if "source_files" not in project_section: raise ProjectYamlError( @@ -103,10 +106,12 @@ class ProjectInfo: ) if len(project_section["source_files"]) == 0: raise ProjectYamlError("No source files specified") - self.source_files: List[str] = project_section["source_files"] + self.source_files = project_section["source_files"] - self.discord: Optional[str] = project_section.get("discord") - self.clock_hz: Optional[int] = project_section["clock_hz"] if "pinout" not in yaml_data: raise ProjectYamlError("Missing 'pinout' section") self.pinout = PinoutSection(yaml_data["pinout"]) + + self.discord: Optional[str] = project_section.get("discord") + self.doc_link: Optional[str] = project_section.get("doc_link") + self.clock_hz: Optional[int] = project_section["clock_hz"]