TinyTapeout/tt-support-tools
1
0
Fork 0
mirror of https://github.com/TinyTapeout/tt-support-tools.git synced 2026-11-03 04:36:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(docs): update code to support new info.yaml format

Browse Source
This commit is contained in:
Uri Shaked
2023-12-23 22:37:55 +02:00
parent c2c0d8ffbd
commit 45633298e8
9 changed files with 79 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.py
+2
View File
@@ -8,3 +8,5 @@ class Config(TypedDict):
name: str
project_dir: str
start_date: str
end_date: str
openframe: bool
configure.py
+8 -32
View File
@@ -92,7 +92,7 @@ class Projects:
logging.info(f"loaded {len(self.projects)} projects")
def assert_unique(self, check):
def assert_unique(self, check: List[str]):
duplicates = [
item for item, count in collections.Counter(check).items() if count > 1
]
@@ -112,7 +112,6 @@ class Projects:
min_util = 100.0
max_util_project = None
languages: Dict[str, int] = {}
tags: List[str] = []
for project in self.projects:
try:
@@ -122,6 +121,9 @@ class Projects:
except KeyError:
continue
if project.is_chip_rom():
continue
delt = datetime.timedelta(
hours=dt.hour, minutes=dt.minute, seconds=dt.second
)
@@ -164,8 +166,7 @@ class Projects:
num_cells = project.get_cell_count_from_synth()
total_physical_cells += num_cells
yaml_data = project.get_project_doc_yaml()
lang = yaml_data["language"].lower()
lang = project.info.language
if lang in languages:
languages[lang] += 1
else:
@@ -183,11 +184,6 @@ class Projects:
if util < min_util:
min_util = util
try:
tags += yaml_data["tag"].split(",")
except KeyError:
pass
logging.info(f"build time for all projects {total_seconds / 3600} hrs")
logging.info(f"total wire length {total_wire_length} um")
logging.info(f"total cells {total_physical_cells}")
@@ -196,23 +192,6 @@ class Projects:
logging.info(f"max util {max_util} for project {max_util_project}")
logging.info(f"min util {min_util}")
logging.info(f"languages {languages}")
tags = [x.strip().lower() for x in tags]
tags = list(filter(lambda a: a != "", tags))
def count_items(lst: List[str]):
freq: Dict[str, int] = {}
for item in lst:
if item not in freq:
freq[item] = 1
else:
freq[item] += 1
sorted_items = sorted(freq.items(), key=lambda x: x[1], reverse=True)
for item, count in sorted_items[0:9]:
logging.info(f"{item:10}: {count}")
logging.info("top 10 tags")
count_items(tags)
if __name__ == "__main__":
@@ -315,7 +294,7 @@ if __name__ == "__main__":
else:
modules_yaml_name = "modules.yaml"
docs = Docs(config, projects.projects, args=args)
docs = Docs(config, projects.projects)
shuttle = ShuttleConfig(config, projects.projects, modules_yaml_name)
rom = ROMFile(config)
@@ -343,13 +322,10 @@ if __name__ == "__main__":
if args.update_image:
docs.update_image()
if args.dump_json:
docs.dump_json()
if args.dump_markdown:
shuttle.configure_mux()
docs.dump_markdown()
docs.write_datasheet(args.dump_markdown, args.dump_pdf)
if args.build_hugo_content:
shuttle.configure_mux()
docs.build_hugo_content()
docs.build_hugo_content(args.build_hugo_content)
docs/doc_template.md
+11 -20
View File
@@ -1,7 +1,5 @@
## {title} [{mux_address}]
{picture_link}
* Author: {author}
* Description: {description}
* [GitHub repository](__git_url__)
@@ -9,25 +7,18 @@
* Mux address: {mux_address}
* [Extra docs](__doc_link__)
* Clock: {clock_hz} Hz
* External hardware: {external_hw}
### How it works
{how_it_works}
### How to test
{how_to_test}
{user_docs}
### Pinout
| # | Input | Output | Bidirectional |
|---|--------------|--------------| -------------------|
| 0 | {inputs[0]} | {outputs[0]} | {bidirectional[0]} |
| 1 | {inputs[1]} | {outputs[1]} | {bidirectional[1]} |
| 2 | {inputs[2]} | {outputs[2]} | {bidirectional[2]} |
| 3 | {inputs[3]} | {outputs[3]} | {bidirectional[3]} |
| 4 | {inputs[4]} | {outputs[4]} | {bidirectional[4]} |
| 5 | {inputs[5]} | {outputs[5]} | {bidirectional[5]} |
| 6 | {inputs[6]} | {outputs[6]} | {bidirectional[6]} |
| 7 | {inputs[7]} | {outputs[7]} | {bidirectional[7]} |
| # | Input | Output | Bidirectional |
| - | -------------- | -------------- | --------------- |
| 0 | {pinout.ui[0]} | {pinout.uo[0]} | {pinout.uio[0]} |
| 1 | {pinout.ui[1]} | {pinout.uo[1]} | {pinout.uio[1]} |
| 2 | {pinout.ui[2]} | {pinout.uo[2]} | {pinout.uio[2]} |
| 3 | {pinout.ui[3]} | {pinout.uo[3]} | {pinout.uio[3]} |
| 4 | {pinout.ui[4]} | {pinout.uo[4]} | {pinout.uio[4]} |
| 5 | {pinout.ui[5]} | {pinout.uo[5]} | {pinout.uio[5]} |
| 6 | {pinout.ui[6]} | {pinout.uo[6]} | {pinout.uio[6]} |
| 7 | {pinout.ui[7]} | {pinout.uo[7]} | {pinout.uio[7]} |
docs/hugo_index_template.md
+4 -4
View File
@@ -1,10 +1,10 @@
---
title: "Tiny Tapeout 05"
title: "{{shuttle_name}}"
weight: 50
description: 174 designs, closed 4th November 2023
description: {{project_count}} designs, closed {{end_date}}
---
{{% toc %}}
{{{{% toc %}}}}
# Design details
@@ -22,5 +22,5 @@ description: 174 designs, closed 4th November 2023
# Chip map
{{< shuttle-map "tt05" >}}
{{{{< shuttle-map "{shuttle_id}" >}}}}
docs/hugo_template.md
+12 -21
View File
@@ -13,31 +13,22 @@ weight: {weight}
* {project_type} project
* [Extra docs]({doc_link})
* Clock: {clock_hz} Hz
* External hardware: {external_hw}
{picture_link}
### How it works
{how_it_works}
### How to test
{how_to_test}
{user_docs}
### IO
| # | Input | Output | Bidirectional |
|---|--------------|--------------| -------------------|
| 0 | {inputs[0]} | {outputs[0]} | {bidirectional[0]} |
| 1 | {inputs[1]} | {outputs[1]} | {bidirectional[1]} |
| 2 | {inputs[2]} | {outputs[2]} | {bidirectional[2]} |
| 3 | {inputs[3]} | {outputs[3]} | {bidirectional[3]} |
| 4 | {inputs[4]} | {outputs[4]} | {bidirectional[4]} |
| 5 | {inputs[5]} | {outputs[5]} | {bidirectional[5]} |
| 6 | {inputs[6]} | {outputs[6]} | {bidirectional[6]} |
| 7 | {inputs[7]} | {outputs[7]} | {bidirectional[7]} |
| # | Input | Output | Bidirectional |
| - | -------------- | -------------- | --------------- |
| 0 | {pinout.ui[0]} | {pinout.uo[0]} | {pinout.uio[0]} |
| 1 | {pinout.ui[1]} | {pinout.uo[1]} | {pinout.uio[1]} |
| 2 | {pinout.ui[2]} | {pinout.uo[2]} | {pinout.uio[2]} |
| 3 | {pinout.ui[3]} | {pinout.uo[3]} | {pinout.uio[3]} |
| 4 | {pinout.ui[4]} | {pinout.uo[4]} | {pinout.uio[4]} |
| 5 | {pinout.ui[5]} | {pinout.uo[5]} | {pinout.uio[5]} |
| 6 | {pinout.ui[6]} | {pinout.uo[6]} | {pinout.uio[6]} |
| 7 | {pinout.ui[7]} | {pinout.uo[7]} | {pinout.uio[7]} |
### Chip location
{{{{< shuttle-map "tt05" "{mux_address}" >}}}}
{{{{< shuttle-map "{shuttle_id}" "{mux_address}" >}}}}
documentation.py
+20 -80
View File
@@ -1,28 +1,22 @@
import json
import logging
import os
import shutil
import subprocess
from typing import List
from typing import List, Optional
import frontmatter # type: ignore
import git # type: ignore
from config import Config
from git_utils import get_first_remote
from markdown_utils import latex_centered_image, rewrite_image_paths
from markdown_utils import rewrite_image_paths
from project import Project
class DocsArgs:
dump_json: str
class Docs:
def __init__(self, config: Config, projects: List[Project], args: DocsArgs):
def __init__(self, config: Config, projects: List[Project]):
self.config = config
self.projects = projects
self.args = args
self.script_dir = os.path.dirname(os.path.realpath(__file__))
# stuff related to docs
@@ -52,17 +46,6 @@ class Docs:
logging.info(cmd)
os.system(cmd)
# create a json file of all the project info, this is then used by tinytapeout.com to show projects
def dump_json(self):
designs = []
for project in self.projects:
design = project.get_yaml()
designs.append(design)
with open(self.args.dump_json, "w") as fh:
fh.write(json.dumps(designs, indent=4))
logging.info(f"wrote json to {self.args.dump_json}")
def load_doc_template(self, name: str) -> str:
root = os.path.join(self.script_dir, "docs")
doc_path = os.path.join(root, name)
@@ -76,14 +59,14 @@ class Docs:
else:
return doc.content + "\n"
def dump_markdown(self):
def write_datasheet(self, markdown_file: str, pdf_file: Optional[str] = None):
doc_header = self.load_doc_template("doc_header.md")
doc_template = self.load_doc_template("doc_template.md")
doc_pinout = self.load_doc_template("PINOUT.md")
doc_info = self.load_doc_template("../../tt-multiplexer/docs/INFO.md")
doc_credits = self.load_doc_template("CREDITS.md")
with open(self.args.dump_markdown, "w") as fh:
with open(markdown_file, "w") as fh:
repo = git.Repo(".")
fh.write(
doc_header.format(name=self.config["name"], repo=get_first_remote(repo))
@@ -92,30 +75,16 @@ class Docs:
self.projects.sort(key=lambda x: x.mux_address)
for project in self.projects:
yaml_data = project.get_project_doc_yaml()
yaml_data = project.get_project_docs_dict()
yaml_data["mux_address"] = project.mux_address
logging.info(f"building datasheet for {project}")
# handle pictures
yaml_data["picture_link"] = ""
if yaml_data["picture"]:
extension = os.path.splitext(yaml_data["picture"])[1]
picture_path = os.path.join(
project.local_dir, f"picture{extension}"
)
if os.path.exists(picture_path):
yaml_data["picture_link"] = latex_centered_image(picture_path)
else:
logging.warning(f"picture {picture_path} not found, skipping")
# ensure there are no LaTeX escape sequences in various fields, and that optional fields are set
for key in [
"author",
"description",
"how_it_works",
"how_to_test",
"external_hw",
"user_docs",
"clock_hz",
"git_url",
"doc_link",
@@ -127,12 +96,6 @@ class Docs:
else:
yaml_data[key] = ""
# many people remove unused pins in input / output / bidirectional
for key in ["inputs", "outputs", "bidirectional"]:
if not yaml_data[key]:
yaml_data[key] = []
yaml_data[key].extend((8 - len(yaml_data[key])) * ["n/a"])
# now build the doc & print it
try:
doc = (
@@ -152,17 +115,16 @@ class Docs:
fh.write("\n\\clearpage\n")
fh.write(doc_credits)
logging.info(f"wrote markdown to {self.args.dump_markdown}")
logging.info(f"wrote markdown to {markdown_file}")
if self.args.dump_pdf:
pdf_cmd = f"pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {self.args.dump_markdown} -o {self.args.dump_pdf}"
if pdf_file is not None:
pdf_cmd = f"pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {markdown_file} -o {pdf_file}"
logging.info(pdf_cmd)
p = subprocess.run(pdf_cmd, shell=True)
if p.returncode != 0:
logging.error("pdf command failed")
def build_hugo_content(self) -> None:
hugo_root = self.args.build_hugo_content
def build_hugo_content(self, hugo_root: str) -> None:
hugo_images = os.path.join(hugo_root, "images")
shutil.rmtree(hugo_root)
os.makedirs(hugo_root)
@@ -185,8 +147,14 @@ class Docs:
# index page
logging.info("building pages")
shuttle_info = {
"shuttle_name": self.config["name"],
"shuttle_id": self.config["id"],
"project_count": len(self.projects),
"end_date": self.config["end_date"],
}
with open(os.path.join(hugo_root, "_index.md"), "w") as fh:
fh.write(index_template)
fh.write(index_template.format(**shuttle_info))
fh.write("# All projects\n")
fh.write("| Index | Title | Author |\n")
fh.write("| ----- | ----- | -------|\n")
@@ -199,41 +167,13 @@ class Docs:
project_image_dir = os.path.join(project_dir, "images")
os.makedirs(project_dir)
os.makedirs(project_image_dir)
yaml_data = project.get_project_doc_yaml()
yaml_data = project.get_project_docs_dict()
yaml_data["mux_address"] = project.mux_address
if '""' in yaml_data["title"]:
yaml_data["title"] = yaml_data["title"].replace('""', "")
yaml_data["index"] = project.index
yaml_data["weight"] = project.index + 1
yaml_data["git_action"] = project.get_workflow_url_when_submitted()
for key in "external_hw", "clock_hz":
if key not in yaml_data:
yaml_data[key] = ""
yaml_data["shuttle_id"] = self.config["id"]
# many people remove unused pins in input / output / bidirectional
for key in ["inputs", "outputs", "bidirectional"]:
yaml_data[key].extend((8 - len(yaml_data[key])) * ["n/a"])
yaml_data["picture_link"] = ""
if yaml_data["picture"]:
extension = os.path.splitext(yaml_data["picture"])[1]
picture_path = os.path.join(
project.local_dir, f"picture{extension}"
)
picture_basename = os.path.basename(picture_path)
try:
shutil.copyfile(
picture_path,
os.path.join(project_image_dir, picture_basename),
)
yaml_data[
"picture_link"
] = f"![picture](images/{picture_basename})"
logging.warning(f"picture found {picture_path}")
except FileNotFoundError:
logging.warning(f"picture not found {picture_path}")
yaml_data["picture_link"] = "Image path is broken"
doc = doc_template.format(**yaml_data)
with open(os.path.join(project_dir, "_index.md"), "w") as pfh:
pfh.write(doc)
markdown_utils.py
+6 -8
View File
@@ -1,4 +1,5 @@
import os
from typing import Any, Dict
import mistune
from mistune.renderers.markdown import MarkdownRenderer
@@ -8,9 +9,12 @@ class HeadingsRenderer(MarkdownRenderer):
def __init__(self, min_level: int):
super().__init__()
self.min_level = min_level
self.initial_level = float("inf")
def heading(self, token, state):
token["attrs"]["level"] += self.min_level - 1
def heading(self, token: Dict[str, Any], state: Any):
if self.initial_level is float("inf"):
self.initial_level = token["attrs"]["level"]
token["attrs"]["level"] += self.min_level - max(1, self.initial_level)
return super().heading(token, state)
@@ -34,9 +38,3 @@ class ImagePathRewriterRenderer(MarkdownRenderer):
def rewrite_image_paths(source: str, prefix: str) -> str:
markdown = mistune.create_markdown(renderer=ImagePathRewriterRenderer(prefix))
return markdown(source)
def latex_centered_image(src: str):
return "\n".join(
[r"\begin{center}", rf"\includegraphics[]{{{src}}}", r"\end{center}"]
)
project.py
+7 -11
View File
@@ -217,20 +217,16 @@ class Project:
else:
return "HDL"
def get_project_doc_yaml(self):
# fstring dict support is limited to one level deep, so put the git url and wokwi url in the docs key
docs = self.yaml["documentation"]
def get_project_docs_dict(self):
docs = self.info.__dict__.copy()
docs["project_type"] = self.get_project_type_string()
docs["git_url"] = self.git_url
# "How it works" and "How to test" may include markdown headings - make sure they don't break the ToC
docs["how_it_works"] = limit_markdown_headings(
docs["how_it_works"], min_level=4
)
docs["how_to_test"] = limit_markdown_headings(docs["how_to_test"], min_level=4)
with open(os.path.join(self.local_dir, "docs/info.md")) as fh:
docs["user_docs"] = limit_markdown_headings(fh.read(), min_level=3)
return docs
def get_wokwi_url(self):
return f"https://wokwi.com/projects/{self.wokwi_id}"
return f"https://wokwi.com/projects/{self.info.wokwi_id}"
# top module name is defined in one of the source files, which one?
def find_top_verilog(self):
@@ -538,7 +534,7 @@ class Project:
logging.error("pdf command failed")
# Read and return top-level GDS data from the final GDS file, using gdstk:
def get_final_gds_top_cells(self) -> str:
def get_final_gds_top_cells(self):
gds = glob.glob(
os.path.join(self.local_dir, "runs/wokwi/results/final/gds/*gds")
)
@@ -639,7 +635,7 @@ class Project:
# Compress with pngquant:
final_png = "gds_render.png"
if self.yaml["project"]["tiles"] == "8x2":
if self.info.tiles == "8x2":
quality = "0-10" # Compress more for 8x2 tiles.
else:
quality = "0-30"
project_info.py
+9 -4
View File
@@ -26,6 +26,9 @@ class PinoutSection:
class ProjectInfo:
top_module: str
source_files: List[str]
def __init__(self, yaml_data: Dict[str, Any]):
# Validate Version
yaml_version = yaml_data.get("yaml_version")
@@ -95,7 +98,7 @@ class ProjectInfo:
raise ProjectYamlError(
"Top module must start with 'tt_um_' (e.g. tt_um_my_project)"
)
self.top_module: str = top_module
self.top_module = top_module
if "source_files" not in project_section:
raise ProjectYamlError(
@@ -103,10 +106,12 @@ class ProjectInfo:
)
if len(project_section["source_files"]) == 0:
raise ProjectYamlError("No source files specified")
self.source_files: List[str] = project_section["source_files"]
self.source_files = project_section["source_files"]
self.discord: Optional[str] = project_section.get("discord")
self.clock_hz: Optional[int] = project_section["clock_hz"]
if "pinout" not in yaml_data:
raise ProjectYamlError("Missing 'pinout' section")
self.pinout = PinoutSection(yaml_data["pinout"])
self.discord: Optional[str] = project_section.get("discord")
self.doc_link: Optional[str] = project_section.get("doc_link")
self.clock_hz: Optional[int] = project_section["clock_hz"]