TinyTapeout/tt-support-tools
1
0
Fork 0
mirror of https://github.com/TinyTapeout/tt-support-tools.git synced 2026-11-02 20:26:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

style: fix lint errors + reformat source with black

Browse Source
This commit is contained in:
Uri Shaked
2023-09-21 22:30:52 +03:00
parent 7d48aab542
commit aa8f760375
14 changed files with 1285 additions and 954 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cells.py
+7 -4
View File
@@ -1,6 +1,7 @@
import os
from typing import List, Dict, TypedDict, Optional
import json
import os
from typing import Dict, List, Optional, TypedDict
class Port(TypedDict):
kind: str
@@ -8,6 +9,7 @@ class Port(TypedDict):
direction: str
description: str
class Cell(TypedDict):
description: str
file_prefix: str
@@ -19,10 +21,11 @@ class Cell(TypedDict):
verilog_name: str
equation: Optional[str]
Cells = Dict[str, Cell]
def load_cells() -> Cells:
script_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(script_dir, 'cells.json')) as fh:
with open(os.path.join(script_dir, "cells.json")) as fh:
return json.load(fh)
config.py
+1
View File
@@ -1,5 +1,6 @@
from typing import TypedDict
class Config(TypedDict):
"""TypedDict for Tiny Tapeout's config.yaml file."""
configure.py
+138 -62
View File
@@ -1,51 +1,62 @@
#!/usr/bin/env python3
import argparse
import collections
import datetime
import yaml
import json
import argparse, logging, sys, os, collections
from typing import Dict, List, TypedDict
from config import Config
from project import Project
from documentation import Docs
from shuttle import ShuttleConfig
from rom import ROMFile
import logging
import os
import sys
# pipe handling
from signal import signal, SIGPIPE, SIG_DFL
from signal import SIG_DFL, SIGPIPE, signal
from typing import Dict, List, TypedDict
import yaml
from config import Config
from documentation import Docs
from project import Project
from rom import ROMFile
from shuttle import ShuttleConfig
signal(SIGPIPE, SIG_DFL)
class Projects():
class Projects:
def __init__(self, config: Config, args):
self.args = args
self.config = config
self.project_dir = config['project_dir']
self.project_dir = config["project_dir"]
if not os.path.exists(self.project_dir):
os.makedirs(self.project_dir)
self.projects: List[Project] = []
project_list = [entry for entry in os.listdir(self.project_dir) if os.path.isdir(os.path.join(self.project_dir, entry))]
project_list = [
entry
for entry in os.listdir(self.project_dir)
if os.path.isdir(os.path.join(self.project_dir, entry))
]
if args.test:
project_list = ['tt_um_chip_rom', 'tt_um_factory_test']
project_list = ["tt_um_chip_rom", "tt_um_factory_test"]
elif args.sta_projects:
project_list = ['tt_um_loopback']
project_list = ["tt_um_loopback"]
for index, project_id in enumerate(project_list):
project_dir = os.path.join(self.project_dir, project_id)
commit_id_file = os.path.join(project_dir, 'commit_id.json')
commit_id_file = os.path.join(project_dir, "commit_id.json")
if not os.path.exists(commit_id_file):
logging.warning(f"no commit_id.json in {project_dir}, skipping")
continue
commit_id_data = json.load(open(commit_id_file))
project = Project(index, commit_id_data['repo'], project_dir, args, is_user_project=False)
project.commit_id = commit_id_data['commit']
project.sort_id = commit_id_data['sort_id']
project = Project(
index, commit_id_data["repo"], project_dir, args, is_user_project=False
)
project.commit_id = commit_id_data["commit"]
project.sort_id = commit_id_data["sort_id"]
# projects should now be installed, so load all the data from the yaml files
# fill projects will load from the fill project's directory
@@ -65,7 +76,9 @@ class Projects():
self.projects.sort(key=lambda x: x.sort_id)
all_macro_instances = [project.get_macro_instance() for project in self.projects]
all_macro_instances = [
project.get_macro_instance() for project in self.projects
]
self.assert_unique(all_macro_instances)
all_gds_files = [project.get_macro_gds_filename() for project in self.projects]
@@ -74,7 +87,9 @@ class Projects():
logging.info(f"loaded {len(self.projects)} projects")
def assert_unique(self, check):
duplicates = [item for item, count in collections.Counter(check).items() if count > 1]
duplicates = [
item for item, count in collections.Counter(check).items() if count > 1
]
if duplicates:
logging.error("duplicate projects: {}".format(duplicates))
exit(1)
@@ -95,47 +110,56 @@ class Projects():
for project in self.projects:
try:
dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss')
dt = datetime.datetime.strptime(
project.metrics["total_runtime"][:-3], "%Hh%Mm%Ss"
)
except KeyError:
continue
delt = datetime.timedelta(hours=dt.hour, minutes=dt.minute, seconds=dt.second)
delt = datetime.timedelta(
hours=dt.hour, minutes=dt.minute, seconds=dt.second
)
total_seconds += delt.total_seconds()
cell_count = project.get_cell_counts_from_gl()
script_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(script_dir, 'categories.json')) as fh:
with open(os.path.join(script_dir, "categories.json")) as fh:
categories = json.load(fh)
CategoryInfo = TypedDict("CategoryInfo", {'count': int, 'examples': List[str]})
CategoryInfo = TypedDict(
"CategoryInfo", {"count": int, "examples": List[str]}
)
by_category: Dict[str, CategoryInfo] = {}
total = 0
for cell_name in cell_count:
cat_index = categories['map'][cell_name]
cat_name = categories['categories'][cat_index]
cat_index = categories["map"][cell_name]
cat_name = categories["categories"][cat_index]
if cat_name in by_category:
by_category[cat_name]['count'] += cell_count[cell_name]
by_category[cat_name]['examples'].append(cell_name)
by_category[cat_name]["count"] += cell_count[cell_name]
by_category[cat_name]["examples"].append(cell_name)
else:
by_category[cat_name] = {'count' : cell_count[cell_name], 'examples' : [cell_name]}
by_category[cat_name] = {
"count": cell_count[cell_name],
"examples": [cell_name],
}
if cat_name not in ['Fill', 'Tap', 'Buffer', 'Misc']:
if cat_name not in ["Fill", "Tap", "Buffer", "Misc"]:
total += cell_count[cell_name]
if total < 10:
del by_category['Fill']
del by_category['Tap']
if 'Buffer' in by_category:
del by_category['Buffer']
del by_category["Fill"]
del by_category["Tap"]
if "Buffer" in by_category:
del by_category["Buffer"]
print(project.get_macro_name(), total, by_category)
total_wire_length += int(project.metrics['wire_length'])
total_wires_count += int(project.metrics['wires_count'])
util = float(project.metrics['OpenDP_Util'])
total_wire_length += int(project.metrics["wire_length"])
total_wires_count += int(project.metrics["wires_count"])
util = float(project.metrics["OpenDP_Util"])
num_cells = project.get_cell_count_from_synth()
total_physical_cells += num_cells
yaml_data = project.get_project_doc_yaml()
lang = yaml_data['language'].lower()
lang = yaml_data["language"].lower()
if lang in languages:
languages[lang] += 1
else:
@@ -154,7 +178,7 @@ class Projects():
min_util = util
try:
tags += yaml_data['tag'].split(',')
tags += yaml_data["tag"].split(",")
except KeyError:
pass
@@ -185,36 +209,88 @@ class Projects():
count_items(tags)
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="TinyTapeout configuration and docs")
with open('config.yaml') as fh:
with open("config.yaml") as fh:
config = yaml.safe_load(fh)
parser.add_argument('--list', help="list projects", action='store_const', const=True)
parser.add_argument('--single', help="do action on single project", type=int, default=-1)
parser.add_argument('--update-shuttle', help='configure shuttle for build', action='store_const', const=True)
parser.add_argument('--copy-macros', help='copy macros for building the tt_top project', action='store_const', const=True)
parser.add_argument('--copy-final-results', help='copy final project files to gds/lef directories', action='store_const', const=True)
parser.add_argument('--create-efabless-submission', help='create efabless submission files', action='store_const', const=True)
parser.add_argument('--harden', help="harden project", action="store_const", const=True)
parser.add_argument('--test', help='use test projects', action='store_const', const=True)
parser.add_argument('--sta-projects', help='use sta projects', action='store_const', const=True)
parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO)
parser.add_argument('--log-email', help="print persons email in messages", action="store_const", const=True)
parser.add_argument('--update-image', help="update the image", action="store_const", const=True)
parser.add_argument('--dump-json', help="dump json of all project data to given file")
parser.add_argument('--dump-markdown', help="dump markdown of all project data to given file")
parser.add_argument('--dump-pdf', help="create pdf from the markdown")
parser.add_argument('--build-hugo-content', help="directory to where to build hugo content")
parser.add_argument('--metrics', help="print some project metrics", action="store_const", const=True)
parser.add_argument(
"--list", help="list projects", action="store_const", const=True
)
parser.add_argument(
"--single", help="do action on single project", type=int, default=-1
)
parser.add_argument(
"--update-shuttle",
help="configure shuttle for build",
action="store_const",
const=True,
)
parser.add_argument(
"--copy-macros",
help="copy macros for building the tt_top project",
action="store_const",
const=True,
)
parser.add_argument(
"--copy-final-results",
help="copy final project files to gds/lef directories",
action="store_const",
const=True,
)
parser.add_argument(
"--create-efabless-submission",
help="create efabless submission files",
action="store_const",
const=True,
)
parser.add_argument(
"--harden", help="harden project", action="store_const", const=True
)
parser.add_argument(
"--test", help="use test projects", action="store_const", const=True
)
parser.add_argument(
"--sta-projects", help="use sta projects", action="store_const", const=True
)
parser.add_argument(
"--debug",
help="debug logging",
action="store_const",
dest="loglevel",
const=logging.DEBUG,
default=logging.INFO,
)
parser.add_argument(
"--log-email",
help="print persons email in messages",
action="store_const",
const=True,
)
parser.add_argument(
"--update-image", help="update the image", action="store_const", const=True
)
parser.add_argument(
"--dump-json", help="dump json of all project data to given file"
)
parser.add_argument(
"--dump-markdown", help="dump markdown of all project data to given file"
)
parser.add_argument("--dump-pdf", help="create pdf from the markdown")
parser.add_argument(
"--build-hugo-content", help="directory to where to build hugo content"
)
parser.add_argument(
"--metrics", help="print some project metrics", action="store_const", const=True
)
args = parser.parse_args()
# setup log
log_format = logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s')
log_format = logging.Formatter("%(asctime)s - %(levelname)-8s - %(message)s")
# configure the client logging
log = logging.getLogger('')
log = logging.getLogger("")
# has to be set to debug as is the root logger
log.setLevel(args.loglevel)
discord_bot.py
+33 -29
View File
@@ -2,14 +2,13 @@
import os
import sys
import logging
import discord
import yaml
import re
TOKEN = os.environ['DISCORD_TOKEN']
TARG = os.environ['TARGET_SERVER']
ROLE_ID = os.environ['ROLE_ID']
TOKEN = os.environ["DISCORD_TOKEN"]
TARG = os.environ["TARGET_SERVER"]
ROLE_ID = os.environ["ROLE_ID"]
intents = discord.Intents.default()
intents.members = True
@@ -17,50 +16,52 @@ intents.members = True
client = discord.Client(intents=intents)
exit_code = 0
@client.event
async def on_ready():
global exit_code
try:
print(f'{client.user} has connected to Discord!')
print(f"{client.user} has connected to Discord!")
server = None
for g in client.guilds:
if g.name == TARG:
server = g
break
if server == None:
print(f'Error: Bot is not in a server named {TARG}')
if server is None:
print(f"Error: Bot is not in a server named {TARG}")
exit_code = 1
await client.close()
role = server.get_role(int(ROLE_ID))
if not role:
print('Error: requested role does not exist on server!')
print("Error: requested role does not exist on server!")
exit_code = 1
await client.close()
participants = []
for project in os.listdir('./projects'):
info_file = os.path.join('projects', project, 'info.yaml')
for project in os.listdir("./projects"):
info_file = os.path.join("projects", project, "info.yaml")
if not os.path.exists(info_file):
print(f'info.yaml does not exist for project {project}. Skipping.')
print(f"info.yaml does not exist for project {project}. Skipping.")
else:
try:
stream = open(info_file, 'r')
stream = open(info_file, "r")
data = yaml.load(stream, Loader=yaml.FullLoader)
stream.close()
if 'discord' in data['documentation']:
dname = data['documentation']['discord']
if "discord" in data["documentation"]:
dname = data["documentation"]["discord"]
if len(dname):
if '#' in dname:
dname = dname.split('#')[0]
if "#" in dname:
dname = dname.split("#")[0]
# print(f'adding {dname}')
participants.append(dname)
else:
print(f'Project {project} has no discord username listed. Skipping.')
print(
f"Project {project} has no discord username listed. Skipping."
)
except Exception as ex:
print(f'Error parsing info.yaml for project {project}. Skipping.')
print(f'Error was: {ex}')
print(f"Error parsing info.yaml for project {project}. Skipping.")
print(f"Error was: {ex}")
members = []
for p in participants:
@@ -72,26 +73,29 @@ async def on_ready():
members.append(m)
break
else:
print(f'{p} not found in members')
print(f"{p} not found in members")
new_role_members = []
for m in members:
if len([d for d in m.roles if d.id == role.id]) == 0:
print(f'Giving the role to {m.name}')
print(f"Giving the role to {m.name}")
new_role_members.append(m.name)
await m.add_roles(role, reason='Automatic role assignment by GitHub actions pipeline.')
await m.add_roles(
role, reason="Automatic role assignment by GitHub actions pipeline."
)
else:
print(f'{m.name} already has role')
print(f"{m.name} already has role")
print('All done!')
print("All done!")
print("gave new role to:")
print('@' + ' @'.join(new_role_members))
print("@" + " @".join(new_role_members))
await client.close()
except Exception as ex:
print('Exception was thrown!')
print("Exception was thrown!")
print(ex)
exit_code = 1
await client.close()
client.run(TOKEN)
sys.exit(exit_code)
documentation.py
+86 -58
View File
@@ -1,17 +1,18 @@
import os
import logging
import subprocess
import shutil
import json
import git
import logging
import os
import shutil
import subprocess
import frontmatter # type: ignore
import git
from config import Config
from git_utils import get_first_remote
from markdown_utils import latex_centered_image, rewrite_image_paths
class Docs():
class Docs:
def __init__(self, config: Config, projects, args):
self.config = config
self.projects = projects
@@ -21,9 +22,9 @@ class Docs():
# stuff related to docs
def build_index(self):
logging.info("building doc index")
with open(os.path.join(self.script_dir, 'docs', 'README_init.md')) as fh:
with open(os.path.join(self.script_dir, "docs", "README_init.md")) as fh:
readme = fh.read()
with open("README.md", 'w') as fh:
with open("README.md", "w") as fh:
fh.write(readme)
fh.write("| Address | Author | Title | Type | Git Repo |\n")
fh.write("| ------- | ------ | ------| -----| ---------|\n")
@@ -32,10 +33,10 @@ class Docs():
fh.write(project.get_index_row())
def update_image(self):
ruby = os.path.join(self.script_dir, 'caravel_template', 'dump_pic.rb')
klayoutrc = os.path.join(self.script_dir, 'caravel_template', 'klayoutrc')
lyp = os.path.join(self.script_dir, 'caravel_template', 'caravel.lyp')
cmd = f'klayout -l {lyp} gds/user_project_wrapper.gds* -r {ruby} -c {klayoutrc}'
ruby = os.path.join(self.script_dir, "caravel_template", "dump_pic.rb")
klayoutrc = os.path.join(self.script_dir, "caravel_template", "klayoutrc")
lyp = os.path.join(self.script_dir, "caravel_template", "caravel.lyp")
cmd = f"klayout -l {lyp} gds/user_project_wrapper.gds* -r {ruby} -c {klayoutrc}"
logging.info(cmd)
os.system(cmd)
@@ -46,68 +47,83 @@ class Docs():
design = project.get_yaml()
designs.append(design)
with open(self.args.dump_json, 'w') as fh:
with open(self.args.dump_json, "w") as fh:
fh.write(json.dumps(designs, indent=4))
logging.info(f'wrote json to {self.args.dump_json}')
logging.info(f"wrote json to {self.args.dump_json}")
def load_doc_template(self, name):
root = os.path.join(self.script_dir, 'docs')
root = os.path.join(self.script_dir, "docs")
doc_path = os.path.join(root, name)
image_root = os.path.relpath(os.path.dirname(doc_path), ".")
doc = frontmatter.load(doc_path)
doc.content = rewrite_image_paths(doc.content, image_root)
if 'title' in doc:
doc['title'] = rewrite_image_paths(doc['title'], image_root)
if "title" in doc:
doc["title"] = rewrite_image_paths(doc["title"], image_root)
if len(doc.keys()) > 0:
return frontmatter.dumps(doc) + "\n"
else:
return doc.content + "\n"
def dump_markdown(self):
doc_header = self.load_doc_template('doc_header.md')
doc_template = self.load_doc_template('doc_template.md')
doc_pinout = self.load_doc_template('PINOUT.md')
doc_header = self.load_doc_template("doc_header.md")
doc_template = self.load_doc_template("doc_template.md")
doc_pinout = self.load_doc_template("PINOUT.md")
doc_info = self.load_doc_template("../../tt-multiplexer/docs/INFO.md")
doc_credits = self.load_doc_template('CREDITS.md')
doc_credits = self.load_doc_template("CREDITS.md")
with open(self.args.dump_markdown, 'w') as fh:
with open(self.args.dump_markdown, "w") as fh:
repo = git.Repo(".")
fh.write(doc_header.format(name=self.config['name'], repo=get_first_remote(repo)))
fh.write(
doc_header.format(name=self.config["name"], repo=get_first_remote(repo))
)
self.projects.sort(key=lambda x: x.mux_address)
for project in self.projects:
yaml_data = project.get_project_doc_yaml()
yaml_data['mux_address'] = project.mux_address
yaml_data["mux_address"] = project.mux_address
logging.info(f"building datasheet for {project}")
# handle pictures
yaml_data['picture_link'] = ''
if yaml_data['picture']:
extension = os.path.splitext(yaml_data['picture'])[1]
picture_path = os.path.join(project.local_dir, f"picture{extension}")
yaml_data["picture_link"] = ""
if yaml_data["picture"]:
extension = os.path.splitext(yaml_data["picture"])[1]
picture_path = os.path.join(
project.local_dir, f"picture{extension}"
)
if os.path.exists(picture_path):
yaml_data['picture_link'] = latex_centered_image(picture_path)
yaml_data["picture_link"] = latex_centered_image(picture_path)
else:
logging.warning(f"picture {picture_path} not found, skipping")
# ensure there are no LaTeX escape sequences in various fields, and that optional fields are set
for key in ['author', 'description', 'how_it_works', 'how_to_test', 'external_hw', 'clock_hz', 'git_url', 'doc_link']:
for key in [
"author",
"description",
"how_it_works",
"how_to_test",
"external_hw",
"clock_hz",
"git_url",
"doc_link",
]:
if key in yaml_data:
yaml_data[key] = str(yaml_data[key]).replace("\\", "\mbox{\\textbackslash}")
yaml_data[key] = str(yaml_data[key]).replace(
"\\", "\\mbox{\\textbackslash}"
)
else:
yaml_data[key] = ""
# many people remove unused pins in input / output / bidirectional
for key in ['inputs', 'outputs', 'bidirectional']:
yaml_data[key].extend((8 - len(yaml_data[key])) * ['n/a'])
for key in ["inputs", "outputs", "bidirectional"]:
yaml_data[key].extend((8 - len(yaml_data[key])) * ["n/a"])
# now build the doc & print it
try:
doc = (doc_template
.replace('__git_url__', '{git_url}')
.replace('__doc_link__', '{doc_link}')
doc = (
doc_template.replace("__git_url__", "{git_url}")
.replace("__doc_link__", "{doc_link}")
.format(**yaml_data)
)
fh.write(doc)
@@ -122,10 +138,10 @@ class Docs():
fh.write("\n\\clearpage\n")
fh.write(doc_credits)
logging.info(f'wrote markdown to {self.args.dump_markdown}')
logging.info(f"wrote markdown to {self.args.dump_markdown}")
if self.args.dump_pdf:
pdf_cmd = f'pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {self.args.dump_markdown} -o {self.args.dump_pdf}'
pdf_cmd = f"pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {self.args.dump_markdown} -o {self.args.dump_pdf}"
logging.info(pdf_cmd)
p = subprocess.run(pdf_cmd, shell=True)
if p.returncode != 0:
@@ -133,49 +149,61 @@ class Docs():
def build_hugo_content(self):
hugo_root = self.args.build_hugo_content
hugo_images = os.path.join(hugo_root, 'images')
hugo_images = os.path.join(hugo_root, "images")
shutil.rmtree(hugo_root)
os.makedirs(hugo_root)
os.makedirs(hugo_images)
with open(os.path.join(self.script_dir, 'docs', 'hugo_template.md')) as fh:
with open(os.path.join(self.script_dir, "docs", "hugo_template.md")) as fh:
doc_template = fh.read()
with open(os.path.join(self.script_dir, 'docs', 'hugo_index_template.md')) as fh:
with open(
os.path.join(self.script_dir, "docs", "hugo_index_template.md")
) as fh:
index_template = fh.read()
# copy image
shutil.copyfile('pics/tinytapeout_numbered.png', os.path.join(hugo_images, 'tinytapeout-03.png')) # TODO fix hardcoded run
shutil.copyfile(
"pics/tinytapeout_numbered.png",
os.path.join(hugo_images, "tinytapeout-03.png"),
) # TODO fix hardcoded run
# index page
logging.info("building pages - can take a minute as fetching latest GDS action URLs for all projects")
with open(os.path.join(hugo_root, '_index.md'), 'w') as fh:
logging.info(
"building pages - can take a minute as fetching latest GDS action URLs for all projects"
)
with open(os.path.join(hugo_root, "_index.md"), "w") as fh:
fh.write(index_template)
fh.write('# All projects\n')
fh.write("# All projects\n")
fh.write("| Index | Title | Author |\n")
fh.write("| ----- | ----- | -------|\n")
for project in self.projects:
logging.info(project)
fh.write(project.get_hugo_row())
project_dir = os.path.join(hugo_root, f'{project.get_index() :03}')
project_image_dir = os.path.join(project_dir, 'images')
project_dir = os.path.join(hugo_root, f"{project.get_index() :03}")
project_image_dir = os.path.join(project_dir, "images")
os.makedirs(project_dir)
os.makedirs(project_image_dir)
yaml_data = project.get_project_doc_yaml()
yaml_data['index'] = project.index
yaml_data['weight'] = project.index + 1
yaml_data['git_action'] = project.get_latest_action_url()
yaml_data['picture_link'] = ''
if yaml_data['picture']:
picture_name = yaml_data['picture']
yaml_data["index"] = project.index
yaml_data["weight"] = project.index + 1
yaml_data["git_action"] = project.get_latest_action_url()
yaml_data["picture_link"] = ""
if yaml_data["picture"]:
picture_name = yaml_data["picture"]
picture_filename = os.path.join(project.local_dir, picture_name)
picture_basename = os.path.basename(picture_filename)
try:
shutil.copyfile(picture_filename, os.path.join(project_image_dir, picture_basename))
yaml_data['picture_link'] = f'![picture](images/{picture_basename})'
shutil.copyfile(
picture_filename,
os.path.join(project_image_dir, picture_basename),
)
yaml_data[
"picture_link"
] = f"![picture](images/{picture_basename})"
except FileNotFoundError:
yaml_data['picture_link'] = 'Image path is broken'
yaml_data["picture_link"] = "Image path is broken"
doc = doc_template.format(**yaml_data)
with open(os.path.join(project_dir, '_index.md'), 'w') as pfh:
with open(os.path.join(project_dir, "_index.md"), "w") as pfh:
pfh.write(doc)
git_utils.py
+42 -30
View File
@@ -1,12 +1,14 @@
import base64
from urllib.parse import urlparse
import errno
import logging
import requests
from git.repo import Repo
import os
import sys
import typing
import os
import errno
from urllib.parse import urlparse
import requests
from git.repo import Repo
def fetch_file(url: str, filename: str):
logging.info("trying to download {}".format(url))
@@ -15,67 +17,77 @@ def fetch_file(url: str, filename: str):
logging.warning("couldn't download {}".format(url))
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename)
with open(filename, 'wb') as fh:
with open(filename, "wb") as fh:
logging.info("written to {}".format(filename))
fh.write(r.content)
def check_status(r: requests.Response):
if r.status_code == 401:
logging.error("unauthorised, check INFO.md for information about GitHub API keys")
logging.error(
"unauthorised, check INFO.md for information about GitHub API keys"
)
exit(1)
def headers_try_to_add_authorization_from_environment(headers: typing.Dict[str, str]) -> bool:
gh_token = os.getenv('GH_TOKEN', '') # override like gh CLI
def headers_try_to_add_authorization_from_environment(
headers: typing.Dict[str, str]
) -> bool:
gh_token = os.getenv("GH_TOKEN", "") # override like gh CLI
if not gh_token:
gh_token = os.getenv('GITHUB_TOKEN', '') # GHA inherited
gh_token = os.getenv("GITHUB_TOKEN", "") # GHA inherited
if len(gh_token) > 0:
# As per https://docs.github.com/en/rest/overview/authenticating-to-the-rest-api
headers['authorization'] = 'Bearer ' + gh_token
headers["authorization"] = "Bearer " + gh_token
return True
# Use a token instead which is designed to limit exposure of passwords
# I can't find any GH docs explaining use cases for Basic auth and confirming a token
# can be used instead of PASSWORD in the password field of authorization header.
gh_username = os.getenv('GH_USERNAME', '') # override like gh CLI
gh_username = os.getenv("GH_USERNAME", "") # override like gh CLI
if not gh_username:
gh_username = os.getenv('GITHUB_ACTOR', '') # GHA inherited
gh_username = os.getenv("GITHUB_ACTOR", "") # GHA inherited
gh_password = os.getenv('GH_PASSWORD', '')
gh_password = os.getenv("GH_PASSWORD", "")
if len(gh_username) > 0 and len(gh_password) > 0:
auth_string = gh_username + ':' + gh_password
encoded = base64.b64encode(auth_string.encode('ascii'))
headers['authorization'] = 'Basic ' + encoded.decode('ascii')
auth_string = gh_username + ":" + gh_password
encoded = base64.b64encode(auth_string.encode("ascii"))
headers["authorization"] = "Basic " + encoded.decode("ascii")
return True
print("WARNING: No github token found from environment, trying public API requests without, see docs/INFO.md#instructions-to-build-gds", file=sys.stderr)
print(
"WARNING: No github token found from environment, trying public API requests without, see docs/INFO.md#instructions-to-build-gds",
file=sys.stderr,
)
return False
def get_most_recent_action_page(
commits: typing.List[typing.Dict[str, str]],
runs: typing.List[typing.Dict[str, str]]) -> typing.Optional[str]:
release_sha_to_page_url = {run['head_sha']: run['html_url'] for run in runs if run['name'] == 'gds'}
runs: typing.List[typing.Dict[str, str]],
) -> typing.Optional[str]:
release_sha_to_page_url = {
run["head_sha"]: run["html_url"] for run in runs if run["name"] == "gds"
}
for commit in commits:
if commit['sha'] in release_sha_to_page_url:
return release_sha_to_page_url[commit['sha']]
if commit["sha"] in release_sha_to_page_url:
return release_sha_to_page_url[commit["sha"]]
return None
def split_git_url(url: str):
res = urlparse(url)
try:
_, user_name, repo = res.path.split('/')
_, user_name, repo = res.path.split("/")
except ValueError:
logging.error(f"couldn't split repo from {url}")
exit(1)
repo = repo.replace('.git', '')
repo = repo.replace(".git", "")
return user_name, repo
def get_latest_action_url(url: str):
logging.debug(url)
user_name, repo = split_git_url(url)
@@ -87,10 +99,10 @@ def get_latest_action_url(url: str):
headers_try_to_add_authorization_from_environment(headers)
# first fetch the git commit history
api_url = f'https://api.github.com/repos/{user_name}/{repo}/commits'
api_url = f"https://api.github.com/repos/{user_name}/{repo}/commits"
r = requests.get(api_url, headers=headers)
check_status(r)
requests_remaining = int(r.headers['X-RateLimit-Remaining'])
requests_remaining = int(r.headers["X-RateLimit-Remaining"])
if requests_remaining == 0:
logging.error("no API requests remaining")
exit(1)
@@ -98,11 +110,11 @@ def get_latest_action_url(url: str):
commits = r.json()
# get runs
api_url = f'https://api.github.com/repos/{user_name}/{repo}/actions/runs'
r = requests.get(api_url, headers=headers, params={'per_page': 100})
api_url = f"https://api.github.com/repos/{user_name}/{repo}/actions/runs"
r = requests.get(api_url, headers=headers, params={"per_page": 100})
check_status(r)
runs = r.json()
page_url = get_most_recent_action_page(commits, runs['workflow_runs'])
page_url = get_most_recent_action_page(commits, runs["workflow_runs"])
return page_url
markdown_utils.py
+2 -1
View File
@@ -1,4 +1,5 @@
import os
import mistune
from mistune.renderers.markdown import MarkdownRenderer
@@ -25,7 +26,7 @@ class ImagePathRewriterRenderer(MarkdownRenderer):
def image(self, token, state):
url = token["attrs"]["url"]
if not "://" in url and not url.startswith("/"):
if "://" not in url and not url.startswith("/"):
token["attrs"]["url"] = os.path.join(self.prefix, url)
return super().image(token, state)
project.py
+277 -173
View File
@@ -1,25 +1,28 @@
import yaml
import glob
import os
import logging
import re
import subprocess
import gdstk # type: ignore
import cairosvg # type: ignore
import csv
import glob
import json
import git_utils
from git.repo import Repo
import logging
import os
import re
import shutil
import subprocess
import typing
from cells import load_cells, Cell
import cairosvg # type: ignore
import gdstk # type: ignore
import yaml
from git.repo import Repo
import git_utils
from cells import Cell, load_cells
from markdown_utils import limit_markdown_headings
CELL_URL = 'https://skywater-pdk.readthedocs.io/en/main/contents/libraries/sky130_fd_sc_hd/cells/'
CELL_URL = "https://skywater-pdk.readthedocs.io/en/main/contents/libraries/sky130_fd_sc_hd/cells/"
with open(os.path.join(os.path.dirname(__file__), 'tile_sizes.yaml'), 'r') as stream:
tile_sizes = (yaml.safe_load(stream))
with open(os.path.join(os.path.dirname(__file__), "tile_sizes.yaml"), "r") as stream:
tile_sizes = yaml.safe_load(stream)
class Args:
openlane2: bool
@@ -27,19 +30,30 @@ class Args:
print_cell_category: bool
class Project():
class Project:
top_verilog_filename: str
mux_address: int
commit_id: str
sort_id: int
def __init__(self, index: int, git_url: str, local_dir: str, args: Args, is_user_project: bool):
def __init__(
self,
index: int,
git_url: str,
local_dir: str,
args: Args,
is_user_project: bool,
):
self.git_url = git_url
self.args = args
self.index = index
self.local_dir = local_dir
self.is_user_project = is_user_project
self.src_dir = os.path.join(self.local_dir, 'src') if self.is_user_project else self.local_dir
self.src_dir = (
os.path.join(self.local_dir, "src")
if self.is_user_project
else self.local_dir
)
def post_clone_setup(self):
self.load_yaml()
@@ -60,7 +74,7 @@ class Project():
sources = [os.path.join(self.src_dir, src) for src in self.src_files]
source_list = " ".join(sources)
json_file = 'ports.json'
json_file = "ports.json"
yosys_cmd = f"yowasp-yosys -qp 'read_verilog -lib -sv {source_list}; hierarchy -top {top} ; proc; write_json {json_file}'"
p = subprocess.run(yosys_cmd, shell=True)
if p.returncode != 0:
@@ -71,24 +85,26 @@ class Project():
ports = json.load(fh)
os.unlink(json_file)
module_ports = ports['modules'][top]['ports']
module_ports = ports["modules"][top]["ports"]
required_ports = [
['clk', 1],
['ena', 1],
['rst_n', 1],
['ui_in', 8],
['uio_in', 8],
['uio_oe', 8],
['uio_out', 8],
['uo_out', 8],
["clk", 1],
["ena", 1],
["rst_n", 1],
["ui_in", 8],
["uio_in", 8],
["uio_oe", 8],
["uio_out", 8],
["uo_out", 8],
]
for port, bits in required_ports:
if port not in module_ports:
logging.error(f"{self} port '{port}' missing from top module ('{top}')")
exit(1)
actual_bits = len(module_ports[port]['bits'])
actual_bits = len(module_ports[port]["bits"])
if actual_bits != bits:
logging.error(f"{self} incorrect width for port '{port}' in module '{top}': {bits} bits required, {actual_bits} found")
logging.error(
f"{self} incorrect width for port '{port}' in module '{top}': {bits} bits required, {actual_bits} found"
)
exit(1)
def check_num_cells(self):
@@ -109,10 +125,12 @@ class Project():
def load_yaml(self):
try:
with open(os.path.join(self.local_dir, 'info.yaml')) as fh:
with open(os.path.join(self.local_dir, "info.yaml")) as fh:
self.yaml = yaml.safe_load(fh)
except FileNotFoundError:
logging.error(f"yaml file not found for {self} - do you need to --clone the project repos?")
logging.error(
f"yaml file not found for {self} - do you need to --clone the project repos?"
)
exit(1)
# find and save the location of the source files
@@ -120,15 +138,15 @@ class Project():
def setup_source_files(self):
# wokwi_id must be an int or 0
try:
self.wokwi_id = int(self.yaml['project']['wokwi_id'])
self.wokwi_id = int(self.yaml["project"]["wokwi_id"])
except ValueError:
logging.error("wokwi id must be an integer")
exit(1)
self.yaml['project']['git_url'] = self.git_url
self.yaml["project"]["git_url"] = self.git_url
if self.is_hdl():
self.top_module = self.yaml['project']['top_module']
self.top_module = self.yaml["project"]["top_module"]
if self.is_user_project:
self.src_files = self.get_hdl_source()
self.top_verilog_filename = self.find_top_verilog()
@@ -141,24 +159,26 @@ class Project():
if not self.is_user_project:
self.src_files = [self.get_gl_verilog_filename()]
self.unprefixed_name = re.sub('^tt_um_', '', self.top_module)
self.unprefixed_name = re.sub("^tt_um_", "", self.top_module)
if not self.top_module.startswith("tt_um_"):
logging.error(f'top module name must start with "tt_um_" (current value: "{self.top_module}")')
logging.error(
f'top module name must start with "tt_um_" (current value: "{self.top_module}")'
)
exit(1)
self.macro_instance = f"{self.top_module}"
def get_wokwi_source(self):
src_file = "tt_um_wokwi_{}.v".format(self.wokwi_id)
return [src_file, 'cells.v']
return [src_file, "cells.v"]
def get_hdl_source(self):
if 'source_files' not in self.yaml['project']:
if "source_files" not in self.yaml["project"]:
logging.error("source files must be provided if wokwi_id is set to 0")
exit(1)
source_files = self.yaml['project']['source_files']
source_files = self.yaml["project"]["source_files"]
if source_files is None:
logging.error("must be more than 1 source file")
exit(1)
@@ -167,16 +187,18 @@ class Project():
logging.error("must be more than 1 source file")
exit(1)
if 'top_module' not in self.yaml['project']:
if "top_module" not in self.yaml["project"]:
logging.error("must provide a top module name")
exit(1)
if self.yaml['project']['top_module'] == 'top':
logging.error("top module cannot be called top - prepend your repo name to make it unique")
if self.yaml["project"]["top_module"] == "top":
logging.error(
"top module cannot be called top - prepend your repo name to make it unique"
)
exit(1)
for filename in source_files:
if '*' in filename:
if "*" in filename:
logging.error("* not allowed, please specify each file")
exit(1)
if not os.path.exists(os.path.join(self.src_dir, filename)):
@@ -203,16 +225,18 @@ class Project():
def get_project_doc_yaml(self):
# fstring dict support is limited to one level deep, so put the git url and wokwi url in the docs key
docs = self.yaml['documentation']
docs['project_type'] = self.get_project_type_string()
docs['git_url'] = self.git_url
docs = self.yaml["documentation"]
docs["project_type"] = self.get_project_type_string()
docs["git_url"] = self.git_url
# "How it works" and "How to test" may include markdown headings - make sure they don't break the ToC
docs['how_it_works'] = limit_markdown_headings(docs['how_it_works'], min_level=4)
docs['how_to_test'] = limit_markdown_headings(docs['how_to_test'], min_level=4)
docs["how_it_works"] = limit_markdown_headings(
docs["how_it_works"], min_level=4
)
docs["how_to_test"] = limit_markdown_headings(docs["how_to_test"], min_level=4)
return docs
def get_wokwi_url(self):
return f'https://wokwi.com/projects/{self.wokwi_id}'
return f"https://wokwi.com/projects/{self.wokwi_id}"
# top module name is defined in one of the source files, which one?
def find_top_verilog(self):
@@ -225,10 +249,14 @@ class Project():
if match.group(1) == self.top_module:
top_verilog.append(src)
if len(top_verilog) == 0:
logging.error(f"Couldn't find verilog module '{self.top_module}' in any of the project's source files")
logging.error(
f"Couldn't find verilog module '{self.top_module}' in any of the project's source files"
)
exit(1)
if len(top_verilog) > 1:
logging.error(f"Top verilog module '{self.top_module}' found in multiple source files: {', '.join(top_verilog)}")
logging.error(
f"Top verilog module '{self.top_module}' found in multiple source files: {', '.join(top_verilog)}"
)
exit(1)
return top_verilog[0]
@@ -243,11 +271,13 @@ class Project():
return f"{repo.active_branch.name} {repo.commit().hexsha[:8]}"
def get_workflow_url(self):
GITHUB_SERVER_URL = os.getenv('GITHUB_SERVER_URL')
GITHUB_REPOSITORY = os.getenv('GITHUB_REPOSITORY')
GITHUB_RUN_ID = os.getenv('GITHUB_RUN_ID')
GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL")
GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY")
GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID")
if GITHUB_SERVER_URL and GITHUB_REPOSITORY and GITHUB_RUN_ID:
return f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}"
return (
f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}"
)
def __str__(self):
"""
@@ -275,20 +305,26 @@ class Project():
def get_metrics_path(self):
if self.is_user_project:
if self.args.openlane2:
return os.path.join(self.local_dir, 'runs/wokwi/final/metrics.csv')
return os.path.join(self.local_dir, "runs/wokwi/final/metrics.csv")
else:
return os.path.join(self.local_dir, 'runs/wokwi/reports/metrics.csv')
return os.path.join(self.local_dir, "runs/wokwi/reports/metrics.csv")
else:
return os.path.join(self.local_dir, 'metrics.csv')
return os.path.join(self.local_dir, "metrics.csv")
def get_gl_path(self):
if self.is_user_project:
if self.args.openlane2:
return glob.glob(os.path.join(self.local_dir, 'runs/wokwi/final/nl/*.nl.v'))[0]
return glob.glob(
os.path.join(self.local_dir, "runs/wokwi/final/nl/*.nl.v")
)[0]
else:
return glob.glob(os.path.join(self.local_dir, 'runs/wokwi/results/final/verilog/gl/*.nl.v'))[0]
return glob.glob(
os.path.join(
self.local_dir, "runs/wokwi/results/final/verilog/gl/*.nl.v"
)
)[0]
else:
return os.path.join(self.local_dir, f'{self.top_module}.v')
return os.path.join(self.local_dir, f"{self.top_module}.v")
# name of the gds file
def get_macro_gds_filename(self):
@@ -314,7 +350,7 @@ class Project():
# a few people use 'top.v', which is OK as long as the top module is called something more unique
# but then it needs to be made unique so the source can be found
filename = os.path.basename(self.top_verilog_filename)
return f'{self.index :03}_{filename}'
return f"{self.index :03}_{filename}"
else:
return self.top_verilog_filename
@@ -340,35 +376,40 @@ class Project():
url = f"https://wokwi.com/api/projects/{self.wokwi_id}/{truthtable_file}"
try:
git_utils.fetch_file(url, os.path.join(self.src_dir, truthtable_file))
logging.info(f'Wokwi project {self.wokwi_id} has a truthtable included, will test!')
logging.info(
f"Wokwi project {self.wokwi_id} has a truthtable included, will test!"
)
self.install_wokwi_testing()
except FileNotFoundError:
pass
def install_wokwi_testing(self, destination_dir: str = 'src', resource_dir: typing.Optional[str] = None):
def install_wokwi_testing(
self, destination_dir: str = "src", resource_dir: typing.Optional[str] = None
):
if resource_dir is None or not len(resource_dir):
resource_dir = os.path.join(os.path.dirname(__file__), 'testing')
resource_dir = os.path.join(os.path.dirname(__file__), "testing")
# directories in testing/lib to copy over
pyLibsDir = os.path.join(resource_dir, 'lib')
pyLibsDir = os.path.join(resource_dir, "lib")
# src template dir
srcTplDir = os.path.join(resource_dir, 'src-tpl')
srcTplDir = os.path.join(resource_dir, "src-tpl")
for libfullpath in glob.glob(os.path.join(pyLibsDir, '*')):
logging.info(f'Copying {libfullpath} to {destination_dir}')
for libfullpath in glob.glob(os.path.join(pyLibsDir, "*")):
logging.info(f"Copying {libfullpath} to {destination_dir}")
shutil.copytree(
libfullpath,
os.path.join(destination_dir, os.path.basename(libfullpath)),
dirs_exist_ok=True)
dirs_exist_ok=True,
)
for srcTemplate in glob.glob(os.path.join(srcTplDir, '*')):
with open(srcTemplate, 'r') as f:
for srcTemplate in glob.glob(os.path.join(srcTplDir, "*")):
with open(srcTemplate, "r") as f:
contents = f.read()
customizedContents = re.sub('WOKWI_ID', str(self.wokwi_id), contents)
customizedContents = re.sub("WOKWI_ID", str(self.wokwi_id), contents)
outputFname = os.path.basename(srcTemplate)
with open(os.path.join(destination_dir, outputFname), 'w') as outf:
logging.info(f'writing src tpl to {outputFname}')
with open(os.path.join(destination_dir, outputFname), "w") as outf:
logging.info(f"writing src tpl to {outputFname}")
outf.write(customizedContents)
outf.close()
@@ -377,41 +418,45 @@ class Project():
self.fetch_wokwi_files()
self.check_ports()
logging.info("creating include file")
filename = 'user_config.tcl'
with open(os.path.join(self.src_dir, filename), 'w') as fh:
filename = "user_config.tcl"
with open(os.path.join(self.src_dir, filename), "w") as fh:
fh.write("set ::env(DESIGN_NAME) {}\n".format(self.top_module))
fh.write('set ::env(VERILOG_FILES) "\\\n')
for line, source in enumerate(self.src_files):
fh.write(" $::env(DESIGN_DIR)/" + source)
if line != len(self.src_files) - 1:
fh.write(' \\\n')
fh.write(" \\\n")
fh.write('"\n\n')
tiles = self.yaml['project']['tiles']
tiles = self.yaml["project"]["tiles"]
die_area = tile_sizes[tiles]
fh.write(f'# Project area: {tiles} tiles\n')
fh.write(f"# Project area: {tiles} tiles\n")
fh.write(f'set ::env(DIE_AREA) "{die_area}"\n')
fh.write(f'set ::env(FP_DEF_TEMPLATE) "$::env(DESIGN_DIR)/../tt/def/tt_block_{tiles}.def"\n')
fh.write(
f'set ::env(FP_DEF_TEMPLATE) "$::env(DESIGN_DIR)/../tt/def/tt_block_{tiles}.def"\n'
)
def golden_harden(self):
logging.info(f"hardening {self}")
# copy golden config
shutil.copyfile('golden_config.tcl', os.path.join(self.src_dir, 'config.tcl'))
shutil.copyfile("golden_config.tcl", os.path.join(self.src_dir, "config.tcl"))
self.harden()
def copy_picture_for_docs(self):
picture = self.yaml['documentation']['picture']
picture = self.yaml["documentation"]["picture"]
if not picture:
return
extension = os.path.splitext(picture)[1]
supported_extensions = ['.png', '.jpg', '.jpeg', '.svg', '.pdf']
supported_extensions = [".png", ".jpg", ".jpeg", ".svg", ".pdf"]
if not os.path.exists(picture):
logging.warning(f"Picture file '{picture}' not found in repo, skipping")
elif extension not in supported_extensions:
logging.warning(f"Picture file '{picture}' has unsupported extension '{extension}' (we support {', '.join(supported_extensions)}), skipping")
logging.warning(
f"Picture file '{picture}' has unsupported extension '{extension}' (we support {', '.join(supported_extensions)}), skipping"
)
else:
picture_dir = os.path.join(self.local_dir, 'src/__tinytapeout')
picture_dir = os.path.join(self.local_dir, "src/__tinytapeout")
os.makedirs(picture_dir, exist_ok=True)
shutil.copyfile(picture, os.path.join(picture_dir, f'picture{extension}'))
shutil.copyfile(picture, os.path.join(picture_dir, f"picture{extension}"))
def harden(self):
cwd = os.getcwd()
@@ -423,10 +468,14 @@ class Project():
workflow_url = self.get_workflow_url()
if self.args.openlane2:
if not os.path.exists('runs/wokwi'):
print("OpenLane 2 harden not supported yet, please run OpenLane 2 manually")
if not os.path.exists("runs/wokwi"):
print(
"OpenLane 2 harden not supported yet, please run OpenLane 2 manually"
)
exit(1)
print("Writing commit information on top of the existing OpenLane 2 run (runs/wokwi)")
print(
"Writing commit information on top of the existing OpenLane 2 run (runs/wokwi)"
)
else:
# requires PDK, PDK_ROOT, OPENLANE_ROOT & OPENLANE_IMAGE_NAME to be set in local environment
harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK=$PDK -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -tag wokwi"'
@@ -438,16 +487,20 @@ class Project():
exit(1)
# Write commit information
commit_id_json_path = 'runs/wokwi/results/final/commit_id.json'
commit_id_json_path = "runs/wokwi/results/final/commit_id.json"
if self.args.openlane2:
commit_id_json_path = 'runs/wokwi/final/commit_id.json'
with open(os.path.join(self.local_dir, commit_id_json_path), 'w') as f:
json.dump({
commit_id_json_path = "runs/wokwi/final/commit_id.json"
with open(os.path.join(self.local_dir, commit_id_json_path), "w") as f:
json.dump(
{
"app": f"Tiny Tapeout {tt_version}",
"repo": repo,
"commit": commit_hash,
"workflow_url": workflow_url,
}, f, indent=2)
},
f,
indent=2,
)
f.write("\n")
# Copy user provided picture (if exists) to project directory
@@ -460,18 +513,28 @@ class Project():
def check_yaml_docs(self):
yaml = self.yaml
logging.info("checking docs")
for key in ['author', 'title', 'description', 'how_it_works', 'how_to_test', 'language', 'inputs', 'outputs', 'bidirectional']:
if key not in yaml['documentation']:
for key in [
"author",
"title",
"description",
"how_it_works",
"how_to_test",
"language",
"inputs",
"outputs",
"bidirectional",
]:
if key not in yaml["documentation"]:
logging.error("missing key {} in documentation".format(key))
exit(1)
if yaml['documentation'][key] == "":
if yaml["documentation"][key] == "":
logging.error("missing value for {} in documentation".format(key))
exit(1)
# use pandoc to create a single page PDF preview
def create_pdf(self):
yaml = self.yaml
yaml = yaml['documentation']
yaml = yaml["documentation"]
logging.info("creating pdf")
script_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(script_dir, "docs/project_header.md")) as fh:
@@ -479,13 +542,13 @@ class Project():
with open(os.path.join(script_dir, "docs/project_preview.md")) as fh:
doc_template = fh.read()
with open('datasheet.md', 'w') as fh:
with open("datasheet.md", "w") as fh:
fh.write(doc_header)
# handle pictures
yaml['picture_link'] = ''
if yaml['picture']:
picture_name = yaml['picture']
yaml['picture_link'] = '![picture]({})'.format(picture_name)
yaml["picture_link"] = ""
if yaml["picture"]:
picture_name = yaml["picture"]
yaml["picture_link"] = "![picture]({})".format(picture_name)
# now build the doc & print it
try:
@@ -495,7 +558,7 @@ class Project():
except IndexError:
logging.warning("missing pins in info.yaml, skipping")
pdf_cmd = 'pandoc --pdf-engine=xelatex -i datasheet.md -o datasheet.pdf'
pdf_cmd = "pandoc --pdf-engine=xelatex -i datasheet.md -o datasheet.pdf"
logging.info(pdf_cmd)
p = subprocess.run(pdf_cmd, shell=True)
if p.returncode != 0:
@@ -503,9 +566,11 @@ class Project():
# Read and return top-level GDS data from the final GDS file, using gdstk:
def get_final_gds_top_cells(self):
gds = glob.glob(os.path.join(self.local_dir, 'runs/wokwi/results/final/gds/*gds'))
gds = glob.glob(
os.path.join(self.local_dir, "runs/wokwi/results/final/gds/*gds")
)
if self.args.openlane2:
gds = glob.glob(os.path.join(self.local_dir, 'runs/wokwi/final/gds/*gds'))
gds = glob.glob(os.path.join(self.local_dir, "runs/wokwi/final/gds/*gds"))
library = gdstk.read_gds(gds[0])
top_cells = library.top_level()
return top_cells[0]
@@ -514,7 +579,7 @@ class Project():
# NOTE: This includes all standard GDS layers inc. text labels.
def create_svg(self):
top_cells = self.get_final_gds_top_cells()
top_cells.write_svg('gds_render.svg')
top_cells.write_svg("gds_render.svg")
# Try various QUICK methods to create a more-compressed PNG render of the GDS,
# and fall back to cairosvg if it doesn't work. This is designed for speed,
@@ -522,8 +587,7 @@ class Project():
# For more info, see:
# https://github.com/TinyTapeout/tt-gds-action/issues/8
def create_png(self):
logging.info('Loading GDS data...')
logging.info("Loading GDS data...")
top_cells = self.get_final_gds_top_cells()
# Remove label layers (i.e. delete text), then generate SVG:
@@ -537,27 +601,33 @@ class Project():
(71, 5), # 71/5 - met4.label
]
top_cells.filter(sky130_label_layers)
svg = 'gds_render_preview.svg'
logging.info('Rendering SVG without text label layers: {}'.format(svg))
svg = "gds_render_preview.svg"
logging.info("Rendering SVG without text label layers: {}".format(svg))
top_cells.write_svg(svg, pad=0)
# We should now have gds_render_preview.svg
# Try converting using 'rsvg-convert' command-line utility.
# This should create gds_render_preview.png
png = 'gds_render_preview.png'
logging.info('Converting to PNG using rsvg-convert: {}'.format(png))
png = "gds_render_preview.png"
logging.info("Converting to PNG using rsvg-convert: {}".format(png))
cmd = 'rsvg-convert --unlimited {} -o {} --no-keep-image-data'.format(svg, png)
cmd = "rsvg-convert --unlimited {} -o {} --no-keep-image-data".format(svg, png)
logging.debug(cmd)
p = subprocess.run(cmd, shell=True, capture_output=True)
if p.returncode == 127:
logging.warning('rsvg-convert not found; is package "librsvg2-bin" installed? Falling back to cairosvg. This might take a while...')
logging.warning(
'rsvg-convert not found; is package "librsvg2-bin" installed? Falling back to cairosvg. This might take a while...'
)
# Fall back to cairosvg:
cairosvg.svg2png(url=svg, write_to=png)
elif p.returncode != 0 and b'cannot load more than' in p.stderr:
logging.warning('Too many SVG elements ("{}"). Reducing complexity...'.format(p.stderr.decode().strip()))
elif p.returncode != 0 and b"cannot load more than" in p.stderr:
logging.warning(
'Too many SVG elements ("{}"). Reducing complexity...'.format(
p.stderr.decode().strip()
)
)
# Remove more layers that are hardly visible anyway:
sky130_buried_layers = [
(64, 16), # 64/16 - nwell.pin
@@ -572,74 +642,95 @@ class Project():
# (69,20), # 69/20 - met2.drawing
]
top_cells.filter(sky130_buried_layers)
svg_alt = 'gds_render_preview_alt.svg'
logging.info('Rendering SVG with more layers removed: {}'.format(svg_alt))
svg_alt = "gds_render_preview_alt.svg"
logging.info("Rendering SVG with more layers removed: {}".format(svg_alt))
top_cells.write_svg(svg_alt, pad=0)
logging.info('Converting to PNG using rsvg-convert: {}'.format(png))
logging.info("Converting to PNG using rsvg-convert: {}".format(png))
cmd = 'rsvg-convert --unlimited {} -o {} --no-keep-image-data'.format(svg_alt, png)
cmd = "rsvg-convert --unlimited {} -o {} --no-keep-image-data".format(
svg_alt, png
)
logging.debug(cmd)
p = subprocess.run(cmd, shell=True, capture_output=True)
if p.returncode != 0:
logging.warning('Still cannot convert to SVG ("{}"). Falling back to cairosvg. This might take a while...'.format(p.stderr.decode().strip()))
logging.warning(
'Still cannot convert to SVG ("{}"). Falling back to cairosvg. This might take a while...'.format(
p.stderr.decode().strip()
)
)
# Fall back to cairosvg, and since we're doing that, might as well use the original full-detail SVG anyway:
cairosvg.svg2png(url=svg, write_to=png)
# By now we should have gds_render_preview.png
# Compress with pngquant:
final_png = 'gds_render.png'
if self.yaml['project']['tiles'] == '8x2':
quality = '0-10' # Compress more for 8x2 tiles.
final_png = "gds_render.png"
if self.yaml["project"]["tiles"] == "8x2":
quality = "0-10" # Compress more for 8x2 tiles.
else:
quality = '0-30'
logging.info('Compressing PNG further with pngquant to: {}'.format(final_png))
quality = "0-30"
logging.info("Compressing PNG further with pngquant to: {}".format(final_png))
cmd = 'pngquant --quality {} --speed 1 --nofs --strip --force --output {} {}'.format(quality, final_png, png)
cmd = "pngquant --quality {} --speed 1 --nofs --strip --force --output {} {}".format(
quality, final_png, png
)
logging.debug(cmd)
p = subprocess.run(cmd, shell=True, capture_output=True)
if p.returncode == 127:
logging.warning('pngquant not found; is package "pngquant" installed? Using intermediate (uncompressed) PNG file')
logging.warning(
'pngquant not found; is package "pngquant" installed? Using intermediate (uncompressed) PNG file'
)
os.rename(png, final_png)
elif p.returncode != 0:
logging.warning('pngquant error {} ("{}"). Using intermediate (uncompressed) PNG file'.format(p.returncode, p.stderr.decode().strip()))
logging.warning(
'pngquant error {} ("{}"). Using intermediate (uncompressed) PNG file'.format(
p.returncode, p.stderr.decode().strip()
)
)
os.rename(png, final_png)
logging.info('Final PNG is {} ({:,} bytes)'.format(final_png, os.path.getsize(final_png)))
logging.info(
"Final PNG is {} ({:,} bytes)".format(final_png, os.path.getsize(final_png))
)
def print_warnings(self):
warnings: typing.List[str] = []
synth_log = 'runs/wokwi/logs/synthesis/1-synthesis.log'
synth_log = "runs/wokwi/logs/synthesis/1-synthesis.log"
if self.args.openlane2:
synth_log = 'runs/wokwi/05-yosys-synthesis/yosys-synthesis.log'
synth_log = "runs/wokwi/05-yosys-synthesis/yosys-synthesis.log"
with open(os.path.join(self.local_dir, synth_log)) as f:
for line in f.readlines():
if line.startswith('Warning:'):
if line.startswith("Warning:"):
# bogus warning https://github.com/YosysHQ/yosys/commit/bfacaddca8a2e113e4bc3d6177612ccdba1555c8
if 'WIDTHLABEL' not in line:
if "WIDTHLABEL" not in line:
warnings.append(line.strip())
if len(warnings):
print('# Synthesis warnings')
print("# Synthesis warnings")
print()
for warning in warnings:
print(f'* {warning}')
print(f"* {warning}")
def print_stats(self):
print('# Routing stats')
print("# Routing stats")
print()
print('| Utilisation (%) | Wire length (um) |')
print('|-------------|------------------|')
print('| {} | {} |'.format(self.metrics['OpenDP_Util'], self.metrics['wire_length']))
print("| Utilisation (%) | Wire length (um) |")
print("|-------------|------------------|")
print(
"| {} | {} |".format(
self.metrics["OpenDP_Util"], self.metrics["wire_length"]
)
)
# Print the summaries
def summarize(self):
cell_count = self.get_cell_counts_from_gl()
script_dir = os.path.dirname(os.path.realpath(__file__))
Categories = typing.TypedDict("Categories", {'categories': typing.List[str], 'map': typing.Dict[str, int]})
with open(os.path.join(script_dir, 'categories.json')) as fh:
Categories = typing.TypedDict(
"Categories", {"categories": typing.List[str], "map": typing.Dict[str, int]}
)
with open(os.path.join(script_dir, "categories.json")) as fh:
categories: Categories = json.load(fh)
defs = load_cells()
@@ -647,52 +738,65 @@ class Project():
# print all used cells, sorted by frequency
total = 0
if self.args.print_cell_summary:
print('# Cell usage')
print("# Cell usage")
print()
print('| Cell Name | Description | Count |')
print('|-----------|-------------|-------|')
for name, count in sorted(cell_count.items(), key=lambda item: item[1], reverse=True):
print("| Cell Name | Description | Count |")
print("|-----------|-------------|-------|")
for name, count in sorted(
cell_count.items(), key=lambda item: item[1], reverse=True
):
if count > 0:
total += count
cell_link = f'{CELL_URL}{name}'
print(f'| [{name}]({cell_link}) | {defs[name]["description"]} |{count} |')
cell_link = f"{CELL_URL}{name}"
print(
f'| [{name}]({cell_link}) | {defs[name]["description"]} |{count} |'
)
print(f'| | Total | {total} |')
print(f"| | Total | {total} |")
if self.args.print_cell_category:
CategoryInfo = typing.TypedDict("CategoryInfo", {'count': int, 'examples': typing.List[str]})
CategoryInfo = typing.TypedDict(
"CategoryInfo", {"count": int, "examples": typing.List[str]}
)
by_category: typing.Dict[str, CategoryInfo] = {}
total = 0
for cell_name in cell_count:
cat_index = categories['map'][cell_name]
cat_name = categories['categories'][cat_index]
cat_index = categories["map"][cell_name]
cat_name = categories["categories"][cat_index]
if cat_name in by_category:
by_category[cat_name]['count'] += cell_count[cell_name]
by_category[cat_name]['examples'].append(cell_name)
by_category[cat_name]["count"] += cell_count[cell_name]
by_category[cat_name]["examples"].append(cell_name)
else:
by_category[cat_name] = {'count' : cell_count[cell_name], 'examples' : [cell_name]}
by_category[cat_name] = {
"count": cell_count[cell_name],
"examples": [cell_name],
}
if cat_name not in ['Fill', 'Tap']:
if cat_name not in ["Fill", "Tap"]:
total += cell_count[cell_name]
print('# Cell usage by Category')
print("# Cell usage by Category")
print()
print('| Category | Cells | Count |')
print('|---------------|----------|-------|')
for cat_name, cat_dict in sorted(by_category.items(), key=lambda x: x[1]['count'], reverse=True):
cell_links = [f'[{name}]({CELL_URL}{name})' for name in cat_dict['examples']]
print("| Category | Cells | Count |")
print("|---------------|----------|-------|")
for cat_name, cat_dict in sorted(
by_category.items(), key=lambda x: x[1]["count"], reverse=True
):
cell_links = [
f"[{name}]({CELL_URL}{name})" for name in cat_dict["examples"]
]
print(f'|{cat_name} | {" ".join(cell_links)} | {cat_dict["count"]}|')
print(f'## {total} total cells (excluding fill and tap cells)')
print(f"## {total} total cells (excluding fill and tap cells)")
# get cell count from synth report
def get_cell_count_from_synth(self):
num_cells = 0
try:
yosys_report = f'{self.local_dir}/synthesis-stats.txt'
yosys_report = f"{self.local_dir}/synthesis-stats.txt"
with open(yosys_report) as fh:
for line in fh.readlines():
m = re.search(r'Number of cells:\s+(\d+)', line)
m = re.search(r"Number of cells:\s+(\d+)", line)
if m is not None:
num_cells = int(m.group(1))
@@ -707,30 +811,30 @@ class Project():
total = 0
with open(self.get_gl_path()) as fh:
for line in fh.readlines():
m = re.search(r'sky130_(\S+)__(\S+)_(\d+)', line)
m = re.search(r"sky130_(\S+)__(\S+)_(\d+)", line)
if m is not None:
total += 1
cell_lib = m.group(1)
cell_name = m.group(2)
cell_drive = m.group(3)
assert cell_lib in ['fd_sc_hd', 'ef_sc_hd']
assert cell_lib in ["fd_sc_hd", "ef_sc_hd"]
assert int(cell_drive) > 0
try:
cell_count[cell_name] += 1
except KeyError:
cell_count[cell_name] = 1
return (cell_count)
return cell_count
# Load all the json defs and combine into one big dict, keyed by cellname
# Needs access to the PDK
def create_defs(self):
# replace this with a find
json_files = glob.glob('sky130_fd_sc_hd/latest/cells/*/definition.json')
json_files = glob.glob("sky130_fd_sc_hd/latest/cells/*/definition.json")
definitions: typing.Dict[str, Cell] = {}
for json_file in json_files:
with open(json_file) as fh:
definition: Cell = json.load(fh)
definitions[definition['name']] = definition
definitions[definition["name"]] = definition
with open('cells.json', 'w') as fh:
with open("cells.json", "w") as fh:
json.dump(definitions, fh)
rom.py
+6 -2
View File
@@ -1,6 +1,8 @@
from git.repo import Repo
import os
from urllib.parse import urlparse
from git.repo import Repo
from config import Config
MAX_ROM_TEXT_SIZE = 92
@@ -26,9 +28,11 @@ segment_font = {
"t": 0b01111000,
}
def segment_char(c: str):
return segment_font[c]
class ROMFile:
def __init__(self, config: Config):
self.config = config
@@ -53,7 +57,7 @@ class ROMFile:
assert len(rom_text) < MAX_ROM_TEXT_SIZE, "ROM text too long"
rom[0:4] = map(segment_char, self.config['id'])
rom[0:4] = map(segment_char, self.config["id"])
rom[8:16] = map(segment_char, short_sha.upper())
rom[32 : 32 + len(rom_text)] = rom_text.encode("ascii")
shuttle.py
+78 -40
View File
@@ -1,10 +1,12 @@
import glob
import json
import logging
import os
import yaml
import json
import glob
import shutil
import git
import yaml
def copy_print(src: str, dest: str):
os.makedirs(os.path.dirname(dest), exist_ok=True)
@@ -17,8 +19,7 @@ def copy_print_glob(pattern: str, dest_dir: str):
copy_print(file, os.path.join(dest_dir, os.path.basename(file)))
class ShuttleConfig():
class ShuttleConfig:
def __init__(self, config, projects, modules_yaml_name: str):
self.config = config
self.projects = projects
@@ -26,21 +27,27 @@ class ShuttleConfig():
self.modules_yaml_name = modules_yaml_name
def configure_mux(self):
with open(self.modules_yaml_name, 'r') as modules_file:
with open(self.modules_yaml_name, "r") as modules_file:
module_config = yaml.safe_load(modules_file)
configured_macros = set(map(lambda mod: mod['name'], module_config['modules']))
logging.info(f"found {len(configured_macros)} preconfigured macros: {configured_macros}")
configured_macros = set(
map(lambda mod: mod["name"], module_config["modules"])
)
logging.info(
f"found {len(configured_macros)} preconfigured macros: {configured_macros}"
)
for project in self.projects:
tiles = project.yaml['project']['tiles']
width, height = map(int, tiles.split('x'))
tiles = project.yaml["project"]["tiles"]
width, height = map(int, tiles.split("x"))
if project.unprefixed_name not in configured_macros:
module_config['modules'].append({
'name': project.unprefixed_name,
'width': width,
'height': height,
})
module_config["modules"].append(
{
"name": project.unprefixed_name,
"width": width,
"height": height,
}
)
with open('tt-multiplexer/cfg/modules.yaml', 'w') as mux_modules_file:
with open("tt-multiplexer/cfg/modules.yaml", "w") as mux_modules_file:
yaml.dump(module_config, mux_modules_file)
res = os.system("make -C tt-multiplexer gensrc")
@@ -50,17 +57,17 @@ class ShuttleConfig():
mux_index = {}
mux_index_reverse = {}
with open('tt-multiplexer/cfg/modules_placed.yaml') as placed_modules_file:
with open("tt-multiplexer/cfg/modules_placed.yaml") as placed_modules_file:
placed_modules = yaml.safe_load(placed_modules_file)
for module in placed_modules['modules']:
mux_address = (module['y'] << 5) + module['x']
module_name = 'tt_um_' + module['name']
for module in placed_modules["modules"]:
mux_address = (module["y"] << 5) + module["x"]
module_name = "tt_um_" + module["name"]
project = next(p for p in self.projects if p.top_module == module_name)
project.mux_address = mux_address
mux_index[mux_address] = {
"macro": module_name,
"x": module['x'],
"y": module['y'],
"x": module["x"],
"y": module["y"],
"tiles": f"{module['width']}x{module['height']}",
"repo": project.git_url,
"commit": project.commit_id,
@@ -83,20 +90,23 @@ class ShuttleConfig():
"mux": mux_index,
}
with open('shuttle_index.json', 'w') as shuttle_index_file:
with open("shuttle_index.json", "w") as shuttle_index_file:
json.dump(shuttle_index_data, shuttle_index_file, indent=2)
def list(self):
for project in self.projects:
logging.info(project)
def find_last_run(self, macro: str):
runs = f"tt-multiplexer/ol2/{macro}/runs/"
if macro == "tt_um_chip_rom":
runs = "tt/rom/runs/"
runlist = sorted(
[r for r in os.listdir(runs) if r.startswith("RUN_") and os.path.isdir(os.path.join(runs, r))]
[
r
for r in os.listdir(runs)
if r.startswith("RUN_") and os.path.isdir(os.path.join(runs, r))
]
)
if len(runlist) == 0:
print(f"Error: no runs found for {macro}")
@@ -104,25 +114,34 @@ class ShuttleConfig():
return os.path.join(runs, runlist[-1])
def copy_macros(self):
logging.info('copying macros to tt_top:')
logging.info("copying macros to tt_top:")
copy_print_glob("projects/*/*.gds", "tt-multiplexer/ol2/tt_top/gds")
copy_print_glob("projects/*/*.lef", "tt-multiplexer/ol2/tt_top/lef")
copy_print_glob("projects/*/*.v", "tt-multiplexer/ol2/tt_top/verilog")
macros = ["tt_um_chip_rom", "tt_ctrl", "tt_mux"]
for macro in macros:
lastrun = self.find_last_run(macro)
copy_print(f"{lastrun}/final/gds/{macro}.gds", f"tt-multiplexer/ol2/tt_top/gds/{macro}.gds")
copy_print(f"{lastrun}/final/lef/{macro}.lef", f"tt-multiplexer/ol2/tt_top/lef/{macro}.lef")
copy_print(f"{lastrun}/final/nl/{macro}.nl.v", f"tt-multiplexer/ol2/tt_top/verilog/{macro}.v")
copy_print_glob(f"{lastrun}/final/spef/*/*.spef", "tt-multiplexer/ol2/tt_top/spef")
copy_print(
f"{lastrun}/final/gds/{macro}.gds",
f"tt-multiplexer/ol2/tt_top/gds/{macro}.gds",
)
copy_print(
f"{lastrun}/final/lef/{macro}.lef",
f"tt-multiplexer/ol2/tt_top/lef/{macro}.lef",
)
copy_print(
f"{lastrun}/final/nl/{macro}.nl.v",
f"tt-multiplexer/ol2/tt_top/verilog/{macro}.v",
)
copy_print_glob(
f"{lastrun}/final/spef/*/*.spef", "tt-multiplexer/ol2/tt_top/spef"
)
def copy_final_results(self):
macros = ["tt_um_chip_rom", "tt_ctrl", "tt_mux", "tt_top"]
logging.info('copying final results:')
logging.info("copying final results:")
for macro in macros:
lastrun = self.find_last_run(macro)
macro_name = macro if macro != "tt_top" else "user_project_wrapper"
@@ -130,12 +149,22 @@ class ShuttleConfig():
logging.info(f" FROM {lastrun}")
copy_print(f"{lastrun}/final/gds/{macro_name}.gds", f"gds/{macro_name}.gds")
copy_print(f"{lastrun}/final/lef/{macro_name}.lef", f"lef/{macro_name}.lef")
copy_print(f"{lastrun}/final/pnl/{macro_name}.pnl.v", f"verilog/gl/{macro_name}.v")
copy_print(f"{lastrun}/final/nl/{macro_name}.nl.v", f"verilog/gl/{macro_name}.nl.v")
copy_print(
f"{lastrun}/final/pnl/{macro_name}.pnl.v", f"verilog/gl/{macro_name}.v"
)
copy_print(
f"{lastrun}/final/nl/{macro_name}.nl.v", f"verilog/gl/{macro_name}.nl.v"
)
shutil.copytree(
f"{lastrun}/final/spef", f"spef/", copy_function=copy_print, dirs_exist_ok=True
f"{lastrun}/final/spef",
f"spef/",
copy_function=copy_print,
dirs_exist_ok=True,
)
copy_print(
f"{lastrun}/final/spef/nom_/{macro_name}.nom_.spef",
f"spef/{macro_name}.spef",
)
copy_print(f"{lastrun}/final/spef/nom_/{macro_name}.nom_.spef", f"spef/{macro_name}.spef")
def create_efabless_submission(self):
logging.info("creating efabless submission directory:")
@@ -143,6 +172,15 @@ class ShuttleConfig():
copy_print("README.md", "efabless/README.md")
copy_print("shuttle_index.json", "efabless/shuttle_index.json")
copy_print("verilog/rtl/user_defines.v", "efabless/verilog/rtl/user_defines.v")
copy_print(f"{lastrun}/final/pnl/user_project_wrapper.pnl.v", f"efabless/verilog/gl/user_project_wrapper.v")
copy_print(f"{lastrun}/final/gds/user_project_wrapper.gds", f"efabless/gds/user_project_wrapper.gds")
copy_print(f"{lastrun}/final/lef/user_project_wrapper.lef", f"efabless/lef/user_project_wrapper.lef")
copy_print(
f"{lastrun}/final/pnl/user_project_wrapper.pnl.v",
f"efabless/verilog/gl/user_project_wrapper.v",
)
copy_print(
f"{lastrun}/final/gds/user_project_wrapper.gds",
f"efabless/gds/user_project_wrapper.gds",
)
copy_print(
f"{lastrun}/final/lef/user_project_wrapper.lef",
f"efabless/lef/user_project_wrapper.lef",
)
testing/lib/testutils/truthtable.py
+101 -118
View File
@@ -1,4 +1,4 @@
'''
"""
TruthTable testing
Copyright (C) 2023 Pat Deegan, https://psychogenic.com
@@ -38,13 +38,14 @@
SPDX-FileCopyrightText: © 2023 Pat Deegan, https://psychogenic.com
SPDX-License-Identifier: Apache2.0
'''
"""
from string import Template as StringTemplate
from cocotb.triggers import Timer
from cocotb.binary import BinaryValue
import re
import logging
import re
from string import Template as StringTemplate
from cocotb.binary import BinaryValue
from cocotb.triggers import Timer
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
@@ -57,37 +58,35 @@ class SaneBinaryValue(BinaryValue):
self.hardBit = [True] * numbits
for i in range(numbits):
c = vstr[i]
if c != '1' and c != '0':
if c != "1" and c != "0":
self.hardBit[(numbits - 1) - i] = False
class TruthMapping:
def __init__(self, resultingValue: str):
self.result = SaneBinaryValue(resultingValue)
def __str__(self):
return f'Expecting {self.result.binstr}'
return f"Expecting {self.result.binstr}"
class TruthTable:
'''
"""
Holds the sequence/expected mapping of inputs -> outputs
and provides an async testAll() to perform the testing.
'''
def __init__(self, interStepDelay:int=10, interStepTimeUnits:str='ns'):
"""
def __init__(self, interStepDelay: int = 10, interStepTimeUnits: str = "ns"):
self.mappings = []
self.stepDelayTime = interStepDelay
self.stepDelayUnits = interStepTimeUnits
def addMapping(self, relationship: TruthMapping):
self.mappings.append(relationship)
def getMapping(self, idx: int) -> TruthMapping:
if idx >= len(self.mappings):
raise IndexError('out of bounds on mapping')
raise IndexError("out of bounds on mapping")
return self.mappings[idx]
@@ -105,27 +104,29 @@ class TruthTable:
retStrs = [f"Truthtable with {numsteps} mappings"]
for i in range(numsteps):
step = self[i]
retStrs.append(f'{i + 1}\t{str(step)}')
retStrs.append(f"{i + 1}\t{str(step)}")
return '\n'.join(retStrs)
return "\n".join(retStrs)
def dump(self):
print(str(self))
async def testAll(self, i_bus, o_bus, logger=None):
for i in range(len(self)):
i_bus.value = self[i].state
await Timer(self.stepDelayTime, units=self.stepDelayUnits) # wait a tad
logger.debug(f'Set i_bus to {i_bus.value} vs {self[i].state}')
logger.debug(f"Set i_bus to {i_bus.value} vs {self[i].state}")
for idx in range(8):
logger.debug(f'{i}-{idx}: i_bus[{idx}] == {i_bus[idx]} == {self[i].state[idx]}')
logger.debug(
f"{i}-{idx}: i_bus[{idx}] == {i_bus[idx]} == {self[i].state[idx]}"
)
expectedResult = self[i].result
if logger is not None:
logger.info(f'State {i}, setting input to {self[i].state}, expecting {expectedResult} (got {o_bus.value})')
logger.info(
f"State {i}, setting input to {self[i].state}, expecting {expectedResult} (got {o_bus.value})"
)
# this is so stupid that it's probably wrong... there _must_ be
# a good way to do this, otherwise _what_ is the point of having
@@ -145,60 +146,58 @@ class OneToOneTruthMapping(TruthMapping):
super().__init__(output)
self.state = SaneBinaryValue(state)
def __str__(self):
return f'Set {self.state.binstr}, expect {self.result.binstr}'
return f"Set {self.state.binstr}, expect {self.result.binstr}"
class TestTableEntryParser:
'''
"""
A little class that validates and cleans up
input and output specifications.
'''
"""
def __init__(self, logger):
self.log = logger
self.optionalPrefix = r"(\d'|0b)?"
self.validInputValues = r'(\s*[01tTcCxX-]\s*){8}' # 0/1 hard-coded, t/T toggle, c/C clocking, x or - unchanged
self.validOutputValues = r'(\s*[01xX-]\s*){8}' # 0/1 hard-coded,x or - don't care
self.validInputValues = r"(\s*[01tTcCxX-]\s*){8}" # 0/1 hard-coded, t/T toggle, c/C clocking, x or - unchanged
self.validOutputValues = (
r"(\s*[01xX-]\s*){8}" # 0/1 hard-coded,x or - don't care
)
valueMatchReTpl = StringTemplate(r'\s*$prefix($valmatch)')
valueMatchReTpl = StringTemplate(r"\s*$prefix($valmatch)")
self.inputValueValidatorRe = re.compile(
valueMatchReTpl.substitute(
prefix=self.optionalPrefix, valmatch=self.validInputValues))
prefix=self.optionalPrefix, valmatch=self.validInputValues
)
)
self.outputValueValidatorRe = re.compile(
valueMatchReTpl.substitute(
prefix=self.optionalPrefix, valmatch=self.validOutputValues))
prefix=self.optionalPrefix, valmatch=self.validOutputValues
)
)
def validateAndClean(self, val: str, usingRegex):
if val is None or not len(val):
return None
mtch = usingRegex.match(val)
if mtch is None:
vcln = val.strip()
if not len(vcln):
# just whitespace
return None
if vcln.find('#') >= 0 or vcln.find('//') >=0:
if vcln.find("#") >= 0 or vcln.find("//") >= 0:
# just a comment
return None
if vcln.find('1') >=0 or vcln.find('0') >=0 or vcln.find('x') >= 0:
if vcln.find("1") >= 0 or vcln.find("0") >= 0 or vcln.find("x") >= 0:
self.log.error(f"Bad value passed ? '{val}'")
return None
return re.sub(r'\s+', '', mtch.group(0)).lower()
return re.sub(r"\s+", "", mtch.group(0)).lower()
def inputFrom(self, val: str):
return self.validateAndClean(val, self.inputValueValidatorRe)
@@ -210,7 +209,7 @@ class TestTableEntryParser:
class TestTableParser:
'''
"""
Base class for tabular configuration values for input sequence and
expected outputs.
@@ -220,7 +219,7 @@ class TestTableParser:
Basically does everything except handle the particularities of a
give table format (e.g. markdown).
'''
"""
def __init__(self, logger=None, numIOBits: int = 8):
if logger is None:
@@ -229,16 +228,14 @@ class TestTableParser:
self.numIOBits = numIOBits
self.valueParser = TestTableEntryParser(self.log)
self._truthTable = None
self._textTable = []
self._currentInput = SaneBinaryValue('0'*numIOBits)
self._ignoreOutput = SaneBinaryValue('-'*numIOBits)
self._currentInput = SaneBinaryValue("0" * numIOBits)
self._ignoreOutput = SaneBinaryValue("-" * numIOBits)
def reset(self):
self._currentInput = SaneBinaryValue('00000000')
self._currentInput = SaneBinaryValue("00000000")
self._truthTable = None
self._textTable = []
@@ -249,18 +246,14 @@ class TestTableParser:
return self._truthTable
def _inputStateFromBits(self, inputBits):
newBits = ['0'] * self.numIOBits
newBits = ["0"] * self.numIOBits
for i in range(self.numIOBits):
if inputBits[i] == '0' or inputBits[i] == '1':
if inputBits[i] == "0" or inputBits[i] == "1":
newBits[i] = inputBits[i]
self._currentInput[i] = int(inputBits[i])
elif inputBits[i] == 't':
elif inputBits[i] == "t":
# toggled
self._currentInput[i] = ~self._currentInput[i]
newBits[i] = str(self._currentInput[i])
@@ -268,58 +261,57 @@ class TestTableParser:
# unchanged
newBits[i] = str(self._currentInput[i])
return "".join(newBits)
return ''.join(newBits)
def addStep(self, inputValue:str, outputValue:str=''):
def addStep(self, inputValue: str, outputValue: str = ""):
inputBits = self.valueParser.inputFrom(inputValue)
if inputBits is None or not len(inputBits):
self.log.info(f'Unable to parse input bits: {inputValue}')
self.log.info(f"Unable to parse input bits: {inputValue}")
return False
if inputBits.find('c') >= 0:
if inputBits.find("c") >= 0:
# clocking...
setupInputs = inputBits.replace('c', 'x')
self.log.debug(f'Have clocking in inputs {inputBits}, adding setup step {setupInputs}')
setupInputs = inputBits.replace("c", "x")
self.log.debug(
f"Have clocking in inputs {inputBits}, adding setup step {setupInputs}"
)
self.addStep(setupInputs) # setup
# clock edge 1
clockEdges = inputBits.replace('t', 'x') # don't toggle twice!
clockEdges = inputBits.replace("t", "x") # don't toggle twice!
clockInAndHold = clockEdges.replace('c', 't')
clockInAndHold = clockEdges.replace("c", "t")
self.log.debug(f'Now clock {clockEdges} x2')
self.log.debug(f"Now clock {clockEdges} x2")
self.addStep(clockInAndHold) # clock in
# clock edge 2
self.addStep(clockInAndHold) # clock out
inputBinVal = SaneBinaryValue(self._inputStateFromBits(inputBits))
outputBinVal = self._ignoreOutput
outputBits = ''
outputBits = ""
if outputValue is not None and len(outputValue):
self.log.info(f'outval is "{outputValue}"')
outputBits = self.valueParser.outputFrom(outputValue)
if outputBits is None or not len(outputBits):
self.log.info(f'Unable to parse output bits: {outputBits}')
self.log.info(f"Unable to parse output bits: {outputBits}")
return False
outputBinVal = SaneBinaryValue(outputBits)
self.log.info(
f"Mapped {inputBits} -> {outputBits} ({inputBinVal} -> {outputBinVal})"
)
self.log.info(f'Mapped {inputBits} -> {outputBits} ({inputBinVal} -> {outputBinVal})')
self.truthTable.addMapping(OneToOneTruthMapping(inputBinVal.binstr, outputBinVal.binstr))
self.truthTable.addMapping(
OneToOneTruthMapping(inputBinVal.binstr, outputBinVal.binstr)
)
self._textTable.append((inputBits, outputBits))
return True
@@ -328,33 +320,30 @@ class TestTableParser:
self.log.error("Override generateFrom() in subclass")
return None
def __str__(self):
retStrs = [f'Truth table parser with {len(self._textTable)} steps added.']
retStrs = [f"Truth table parser with {len(self._textTable)} steps added."]
for i in range(len(self._textTable)):
step = self._textTable[i]
if step[1] is not None and len(step[1]):
retStrs.append(f'{i+1}\t{step[0]} -> {step[1]}')
retStrs.append(f"{i+1}\t{step[0]} -> {step[1]}")
else:
retStrs.append(f'{i+1}\t{step[0]}')
retStrs.append(f"{i+1}\t{step[0]}")
return '\n'.join(retStrs)
return "\n".join(retStrs)
def dump(self):
print(str(self))
class MarkdownTestTableParser(TestTableParser):
'''
"""
A specialization of the TestTableParser that handles the
particulars of extracting test inputs/outputs from a markdown
table.
'''
"""
def __init__(self, columnSeparatorRegex:str=r'\|', logger=None):
'''
def __init__(self, columnSeparatorRegex: str = r"\|", logger=None):
"""
We want to support markdown tables.
These may have some header row, followed by a separator (which may
or may not include aligment info, i.e. ':'), finally followed
@@ -377,48 +366,49 @@ class MarkdownTestTableParser(TestTableParser):
...
'''
"""
super().__init__(logger)
self.columnSeperatorRe = columnSeparatorRegex
eol = r'(\r\n|\r|\n)'
eol = r"(\r\n|\r|\n)"
# the |-------|----- ... line
tableHeaderSepTpl = StringTemplate(r'\s*($sep\s*:?-{1,}:?\s*){2,}$sep$eol')
self.headerSepRe = re.compile(tableHeaderSepTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M)
tableHeaderSepTpl = StringTemplate(r"\s*($sep\s*:?-{1,}:?\s*){2,}$sep$eol")
self.headerSepRe = re.compile(
tableHeaderSepTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M
)
# each content line thereafter
# | CONTENT | CONTENT |
contentRowTpl = StringTemplate(r'^\s*(($sep\s*[^$sep\r\n]+){1,})$sep$eol')
self.contentRowRe = re.compile(contentRowTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M)
contentRowTpl = StringTemplate(r"^\s*(($sep\s*[^$sep\r\n]+){1,})$sep$eol")
self.contentRowRe = re.compile(
contentRowTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M
)
self.rowItemSplitterRe = re.compile(self.columnSeperatorRe)
def parseMarkdownTable(self, contents: str):
headerSepSearch = self.headerSepRe.search(contents)
if headerSepSearch is not None:
# did have a header
# dump it, and everything before it
headerSpan = headerSepSearch.span()
headerEndPos = headerSpan[1]
self.log.debug(f"Dumping header to {headerEndPos}\n{contents[:headerEndPos]}")
self.log.debug(
f"Dumping header to {headerEndPos}\n{contents[:headerEndPos]}"
)
contents = contents[headerEndPos:]
# extract valid rows in first pass
self.log.debug(f"Content row RE is {self.contentRowRe}")
contentRowMatches = self.contentRowRe.findall(contents)
if contentRowMatches is None or not len(contentRowMatches):
self.log.error('No valid rows found in markdown table')
self.log.error("No valid rows found in markdown table")
return False
numSteps = 0
for aRow in contentRowMatches:
self.log.debug(f'Matched row {aRow}')
self.log.debug(f"Matched row {aRow}")
entries = self.rowItemSplitterRe.split(aRow[0])
# note, we have entries like
@@ -440,14 +430,9 @@ class MarkdownTestTableParser(TestTableParser):
if self.addStep(inVal, outVal):
numSteps += 1
return numSteps
def generateFrom(self, contents: str):
self.reset()
tableParsed = self.parseMarkdownTable(contents)
@@ -459,15 +444,11 @@ class MarkdownTestTableParser(TestTableParser):
return self.truthTable
def parseSimpleTable(contents: str):
tt = TruthTable()
m = re.compile(r'''^\s*\d'(\d+)\s*\|\s*\d'([zZxX\d]+)''', re.M)
m = re.compile(r"""^\s*\d'(\d+)\s*\|\s*\d'([zZxX\d]+)""", re.M)
for match in m.findall(contents):
if match[1] != 'x':
if match[1] != "x":
# we skip anything where all results are don't care
# result = match[1].replace('x', '-') # now using the BinaryValue override
result = match[1]
@@ -475,6 +456,7 @@ def parseSimpleTable(contents:str):
return tt
def parseMarkdownTable(contents: str, logger=None):
m = MarkdownTestTableParser(logger=logger)
table = m.generateFrom(contents)
@@ -482,7 +464,7 @@ def parseMarkdownTable(contents:str, logger=None):
def loadSimpleTruthTable(filepath: str):
with open(filepath, 'r') as f:
with open(filepath, "r") as f:
contents = f.read()
return parseSimpleTable(contents)
@@ -490,13 +472,14 @@ def loadSimpleTruthTable(filepath:str):
def loadMarkdownTruthTable(filepath: str, logger=None):
with open(filepath, 'r') as f:
with open(filepath, "r") as f:
contents = f.read()
return parseMarkdownTable(contents, logger)
return None
TruthTableExample = '''
TruthTableExample = """
\\ui_in | \\uo_out
---------- | -----------
8'11111000 | 8'xxxxxxx0
@@ -508,10 +491,10 @@ TruthTableExample = '''
8'11111110 | 8'xxxxxxx1
8'11111111 | 8'xxxxxxx0
'''
"""
# with comments and spacing
TruthTableMarkdownExample = '''
TruthTableMarkdownExample = """
|IN: CBA RC | output | comment |
|--------------|--------------|-----------|
| 000 000 00 | -- ----- - | init |
@@ -528,7 +511,7 @@ TruthTableMarkdownExample = '''
| --- 110 -c | -- 11111 - | success |
| --- 000 -c | -- 11111 - | still open|
| --- --- 1c | -- 11100 - | reset |
'''
"""
if __name__ == "__main__":
m = MarkdownTestTableParser()
table = m.generateFrom(TruthTableMarkdownExample)
testing/src-tpl/test.py
+7 -8
View File
@@ -1,28 +1,27 @@
'''
"""
Wokwi TinyTapeout truth-table based auto test sequence.
SPDX-FileCopyrightText: © 2023 Pat Deegan, https://psychogenic.com
SPDX-License-Identifier: Apache2.0
'''
"""
import os
import cocotb
import testutils.truthtable as truthtable
@cocotb.test()
async def truthTableCompare(parentDUT):
'''
"""
truthTableCompare -- loads markdown truth table and, if load succeeded, actually performs the tests.
'''
"""
usermodule = parentDUT.dut
i_bus = parentDUT.ui_in
o_bus = parentDUT.uo_out
tt = truthtable.loadMarkdownTruthTable('truthtable.md', usermodule._log)
tt = truthtable.loadMarkdownTruthTable("truthtable.md", usermodule._log)
if tt is None:
usermodule._log.info('No truth table loaded, no table compare test to run')
usermodule._log.info("No truth table loaded, no table compare test to run")
return
usermodule._log.debug(str(tt))
await tt.testAll(i_bus, o_bus, usermodule._log)
tt_annotate.py
+16 -10
View File
@@ -2,8 +2,8 @@
from PIL import Image, ImageDraw, ImageFont
img = Image.open('pics/tinytapeout.png').convert('RGBA')
overlay = Image.new('RGBA', img.size, (0, 0, 0, 0))
img = Image.open("pics/tinytapeout.png").convert("RGBA")
overlay = Image.new("RGBA", img.size, (0, 0, 0, 0))
draw = ImageDraw.Draw(overlay)
font = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 30)
@@ -12,21 +12,27 @@ xmap = ((88, 272), (2676, 1646)) # two pairs of (chip x coord, picture x coord)
ymap = ((95, 1879), (3240, 213)) # two pairs of (chip y coord, picture y coord)
xshift, yshift = 40, -43 # text position relative to macro origin
for line in open('openlane/user_project_wrapper/macro.cfg'):
for line in open("openlane/user_project_wrapper/macro.cfg"):
macro, x, y, orient = line.split()
if macro == 'scan_controller':
if macro == "scan_controller":
continue
elif macro.startswith('scanchain_'):
elif macro.startswith("scanchain_"):
continue
name, num = macro.rsplit('_', 1)
name, num = macro.rsplit("_", 1)
num, x, y = int(num), int(x), int(y)
x = int((x-xmap[0][0])/(xmap[1][0]-xmap[0][0])*(xmap[1][1]-xmap[0][1])+xmap[0][1])
y = int((y-ymap[0][0])/(ymap[1][0]-ymap[0][0])*(ymap[1][1]-ymap[0][1])+ymap[0][1])
x = int(
(x - xmap[0][0]) / (xmap[1][0] - xmap[0][0]) * (xmap[1][1] - xmap[0][1])
+ xmap[0][1]
)
y = int(
(y - ymap[0][0]) / (ymap[1][0] - ymap[0][0]) * (ymap[1][1] - ymap[0][1])
+ ymap[0][1]
)
x += xshift
y += yshift
msg = str(num)
w, h = draw.textsize(msg, font=font)
draw.text((x - w / 2, y - h / 2), msg, fill=(0, 0, 0, 255), font=font)
combined = Image.alpha_composite(img, overlay).convert('RGB')
combined.save('pics/tinytapeout_numbered.png')
combined = Image.alpha_composite(img, overlay).convert("RGB")
combined.save("pics/tinytapeout_numbered.png")
tt_tool.py
+93 -21
View File
@@ -1,39 +1,111 @@
#!/usr/bin/env python3
import sys
import logging
import argparse
import logging
import sys
from project import Project
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="TT setup")
parser.add_argument('--project-dir', help="location of the project", default='.')
parser.add_argument('--yaml', help="the project's yaml configuration file", default='info.yaml')
parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO)
parser.add_argument('--openlane2', help="use openlane 2", action="store_const", const=True, default=False)
parser.add_argument("--project-dir", help="location of the project", default=".")
parser.add_argument(
"--yaml", help="the project's yaml configuration file", default="info.yaml"
)
parser.add_argument(
"--debug",
help="debug logging",
action="store_const",
dest="loglevel",
const=logging.DEBUG,
default=logging.INFO,
)
parser.add_argument(
"--openlane2",
help="use openlane 2",
action="store_const",
const=True,
default=False,
)
# reports & summaries
parser.add_argument('--print-cell-summary', help="print summary", action="store_const", const=True, default=False)
parser.add_argument('--print-cell-category', help="print category", action="store_const", const=True, default=False)
parser.add_argument('--print-stats', help="print some stats from the run", action="store_const", const=True)
parser.add_argument('--print-warnings', help="print any warnings", action="store_const", const=True)
parser.add_argument('--print-wokwi-id', help="prints the Wokwi project id", action="store_const", const=True)
parser.add_argument(
"--print-cell-summary",
help="print summary",
action="store_const",
const=True,
default=False,
)
parser.add_argument(
"--print-cell-category",
help="print category",
action="store_const",
const=True,
default=False,
)
parser.add_argument(
"--print-stats",
help="print some stats from the run",
action="store_const",
const=True,
)
parser.add_argument(
"--print-warnings", help="print any warnings", action="store_const", const=True
)
parser.add_argument(
"--print-wokwi-id",
help="prints the Wokwi project id",
action="store_const",
const=True,
)
# documentation
parser.add_argument('--check-docs', help="check the documentation part of the yaml", action="store_const", const=True)
parser.add_argument('--create-pdf', help="create a single page PDF", action="store_const", const=True)
parser.add_argument('--create-svg', help="create a svg of the GDS layout", action="store_const", const=True)
parser.add_argument('--create-png', help="create a png of the GDS layout", action="store_const", const=True)
parser.add_argument(
"--check-docs",
help="check the documentation part of the yaml",
action="store_const",
const=True,
)
parser.add_argument(
"--create-pdf",
help="create a single page PDF",
action="store_const",
const=True,
)
parser.add_argument(
"--create-svg",
help="create a svg of the GDS layout",
action="store_const",
const=True,
)
parser.add_argument(
"--create-png",
help="create a png of the GDS layout",
action="store_const",
const=True,
)
# configure
parser.add_argument('--create-user-config', help="create the user_config.tcl file with top module and source files", action="store_const", const=True)
parser.add_argument('--harden', help="use a local OpenLane install to harden the project", action="store_const", const=True)
parser.add_argument(
"--create-user-config",
help="create the user_config.tcl file with top module and source files",
action="store_const",
const=True,
)
parser.add_argument(
"--harden",
help="use a local OpenLane install to harden the project",
action="store_const",
const=True,
)
args = parser.parse_args()
# setup log
log_format = logging.Formatter('%(asctime)s - %(module)-10s - %(levelname)-8s - %(message)s')
log_format = logging.Formatter(
"%(asctime)s - %(module)-10s - %(levelname)-8s - %(message)s"
)
# configure the client logging
log = logging.getLogger('')
log = logging.getLogger("")
# has to be set to debug as is the root logger
log.setLevel(args.loglevel)
@@ -43,7 +115,7 @@ if __name__ == '__main__':
ch.setFormatter(log_format)
log.addHandler(ch)
project = Project(0, 'unknown', args.project_dir, args, is_user_project=True)
project = Project(0, "unknown", args.project_dir, args, is_user_project=True)
project.post_clone_setup()
# handle the options