diff --git a/cells.py b/cells.py index f2e5a89..8bfb2bf 100644 --- a/cells.py +++ b/cells.py @@ -1,6 +1,7 @@ -import os -from typing import List, Dict, TypedDict, Optional import json +import os +from typing import Dict, List, Optional, TypedDict + class Port(TypedDict): kind: str @@ -8,6 +9,7 @@ class Port(TypedDict): direction: str description: str + class Cell(TypedDict): description: str file_prefix: str @@ -19,10 +21,11 @@ class Cell(TypedDict): verilog_name: str equation: Optional[str] + Cells = Dict[str, Cell] -def load_cells() -> Cells: - script_dir = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(script_dir, 'cells.json')) as fh: - return json.load(fh) +def load_cells() -> Cells: + script_dir = os.path.dirname(os.path.realpath(__file__)) + with open(os.path.join(script_dir, "cells.json")) as fh: + return json.load(fh) diff --git a/config.py b/config.py index f8ebc19..6ded8ec 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ from typing import TypedDict + class Config(TypedDict): """TypedDict for Tiny Tapeout's config.yaml file.""" diff --git a/configure.py b/configure.py index 9c5bd20..7e8ac79 100755 --- a/configure.py +++ b/configure.py @@ -1,51 +1,62 @@ #!/usr/bin/env python3 +import argparse +import collections import datetime -import yaml import json -import argparse, logging, sys, os, collections -from typing import Dict, List, TypedDict -from config import Config -from project import Project -from documentation import Docs -from shuttle import ShuttleConfig -from rom import ROMFile - +import logging +import os +import sys # pipe handling -from signal import signal, SIGPIPE, SIG_DFL +from signal import SIG_DFL, SIGPIPE, signal +from typing import Dict, List, TypedDict + +import yaml + +from config import Config +from documentation import Docs +from project import Project +from rom import ROMFile +from shuttle import ShuttleConfig + signal(SIGPIPE, SIG_DFL) -class Projects(): - +class Projects: def __init__(self, config: Config, args): self.args = args self.config = config - self.project_dir = config['project_dir'] + self.project_dir = config["project_dir"] if not os.path.exists(self.project_dir): os.makedirs(self.project_dir) self.projects: List[Project] = [] - project_list = [entry for entry in os.listdir(self.project_dir) if os.path.isdir(os.path.join(self.project_dir, entry))] + project_list = [ + entry + for entry in os.listdir(self.project_dir) + if os.path.isdir(os.path.join(self.project_dir, entry)) + ] if args.test: - project_list = ['tt_um_chip_rom', 'tt_um_factory_test'] + project_list = ["tt_um_chip_rom", "tt_um_factory_test"] elif args.sta_projects: - project_list = ['tt_um_loopback'] + project_list = ["tt_um_loopback"] for index, project_id in enumerate(project_list): project_dir = os.path.join(self.project_dir, project_id) - commit_id_file = os.path.join(project_dir, 'commit_id.json') + commit_id_file = os.path.join(project_dir, "commit_id.json") if not os.path.exists(commit_id_file): logging.warning(f"no commit_id.json in {project_dir}, skipping") continue commit_id_data = json.load(open(commit_id_file)) - project = Project(index, commit_id_data['repo'], project_dir, args, is_user_project=False) - project.commit_id = commit_id_data['commit'] - project.sort_id = commit_id_data['sort_id'] + project = Project( + index, commit_id_data["repo"], project_dir, args, is_user_project=False + ) + project.commit_id = commit_id_data["commit"] + project.sort_id = commit_id_data["sort_id"] # projects should now be installed, so load all the data from the yaml files # fill projects will load from the fill project's directory @@ -65,7 +76,9 @@ class Projects(): self.projects.sort(key=lambda x: x.sort_id) - all_macro_instances = [project.get_macro_instance() for project in self.projects] + all_macro_instances = [ + project.get_macro_instance() for project in self.projects + ] self.assert_unique(all_macro_instances) all_gds_files = [project.get_macro_gds_filename() for project in self.projects] @@ -74,7 +87,9 @@ class Projects(): logging.info(f"loaded {len(self.projects)} projects") def assert_unique(self, check): - duplicates = [item for item, count in collections.Counter(check).items() if count > 1] + duplicates = [ + item for item, count in collections.Counter(check).items() if count > 1 + ] if duplicates: logging.error("duplicate projects: {}".format(duplicates)) exit(1) @@ -95,47 +110,56 @@ class Projects(): for project in self.projects: try: - dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss') + dt = datetime.datetime.strptime( + project.metrics["total_runtime"][:-3], "%Hh%Mm%Ss" + ) except KeyError: continue - delt = datetime.timedelta(hours=dt.hour, minutes=dt.minute, seconds=dt.second) + delt = datetime.timedelta( + hours=dt.hour, minutes=dt.minute, seconds=dt.second + ) total_seconds += delt.total_seconds() cell_count = project.get_cell_counts_from_gl() script_dir = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(script_dir, 'categories.json')) as fh: + with open(os.path.join(script_dir, "categories.json")) as fh: categories = json.load(fh) - CategoryInfo = TypedDict("CategoryInfo", {'count': int, 'examples': List[str]}) + CategoryInfo = TypedDict( + "CategoryInfo", {"count": int, "examples": List[str]} + ) by_category: Dict[str, CategoryInfo] = {} total = 0 for cell_name in cell_count: - cat_index = categories['map'][cell_name] - cat_name = categories['categories'][cat_index] + cat_index = categories["map"][cell_name] + cat_name = categories["categories"][cat_index] if cat_name in by_category: - by_category[cat_name]['count'] += cell_count[cell_name] - by_category[cat_name]['examples'].append(cell_name) + by_category[cat_name]["count"] += cell_count[cell_name] + by_category[cat_name]["examples"].append(cell_name) else: - by_category[cat_name] = {'count' : cell_count[cell_name], 'examples' : [cell_name]} + by_category[cat_name] = { + "count": cell_count[cell_name], + "examples": [cell_name], + } - if cat_name not in ['Fill', 'Tap', 'Buffer', 'Misc']: + if cat_name not in ["Fill", "Tap", "Buffer", "Misc"]: total += cell_count[cell_name] if total < 10: - del by_category['Fill'] - del by_category['Tap'] - if 'Buffer' in by_category: - del by_category['Buffer'] + del by_category["Fill"] + del by_category["Tap"] + if "Buffer" in by_category: + del by_category["Buffer"] print(project.get_macro_name(), total, by_category) - total_wire_length += int(project.metrics['wire_length']) - total_wires_count += int(project.metrics['wires_count']) - util = float(project.metrics['OpenDP_Util']) + total_wire_length += int(project.metrics["wire_length"]) + total_wires_count += int(project.metrics["wires_count"]) + util = float(project.metrics["OpenDP_Util"]) num_cells = project.get_cell_count_from_synth() total_physical_cells += num_cells yaml_data = project.get_project_doc_yaml() - lang = yaml_data['language'].lower() + lang = yaml_data["language"].lower() if lang in languages: languages[lang] += 1 else: @@ -154,7 +178,7 @@ class Projects(): min_util = util try: - tags += yaml_data['tag'].split(',') + tags += yaml_data["tag"].split(",") except KeyError: pass @@ -185,36 +209,88 @@ class Projects(): count_items(tags) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser(description="TinyTapeout configuration and docs") - with open('config.yaml') as fh: + with open("config.yaml") as fh: config = yaml.safe_load(fh) - - parser.add_argument('--list', help="list projects", action='store_const', const=True) - parser.add_argument('--single', help="do action on single project", type=int, default=-1) - parser.add_argument('--update-shuttle', help='configure shuttle for build', action='store_const', const=True) - parser.add_argument('--copy-macros', help='copy macros for building the tt_top project', action='store_const', const=True) - parser.add_argument('--copy-final-results', help='copy final project files to gds/lef directories', action='store_const', const=True) - parser.add_argument('--create-efabless-submission', help='create efabless submission files', action='store_const', const=True) - parser.add_argument('--harden', help="harden project", action="store_const", const=True) - parser.add_argument('--test', help='use test projects', action='store_const', const=True) - parser.add_argument('--sta-projects', help='use sta projects', action='store_const', const=True) - parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) - parser.add_argument('--log-email', help="print persons email in messages", action="store_const", const=True) - parser.add_argument('--update-image', help="update the image", action="store_const", const=True) - parser.add_argument('--dump-json', help="dump json of all project data to given file") - parser.add_argument('--dump-markdown', help="dump markdown of all project data to given file") - parser.add_argument('--dump-pdf', help="create pdf from the markdown") - parser.add_argument('--build-hugo-content', help="directory to where to build hugo content") - parser.add_argument('--metrics', help="print some project metrics", action="store_const", const=True) + + parser.add_argument( + "--list", help="list projects", action="store_const", const=True + ) + parser.add_argument( + "--single", help="do action on single project", type=int, default=-1 + ) + parser.add_argument( + "--update-shuttle", + help="configure shuttle for build", + action="store_const", + const=True, + ) + parser.add_argument( + "--copy-macros", + help="copy macros for building the tt_top project", + action="store_const", + const=True, + ) + parser.add_argument( + "--copy-final-results", + help="copy final project files to gds/lef directories", + action="store_const", + const=True, + ) + parser.add_argument( + "--create-efabless-submission", + help="create efabless submission files", + action="store_const", + const=True, + ) + parser.add_argument( + "--harden", help="harden project", action="store_const", const=True + ) + parser.add_argument( + "--test", help="use test projects", action="store_const", const=True + ) + parser.add_argument( + "--sta-projects", help="use sta projects", action="store_const", const=True + ) + parser.add_argument( + "--debug", + help="debug logging", + action="store_const", + dest="loglevel", + const=logging.DEBUG, + default=logging.INFO, + ) + parser.add_argument( + "--log-email", + help="print persons email in messages", + action="store_const", + const=True, + ) + parser.add_argument( + "--update-image", help="update the image", action="store_const", const=True + ) + parser.add_argument( + "--dump-json", help="dump json of all project data to given file" + ) + parser.add_argument( + "--dump-markdown", help="dump markdown of all project data to given file" + ) + parser.add_argument("--dump-pdf", help="create pdf from the markdown") + parser.add_argument( + "--build-hugo-content", help="directory to where to build hugo content" + ) + parser.add_argument( + "--metrics", help="print some project metrics", action="store_const", const=True + ) args = parser.parse_args() # setup log - log_format = logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s') + log_format = logging.Formatter("%(asctime)s - %(levelname)-8s - %(message)s") # configure the client logging - log = logging.getLogger('') + log = logging.getLogger("") # has to be set to debug as is the root logger log.setLevel(args.loglevel) diff --git a/discord_bot.py b/discord_bot.py index f03c450..367097f 100755 --- a/discord_bot.py +++ b/discord_bot.py @@ -2,14 +2,13 @@ import os import sys -import logging + import discord import yaml -import re -TOKEN = os.environ['DISCORD_TOKEN'] -TARG = os.environ['TARGET_SERVER'] -ROLE_ID = os.environ['ROLE_ID'] +TOKEN = os.environ["DISCORD_TOKEN"] +TARG = os.environ["TARGET_SERVER"] +ROLE_ID = os.environ["ROLE_ID"] intents = discord.Intents.default() intents.members = True @@ -17,50 +16,52 @@ intents.members = True client = discord.Client(intents=intents) exit_code = 0 + @client.event async def on_ready(): + global exit_code try: - print(f'{client.user} has connected to Discord!') + print(f"{client.user} has connected to Discord!") server = None for g in client.guilds: if g.name == TARG: server = g break - if server == None: - print(f'Error: Bot is not in a server named {TARG}') + if server is None: + print(f"Error: Bot is not in a server named {TARG}") exit_code = 1 await client.close() - + role = server.get_role(int(ROLE_ID)) if not role: - print('Error: requested role does not exist on server!') + print("Error: requested role does not exist on server!") exit_code = 1 await client.close() - + participants = [] - for project in os.listdir('./projects'): - info_file = os.path.join('projects', project, 'info.yaml') + for project in os.listdir("./projects"): + info_file = os.path.join("projects", project, "info.yaml") if not os.path.exists(info_file): - print(f'info.yaml does not exist for project {project}. Skipping.') + print(f"info.yaml does not exist for project {project}. Skipping.") else: try: - stream = open(info_file, 'r') + stream = open(info_file, "r") data = yaml.load(stream, Loader=yaml.FullLoader) stream.close() - if 'discord' in data['documentation']: - dname = data['documentation']['discord'] + if "discord" in data["documentation"]: + dname = data["documentation"]["discord"] if len(dname): - if '#' in dname: - dname = dname.split('#')[0] -# print(f'adding {dname}') + if "#" in dname: + dname = dname.split("#")[0] + # print(f'adding {dname}') participants.append(dname) else: - print(f'Project {project} has no discord username listed. Skipping.') + print( + f"Project {project} has no discord username listed. Skipping." + ) except Exception as ex: - print(f'Error parsing info.yaml for project {project}. Skipping.') - print(f'Error was: {ex}') - - + print(f"Error parsing info.yaml for project {project}. Skipping.") + print(f"Error was: {ex}") members = [] for p in participants: @@ -72,26 +73,29 @@ async def on_ready(): members.append(m) break else: - print(f'{p} not found in members') - + print(f"{p} not found in members") + new_role_members = [] for m in members: if len([d for d in m.roles if d.id == role.id]) == 0: - print(f'Giving the role to {m.name}') + print(f"Giving the role to {m.name}") new_role_members.append(m.name) - await m.add_roles(role, reason='Automatic role assignment by GitHub actions pipeline.') + await m.add_roles( + role, reason="Automatic role assignment by GitHub actions pipeline." + ) else: - print(f'{m.name} already has role') - - print('All done!') + print(f"{m.name} already has role") + + print("All done!") print("gave new role to:") - print('@' + ' @'.join(new_role_members)) + print("@" + " @".join(new_role_members)) await client.close() except Exception as ex: - print('Exception was thrown!') + print("Exception was thrown!") print(ex) exit_code = 1 await client.close() + client.run(TOKEN) sys.exit(exit_code) diff --git a/documentation.py b/documentation.py index 246a277..a04ec2a 100644 --- a/documentation.py +++ b/documentation.py @@ -1,17 +1,18 @@ -import os -import logging -import subprocess -import shutil import json +import logging +import os +import shutil +import subprocess + +import frontmatter # type: ignore import git -import frontmatter # type: ignore + from config import Config from git_utils import get_first_remote from markdown_utils import latex_centered_image, rewrite_image_paths -class Docs(): - +class Docs: def __init__(self, config: Config, projects, args): self.config = config self.projects = projects @@ -21,9 +22,9 @@ class Docs(): # stuff related to docs def build_index(self): logging.info("building doc index") - with open(os.path.join(self.script_dir, 'docs', 'README_init.md')) as fh: + with open(os.path.join(self.script_dir, "docs", "README_init.md")) as fh: readme = fh.read() - with open("README.md", 'w') as fh: + with open("README.md", "w") as fh: fh.write(readme) fh.write("| Address | Author | Title | Type | Git Repo |\n") fh.write("| ------- | ------ | ------| -----| ---------|\n") @@ -32,10 +33,10 @@ class Docs(): fh.write(project.get_index_row()) def update_image(self): - ruby = os.path.join(self.script_dir, 'caravel_template', 'dump_pic.rb') - klayoutrc = os.path.join(self.script_dir, 'caravel_template', 'klayoutrc') - lyp = os.path.join(self.script_dir, 'caravel_template', 'caravel.lyp') - cmd = f'klayout -l {lyp} gds/user_project_wrapper.gds* -r {ruby} -c {klayoutrc}' + ruby = os.path.join(self.script_dir, "caravel_template", "dump_pic.rb") + klayoutrc = os.path.join(self.script_dir, "caravel_template", "klayoutrc") + lyp = os.path.join(self.script_dir, "caravel_template", "caravel.lyp") + cmd = f"klayout -l {lyp} gds/user_project_wrapper.gds* -r {ruby} -c {klayoutrc}" logging.info(cmd) os.system(cmd) @@ -46,68 +47,83 @@ class Docs(): design = project.get_yaml() designs.append(design) - with open(self.args.dump_json, 'w') as fh: + with open(self.args.dump_json, "w") as fh: fh.write(json.dumps(designs, indent=4)) - logging.info(f'wrote json to {self.args.dump_json}') + logging.info(f"wrote json to {self.args.dump_json}") def load_doc_template(self, name): - root = os.path.join(self.script_dir, 'docs') + root = os.path.join(self.script_dir, "docs") doc_path = os.path.join(root, name) image_root = os.path.relpath(os.path.dirname(doc_path), ".") doc = frontmatter.load(doc_path) doc.content = rewrite_image_paths(doc.content, image_root) - if 'title' in doc: - doc['title'] = rewrite_image_paths(doc['title'], image_root) + if "title" in doc: + doc["title"] = rewrite_image_paths(doc["title"], image_root) if len(doc.keys()) > 0: return frontmatter.dumps(doc) + "\n" else: return doc.content + "\n" def dump_markdown(self): - doc_header = self.load_doc_template('doc_header.md') - doc_template = self.load_doc_template('doc_template.md') - doc_pinout = self.load_doc_template('PINOUT.md') + doc_header = self.load_doc_template("doc_header.md") + doc_template = self.load_doc_template("doc_template.md") + doc_pinout = self.load_doc_template("PINOUT.md") doc_info = self.load_doc_template("../../tt-multiplexer/docs/INFO.md") - doc_credits = self.load_doc_template('CREDITS.md') + doc_credits = self.load_doc_template("CREDITS.md") - with open(self.args.dump_markdown, 'w') as fh: + with open(self.args.dump_markdown, "w") as fh: repo = git.Repo(".") - fh.write(doc_header.format(name=self.config['name'], repo=get_first_remote(repo))) + fh.write( + doc_header.format(name=self.config["name"], repo=get_first_remote(repo)) + ) self.projects.sort(key=lambda x: x.mux_address) for project in self.projects: yaml_data = project.get_project_doc_yaml() - yaml_data['mux_address'] = project.mux_address + yaml_data["mux_address"] = project.mux_address logging.info(f"building datasheet for {project}") # handle pictures - yaml_data['picture_link'] = '' - if yaml_data['picture']: - extension = os.path.splitext(yaml_data['picture'])[1] - picture_path = os.path.join(project.local_dir, f"picture{extension}") + yaml_data["picture_link"] = "" + if yaml_data["picture"]: + extension = os.path.splitext(yaml_data["picture"])[1] + picture_path = os.path.join( + project.local_dir, f"picture{extension}" + ) if os.path.exists(picture_path): - yaml_data['picture_link'] = latex_centered_image(picture_path) + yaml_data["picture_link"] = latex_centered_image(picture_path) else: logging.warning(f"picture {picture_path} not found, skipping") # ensure there are no LaTeX escape sequences in various fields, and that optional fields are set - for key in ['author', 'description', 'how_it_works', 'how_to_test', 'external_hw', 'clock_hz', 'git_url', 'doc_link']: + for key in [ + "author", + "description", + "how_it_works", + "how_to_test", + "external_hw", + "clock_hz", + "git_url", + "doc_link", + ]: if key in yaml_data: - yaml_data[key] = str(yaml_data[key]).replace("\\", "\mbox{\\textbackslash}") + yaml_data[key] = str(yaml_data[key]).replace( + "\\", "\\mbox{\\textbackslash}" + ) else: yaml_data[key] = "" # many people remove unused pins in input / output / bidirectional - for key in ['inputs', 'outputs', 'bidirectional']: - yaml_data[key].extend((8 - len(yaml_data[key])) * ['n/a']) + for key in ["inputs", "outputs", "bidirectional"]: + yaml_data[key].extend((8 - len(yaml_data[key])) * ["n/a"]) # now build the doc & print it try: - doc = (doc_template - .replace('__git_url__', '{git_url}') - .replace('__doc_link__', '{doc_link}') + doc = ( + doc_template.replace("__git_url__", "{git_url}") + .replace("__doc_link__", "{doc_link}") .format(**yaml_data) ) fh.write(doc) @@ -122,10 +138,10 @@ class Docs(): fh.write("\n\\clearpage\n") fh.write(doc_credits) - logging.info(f'wrote markdown to {self.args.dump_markdown}') + logging.info(f"wrote markdown to {self.args.dump_markdown}") if self.args.dump_pdf: - pdf_cmd = f'pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {self.args.dump_markdown} -o {self.args.dump_pdf}' + pdf_cmd = f"pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {self.args.dump_markdown} -o {self.args.dump_pdf}" logging.info(pdf_cmd) p = subprocess.run(pdf_cmd, shell=True) if p.returncode != 0: @@ -133,49 +149,61 @@ class Docs(): def build_hugo_content(self): hugo_root = self.args.build_hugo_content - hugo_images = os.path.join(hugo_root, 'images') + hugo_images = os.path.join(hugo_root, "images") shutil.rmtree(hugo_root) os.makedirs(hugo_root) os.makedirs(hugo_images) - with open(os.path.join(self.script_dir, 'docs', 'hugo_template.md')) as fh: + with open(os.path.join(self.script_dir, "docs", "hugo_template.md")) as fh: doc_template = fh.read() - with open(os.path.join(self.script_dir, 'docs', 'hugo_index_template.md')) as fh: + with open( + os.path.join(self.script_dir, "docs", "hugo_index_template.md") + ) as fh: index_template = fh.read() # copy image - shutil.copyfile('pics/tinytapeout_numbered.png', os.path.join(hugo_images, 'tinytapeout-03.png')) # TODO fix hardcoded run + shutil.copyfile( + "pics/tinytapeout_numbered.png", + os.path.join(hugo_images, "tinytapeout-03.png"), + ) # TODO fix hardcoded run # index page - logging.info("building pages - can take a minute as fetching latest GDS action URLs for all projects") - with open(os.path.join(hugo_root, '_index.md'), 'w') as fh: + logging.info( + "building pages - can take a minute as fetching latest GDS action URLs for all projects" + ) + with open(os.path.join(hugo_root, "_index.md"), "w") as fh: fh.write(index_template) - fh.write('# All projects\n') + fh.write("# All projects\n") fh.write("| Index | Title | Author |\n") fh.write("| ----- | ----- | -------|\n") for project in self.projects: logging.info(project) fh.write(project.get_hugo_row()) - project_dir = os.path.join(hugo_root, f'{project.get_index() :03}') - project_image_dir = os.path.join(project_dir, 'images') + project_dir = os.path.join(hugo_root, f"{project.get_index() :03}") + project_image_dir = os.path.join(project_dir, "images") os.makedirs(project_dir) os.makedirs(project_image_dir) yaml_data = project.get_project_doc_yaml() - yaml_data['index'] = project.index - yaml_data['weight'] = project.index + 1 - yaml_data['git_action'] = project.get_latest_action_url() - yaml_data['picture_link'] = '' - if yaml_data['picture']: - picture_name = yaml_data['picture'] + yaml_data["index"] = project.index + yaml_data["weight"] = project.index + 1 + yaml_data["git_action"] = project.get_latest_action_url() + yaml_data["picture_link"] = "" + if yaml_data["picture"]: + picture_name = yaml_data["picture"] picture_filename = os.path.join(project.local_dir, picture_name) picture_basename = os.path.basename(picture_filename) try: - shutil.copyfile(picture_filename, os.path.join(project_image_dir, picture_basename)) - yaml_data['picture_link'] = f'![picture](images/{picture_basename})' + shutil.copyfile( + picture_filename, + os.path.join(project_image_dir, picture_basename), + ) + yaml_data[ + "picture_link" + ] = f"![picture](images/{picture_basename})" except FileNotFoundError: - yaml_data['picture_link'] = 'Image path is broken' + yaml_data["picture_link"] = "Image path is broken" doc = doc_template.format(**yaml_data) - with open(os.path.join(project_dir, '_index.md'), 'w') as pfh: + with open(os.path.join(project_dir, "_index.md"), "w") as pfh: pfh.write(doc) diff --git a/git_utils.py b/git_utils.py index cd27c0e..84b8c9c 100644 --- a/git_utils.py +++ b/git_utils.py @@ -1,12 +1,14 @@ import base64 -from urllib.parse import urlparse +import errno import logging -import requests -from git.repo import Repo +import os import sys import typing -import os -import errno +from urllib.parse import urlparse + +import requests +from git.repo import Repo + def fetch_file(url: str, filename: str): logging.info("trying to download {}".format(url)) @@ -15,82 +17,92 @@ def fetch_file(url: str, filename: str): logging.warning("couldn't download {}".format(url)) raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) - with open(filename, 'wb') as fh: + with open(filename, "wb") as fh: logging.info("written to {}".format(filename)) fh.write(r.content) def check_status(r: requests.Response): if r.status_code == 401: - logging.error("unauthorised, check INFO.md for information about GitHub API keys") + logging.error( + "unauthorised, check INFO.md for information about GitHub API keys" + ) exit(1) -def headers_try_to_add_authorization_from_environment(headers: typing.Dict[str, str]) -> bool: - gh_token = os.getenv('GH_TOKEN', '') # override like gh CLI + +def headers_try_to_add_authorization_from_environment( + headers: typing.Dict[str, str] +) -> bool: + gh_token = os.getenv("GH_TOKEN", "") # override like gh CLI if not gh_token: - gh_token = os.getenv('GITHUB_TOKEN', '') # GHA inherited + gh_token = os.getenv("GITHUB_TOKEN", "") # GHA inherited if len(gh_token) > 0: # As per https://docs.github.com/en/rest/overview/authenticating-to-the-rest-api - headers['authorization'] = 'Bearer ' + gh_token + headers["authorization"] = "Bearer " + gh_token return True # Use a token instead which is designed to limit exposure of passwords # I can't find any GH docs explaining use cases for Basic auth and confirming a token # can be used instead of PASSWORD in the password field of authorization header. - gh_username = os.getenv('GH_USERNAME', '') # override like gh CLI + gh_username = os.getenv("GH_USERNAME", "") # override like gh CLI if not gh_username: - gh_username = os.getenv('GITHUB_ACTOR', '') # GHA inherited + gh_username = os.getenv("GITHUB_ACTOR", "") # GHA inherited - gh_password = os.getenv('GH_PASSWORD', '') + gh_password = os.getenv("GH_PASSWORD", "") if len(gh_username) > 0 and len(gh_password) > 0: - auth_string = gh_username + ':' + gh_password - encoded = base64.b64encode(auth_string.encode('ascii')) - headers['authorization'] = 'Basic ' + encoded.decode('ascii') + auth_string = gh_username + ":" + gh_password + encoded = base64.b64encode(auth_string.encode("ascii")) + headers["authorization"] = "Basic " + encoded.decode("ascii") return True - print("WARNING: No github token found from environment, trying public API requests without, see docs/INFO.md#instructions-to-build-gds", file=sys.stderr) + print( + "WARNING: No github token found from environment, trying public API requests without, see docs/INFO.md#instructions-to-build-gds", + file=sys.stderr, + ) return False def get_most_recent_action_page( - commits: typing.List[typing.Dict[str, str]], - runs: typing.List[typing.Dict[str, str]]) -> typing.Optional[str]: - release_sha_to_page_url = {run['head_sha']: run['html_url'] for run in runs if run['name'] == 'gds'} + commits: typing.List[typing.Dict[str, str]], + runs: typing.List[typing.Dict[str, str]], +) -> typing.Optional[str]: + release_sha_to_page_url = { + run["head_sha"]: run["html_url"] for run in runs if run["name"] == "gds" + } for commit in commits: - if commit['sha'] in release_sha_to_page_url: - return release_sha_to_page_url[commit['sha']] + if commit["sha"] in release_sha_to_page_url: + return release_sha_to_page_url[commit["sha"]] return None def split_git_url(url: str): res = urlparse(url) try: - _, user_name, repo = res.path.split('/') + _, user_name, repo = res.path.split("/") except ValueError: logging.error(f"couldn't split repo from {url}") exit(1) - repo = repo.replace('.git', '') + repo = repo.replace(".git", "") return user_name, repo - def get_latest_action_url(url: str): logging.debug(url) user_name, repo = split_git_url(url) headers = { - "Accept" : "application/vnd.github+json", - } + "Accept": "application/vnd.github+json", + } # authenticate for rate limiting headers_try_to_add_authorization_from_environment(headers) # first fetch the git commit history - api_url = f'https://api.github.com/repos/{user_name}/{repo}/commits' + api_url = f"https://api.github.com/repos/{user_name}/{repo}/commits" r = requests.get(api_url, headers=headers) check_status(r) - requests_remaining = int(r.headers['X-RateLimit-Remaining']) + requests_remaining = int(r.headers["X-RateLimit-Remaining"]) if requests_remaining == 0: logging.error("no API requests remaining") exit(1) @@ -98,11 +110,11 @@ def get_latest_action_url(url: str): commits = r.json() # get runs - api_url = f'https://api.github.com/repos/{user_name}/{repo}/actions/runs' - r = requests.get(api_url, headers=headers, params={'per_page': 100}) + api_url = f"https://api.github.com/repos/{user_name}/{repo}/actions/runs" + r = requests.get(api_url, headers=headers, params={"per_page": 100}) check_status(r) runs = r.json() - page_url = get_most_recent_action_page(commits, runs['workflow_runs']) + page_url = get_most_recent_action_page(commits, runs["workflow_runs"]) return page_url diff --git a/markdown_utils.py b/markdown_utils.py index 7964538..a33e100 100644 --- a/markdown_utils.py +++ b/markdown_utils.py @@ -1,4 +1,5 @@ import os + import mistune from mistune.renderers.markdown import MarkdownRenderer @@ -25,7 +26,7 @@ class ImagePathRewriterRenderer(MarkdownRenderer): def image(self, token, state): url = token["attrs"]["url"] - if not "://" in url and not url.startswith("/"): + if "://" not in url and not url.startswith("/"): token["attrs"]["url"] = os.path.join(self.prefix, url) return super().image(token, state) diff --git a/project.py b/project.py index 95c0a44..7088610 100644 --- a/project.py +++ b/project.py @@ -1,25 +1,28 @@ -import yaml -import glob -import os -import logging -import re -import subprocess -import gdstk # type: ignore -import cairosvg # type: ignore import csv +import glob import json -import git_utils -from git.repo import Repo +import logging +import os +import re import shutil +import subprocess import typing -from cells import load_cells, Cell + +import cairosvg # type: ignore +import gdstk # type: ignore +import yaml +from git.repo import Repo + +import git_utils +from cells import Cell, load_cells from markdown_utils import limit_markdown_headings -CELL_URL = 'https://skywater-pdk.readthedocs.io/en/main/contents/libraries/sky130_fd_sc_hd/cells/' +CELL_URL = "https://skywater-pdk.readthedocs.io/en/main/contents/libraries/sky130_fd_sc_hd/cells/" -with open(os.path.join(os.path.dirname(__file__), 'tile_sizes.yaml'), 'r') as stream: - tile_sizes = (yaml.safe_load(stream)) +with open(os.path.join(os.path.dirname(__file__), "tile_sizes.yaml"), "r") as stream: + tile_sizes = yaml.safe_load(stream) + class Args: openlane2: bool @@ -27,19 +30,30 @@ class Args: print_cell_category: bool -class Project(): +class Project: top_verilog_filename: str mux_address: int commit_id: str sort_id: int - def __init__(self, index: int, git_url: str, local_dir: str, args: Args, is_user_project: bool): + def __init__( + self, + index: int, + git_url: str, + local_dir: str, + args: Args, + is_user_project: bool, + ): self.git_url = git_url self.args = args self.index = index self.local_dir = local_dir self.is_user_project = is_user_project - self.src_dir = os.path.join(self.local_dir, 'src') if self.is_user_project else self.local_dir + self.src_dir = ( + os.path.join(self.local_dir, "src") + if self.is_user_project + else self.local_dir + ) def post_clone_setup(self): self.load_yaml() @@ -56,11 +70,11 @@ class Project(): def check_ports(self): top = self.get_macro_name() if not self.is_user_project and top == "tt_um_chip_rom": - return # Chip ROM is auto generated, so we don't have the verilog yet + return # Chip ROM is auto generated, so we don't have the verilog yet sources = [os.path.join(self.src_dir, src) for src in self.src_files] source_list = " ".join(sources) - json_file = 'ports.json' + json_file = "ports.json" yosys_cmd = f"yowasp-yosys -qp 'read_verilog -lib -sv {source_list}; hierarchy -top {top} ; proc; write_json {json_file}'" p = subprocess.run(yosys_cmd, shell=True) if p.returncode != 0: @@ -71,24 +85,26 @@ class Project(): ports = json.load(fh) os.unlink(json_file) - module_ports = ports['modules'][top]['ports'] + module_ports = ports["modules"][top]["ports"] required_ports = [ - ['clk', 1], - ['ena', 1], - ['rst_n', 1], - ['ui_in', 8], - ['uio_in', 8], - ['uio_oe', 8], - ['uio_out', 8], - ['uo_out', 8], + ["clk", 1], + ["ena", 1], + ["rst_n", 1], + ["ui_in", 8], + ["uio_in", 8], + ["uio_oe", 8], + ["uio_out", 8], + ["uo_out", 8], ] for port, bits in required_ports: if port not in module_ports: logging.error(f"{self} port '{port}' missing from top module ('{top}')") exit(1) - actual_bits = len(module_ports[port]['bits']) + actual_bits = len(module_ports[port]["bits"]) if actual_bits != bits: - logging.error(f"{self} incorrect width for port '{port}' in module '{top}': {bits} bits required, {actual_bits} found") + logging.error( + f"{self} incorrect width for port '{port}' in module '{top}': {bits} bits required, {actual_bits} found" + ) exit(1) def check_num_cells(self): @@ -109,10 +125,12 @@ class Project(): def load_yaml(self): try: - with open(os.path.join(self.local_dir, 'info.yaml')) as fh: + with open(os.path.join(self.local_dir, "info.yaml")) as fh: self.yaml = yaml.safe_load(fh) except FileNotFoundError: - logging.error(f"yaml file not found for {self} - do you need to --clone the project repos?") + logging.error( + f"yaml file not found for {self} - do you need to --clone the project repos?" + ) exit(1) # find and save the location of the source files @@ -120,45 +138,47 @@ class Project(): def setup_source_files(self): # wokwi_id must be an int or 0 try: - self.wokwi_id = int(self.yaml['project']['wokwi_id']) + self.wokwi_id = int(self.yaml["project"]["wokwi_id"]) except ValueError: logging.error("wokwi id must be an integer") exit(1) - self.yaml['project']['git_url'] = self.git_url + self.yaml["project"]["git_url"] = self.git_url if self.is_hdl(): - self.top_module = self.yaml['project']['top_module'] + self.top_module = self.yaml["project"]["top_module"] if self.is_user_project: - self.src_files = self.get_hdl_source() - self.top_verilog_filename = self.find_top_verilog() + self.src_files = self.get_hdl_source() + self.top_verilog_filename = self.find_top_verilog() else: - self.top_module = f"tt_um_wokwi_{self.wokwi_id}" + self.top_module = f"tt_um_wokwi_{self.wokwi_id}" if self.is_user_project: - self.src_files = self.get_wokwi_source() - self.top_verilog_filename = self.src_files[0] + self.src_files = self.get_wokwi_source() + self.top_verilog_filename = self.src_files[0] if not self.is_user_project: - self.src_files = [self.get_gl_verilog_filename()] - - self.unprefixed_name = re.sub('^tt_um_', '', self.top_module) - + self.src_files = [self.get_gl_verilog_filename()] + + self.unprefixed_name = re.sub("^tt_um_", "", self.top_module) + if not self.top_module.startswith("tt_um_"): - logging.error(f'top module name must start with "tt_um_" (current value: "{self.top_module}")') + logging.error( + f'top module name must start with "tt_um_" (current value: "{self.top_module}")' + ) exit(1) - self.macro_instance = f"{self.top_module}" + self.macro_instance = f"{self.top_module}" def get_wokwi_source(self): src_file = "tt_um_wokwi_{}.v".format(self.wokwi_id) - return [src_file, 'cells.v'] + return [src_file, "cells.v"] def get_hdl_source(self): - if 'source_files' not in self.yaml['project']: + if "source_files" not in self.yaml["project"]: logging.error("source files must be provided if wokwi_id is set to 0") exit(1) - source_files = self.yaml['project']['source_files'] + source_files = self.yaml["project"]["source_files"] if source_files is None: logging.error("must be more than 1 source file") exit(1) @@ -167,16 +187,18 @@ class Project(): logging.error("must be more than 1 source file") exit(1) - if 'top_module' not in self.yaml['project']: + if "top_module" not in self.yaml["project"]: logging.error("must provide a top module name") exit(1) - if self.yaml['project']['top_module'] == 'top': - logging.error("top module cannot be called top - prepend your repo name to make it unique") + if self.yaml["project"]["top_module"] == "top": + logging.error( + "top module cannot be called top - prepend your repo name to make it unique" + ) exit(1) for filename in source_files: - if '*' in filename: + if "*" in filename: logging.error("* not allowed, please specify each file") exit(1) if not os.path.exists(os.path.join(self.src_dir, filename)): @@ -203,20 +225,22 @@ class Project(): def get_project_doc_yaml(self): # fstring dict support is limited to one level deep, so put the git url and wokwi url in the docs key - docs = self.yaml['documentation'] - docs['project_type'] = self.get_project_type_string() - docs['git_url'] = self.git_url + docs = self.yaml["documentation"] + docs["project_type"] = self.get_project_type_string() + docs["git_url"] = self.git_url # "How it works" and "How to test" may include markdown headings - make sure they don't break the ToC - docs['how_it_works'] = limit_markdown_headings(docs['how_it_works'], min_level=4) - docs['how_to_test'] = limit_markdown_headings(docs['how_to_test'], min_level=4) + docs["how_it_works"] = limit_markdown_headings( + docs["how_it_works"], min_level=4 + ) + docs["how_to_test"] = limit_markdown_headings(docs["how_to_test"], min_level=4) return docs def get_wokwi_url(self): - return f'https://wokwi.com/projects/{self.wokwi_id}' + return f"https://wokwi.com/projects/{self.wokwi_id}" # top module name is defined in one of the source files, which one? def find_top_verilog(self): - rgx_mod = re.compile(r"(?:^|[\s])module[\s]{1,}([\w]+)") + rgx_mod = re.compile(r"(?:^|[\s])module[\s]{1,}([\w]+)") top_verilog: typing.List[str] = [] for src in self.src_files: with open(os.path.join(self.src_dir, src)) as fh: @@ -225,29 +249,35 @@ class Project(): if match.group(1) == self.top_module: top_verilog.append(src) if len(top_verilog) == 0: - logging.error(f"Couldn't find verilog module '{self.top_module}' in any of the project's source files") + logging.error( + f"Couldn't find verilog module '{self.top_module}' in any of the project's source files" + ) exit(1) if len(top_verilog) > 1: - logging.error(f"Top verilog module '{self.top_module}' found in multiple source files: {', '.join(top_verilog)}") + logging.error( + f"Top verilog module '{self.top_module}' found in multiple source files: {', '.join(top_verilog)}" + ) exit(1) return top_verilog[0] def get_git_remote(self): return list(Repo(self.local_dir).remotes[0].urls)[0] - + def get_git_commit_hash(self): return Repo(self.local_dir).commit().hexsha - + def get_tt_tools_version(self): repo = Repo(os.path.join(self.local_dir, "tt")) return f"{repo.active_branch.name} {repo.commit().hexsha[:8]}" - + def get_workflow_url(self): - GITHUB_SERVER_URL = os.getenv('GITHUB_SERVER_URL') - GITHUB_REPOSITORY = os.getenv('GITHUB_REPOSITORY') - GITHUB_RUN_ID = os.getenv('GITHUB_RUN_ID') + GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL") + GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") + GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID") if GITHUB_SERVER_URL and GITHUB_REPOSITORY and GITHUB_RUN_ID: - return f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" + return ( + f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" + ) def __str__(self): """ @@ -275,20 +305,26 @@ class Project(): def get_metrics_path(self): if self.is_user_project: if self.args.openlane2: - return os.path.join(self.local_dir, 'runs/wokwi/final/metrics.csv') + return os.path.join(self.local_dir, "runs/wokwi/final/metrics.csv") else: - return os.path.join(self.local_dir, 'runs/wokwi/reports/metrics.csv') + return os.path.join(self.local_dir, "runs/wokwi/reports/metrics.csv") else: - return os.path.join(self.local_dir, 'metrics.csv') - + return os.path.join(self.local_dir, "metrics.csv") + def get_gl_path(self): if self.is_user_project: if self.args.openlane2: - return glob.glob(os.path.join(self.local_dir, 'runs/wokwi/final/nl/*.nl.v'))[0] + return glob.glob( + os.path.join(self.local_dir, "runs/wokwi/final/nl/*.nl.v") + )[0] else: - return glob.glob(os.path.join(self.local_dir, 'runs/wokwi/results/final/verilog/gl/*.nl.v'))[0] + return glob.glob( + os.path.join( + self.local_dir, "runs/wokwi/results/final/verilog/gl/*.nl.v" + ) + )[0] else: - return os.path.join(self.local_dir, f'{self.top_module}.v') + return os.path.join(self.local_dir, f"{self.top_module}.v") # name of the gds file def get_macro_gds_filename(self): @@ -314,7 +350,7 @@ class Project(): # a few people use 'top.v', which is OK as long as the top module is called something more unique # but then it needs to be made unique so the source can be found filename = os.path.basename(self.top_verilog_filename) - return f'{self.index :03}_{filename}' + return f"{self.index :03}_{filename}" else: return self.top_verilog_filename @@ -340,35 +376,40 @@ class Project(): url = f"https://wokwi.com/api/projects/{self.wokwi_id}/{truthtable_file}" try: git_utils.fetch_file(url, os.path.join(self.src_dir, truthtable_file)) - logging.info(f'Wokwi project {self.wokwi_id} has a truthtable included, will test!') + logging.info( + f"Wokwi project {self.wokwi_id} has a truthtable included, will test!" + ) self.install_wokwi_testing() except FileNotFoundError: pass - def install_wokwi_testing(self, destination_dir: str = 'src', resource_dir: typing.Optional[str] = None): + def install_wokwi_testing( + self, destination_dir: str = "src", resource_dir: typing.Optional[str] = None + ): if resource_dir is None or not len(resource_dir): - resource_dir = os.path.join(os.path.dirname(__file__), 'testing') + resource_dir = os.path.join(os.path.dirname(__file__), "testing") # directories in testing/lib to copy over - pyLibsDir = os.path.join(resource_dir, 'lib') + pyLibsDir = os.path.join(resource_dir, "lib") # src template dir - srcTplDir = os.path.join(resource_dir, 'src-tpl') + srcTplDir = os.path.join(resource_dir, "src-tpl") - for libfullpath in glob.glob(os.path.join(pyLibsDir, '*')): - logging.info(f'Copying {libfullpath} to {destination_dir}') + for libfullpath in glob.glob(os.path.join(pyLibsDir, "*")): + logging.info(f"Copying {libfullpath} to {destination_dir}") shutil.copytree( libfullpath, os.path.join(destination_dir, os.path.basename(libfullpath)), - dirs_exist_ok=True) + dirs_exist_ok=True, + ) - for srcTemplate in glob.glob(os.path.join(srcTplDir, '*')): - with open(srcTemplate, 'r') as f: + for srcTemplate in glob.glob(os.path.join(srcTplDir, "*")): + with open(srcTemplate, "r") as f: contents = f.read() - customizedContents = re.sub('WOKWI_ID', str(self.wokwi_id), contents) + customizedContents = re.sub("WOKWI_ID", str(self.wokwi_id), contents) outputFname = os.path.basename(srcTemplate) - with open(os.path.join(destination_dir, outputFname), 'w') as outf: - logging.info(f'writing src tpl to {outputFname}') + with open(os.path.join(destination_dir, outputFname), "w") as outf: + logging.info(f"writing src tpl to {outputFname}") outf.write(customizedContents) outf.close() @@ -377,56 +418,64 @@ class Project(): self.fetch_wokwi_files() self.check_ports() logging.info("creating include file") - filename = 'user_config.tcl' - with open(os.path.join(self.src_dir, filename), 'w') as fh: + filename = "user_config.tcl" + with open(os.path.join(self.src_dir, filename), "w") as fh: fh.write("set ::env(DESIGN_NAME) {}\n".format(self.top_module)) fh.write('set ::env(VERILOG_FILES) "\\\n') for line, source in enumerate(self.src_files): fh.write(" $::env(DESIGN_DIR)/" + source) if line != len(self.src_files) - 1: - fh.write(' \\\n') + fh.write(" \\\n") fh.write('"\n\n') - tiles = self.yaml['project']['tiles'] + tiles = self.yaml["project"]["tiles"] die_area = tile_sizes[tiles] - fh.write(f'# Project area: {tiles} tiles\n') + fh.write(f"# Project area: {tiles} tiles\n") fh.write(f'set ::env(DIE_AREA) "{die_area}"\n') - fh.write(f'set ::env(FP_DEF_TEMPLATE) "$::env(DESIGN_DIR)/../tt/def/tt_block_{tiles}.def"\n') + fh.write( + f'set ::env(FP_DEF_TEMPLATE) "$::env(DESIGN_DIR)/../tt/def/tt_block_{tiles}.def"\n' + ) def golden_harden(self): logging.info(f"hardening {self}") # copy golden config - shutil.copyfile('golden_config.tcl', os.path.join(self.src_dir, 'config.tcl')) + shutil.copyfile("golden_config.tcl", os.path.join(self.src_dir, "config.tcl")) self.harden() def copy_picture_for_docs(self): - picture = self.yaml['documentation']['picture'] + picture = self.yaml["documentation"]["picture"] if not picture: return extension = os.path.splitext(picture)[1] - supported_extensions = ['.png', '.jpg', '.jpeg', '.svg', '.pdf'] + supported_extensions = [".png", ".jpg", ".jpeg", ".svg", ".pdf"] if not os.path.exists(picture): logging.warning(f"Picture file '{picture}' not found in repo, skipping") elif extension not in supported_extensions: - logging.warning(f"Picture file '{picture}' has unsupported extension '{extension}' (we support {', '.join(supported_extensions)}), skipping") + logging.warning( + f"Picture file '{picture}' has unsupported extension '{extension}' (we support {', '.join(supported_extensions)}), skipping" + ) else: - picture_dir = os.path.join(self.local_dir, 'src/__tinytapeout') + picture_dir = os.path.join(self.local_dir, "src/__tinytapeout") os.makedirs(picture_dir, exist_ok=True) - shutil.copyfile(picture, os.path.join(picture_dir, f'picture{extension}')) + shutil.copyfile(picture, os.path.join(picture_dir, f"picture{extension}")) def harden(self): cwd = os.getcwd() os.chdir(self.local_dir) - + repo = self.get_git_remote() commit_hash = self.get_git_commit_hash() tt_version = self.get_tt_tools_version() workflow_url = self.get_workflow_url() - + if self.args.openlane2: - if not os.path.exists('runs/wokwi'): - print("OpenLane 2 harden not supported yet, please run OpenLane 2 manually") + if not os.path.exists("runs/wokwi"): + print( + "OpenLane 2 harden not supported yet, please run OpenLane 2 manually" + ) exit(1) - print("Writing commit information on top of the existing OpenLane 2 run (runs/wokwi)") + print( + "Writing commit information on top of the existing OpenLane 2 run (runs/wokwi)" + ) else: # requires PDK, PDK_ROOT, OPENLANE_ROOT & OPENLANE_IMAGE_NAME to be set in local environment harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK=$PDK -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -tag wokwi"' @@ -436,20 +485,24 @@ class Project(): if p.returncode != 0: logging.error("harden failed") exit(1) - + # Write commit information - commit_id_json_path = 'runs/wokwi/results/final/commit_id.json' + commit_id_json_path = "runs/wokwi/results/final/commit_id.json" if self.args.openlane2: - commit_id_json_path = 'runs/wokwi/final/commit_id.json' - with open(os.path.join(self.local_dir, commit_id_json_path), 'w') as f: - json.dump({ - "app": f"Tiny Tapeout {tt_version}", - "repo": repo, - "commit": commit_hash, - "workflow_url": workflow_url, - }, f, indent=2) + commit_id_json_path = "runs/wokwi/final/commit_id.json" + with open(os.path.join(self.local_dir, commit_id_json_path), "w") as f: + json.dump( + { + "app": f"Tiny Tapeout {tt_version}", + "repo": repo, + "commit": commit_hash, + "workflow_url": workflow_url, + }, + f, + indent=2, + ) f.write("\n") - + # Copy user provided picture (if exists) to project directory self.copy_picture_for_docs() @@ -460,18 +513,28 @@ class Project(): def check_yaml_docs(self): yaml = self.yaml logging.info("checking docs") - for key in ['author', 'title', 'description', 'how_it_works', 'how_to_test', 'language', 'inputs', 'outputs', 'bidirectional']: - if key not in yaml['documentation']: + for key in [ + "author", + "title", + "description", + "how_it_works", + "how_to_test", + "language", + "inputs", + "outputs", + "bidirectional", + ]: + if key not in yaml["documentation"]: logging.error("missing key {} in documentation".format(key)) exit(1) - if yaml['documentation'][key] == "": + if yaml["documentation"][key] == "": logging.error("missing value for {} in documentation".format(key)) exit(1) # use pandoc to create a single page PDF preview def create_pdf(self): yaml = self.yaml - yaml = yaml['documentation'] + yaml = yaml["documentation"] logging.info("creating pdf") script_dir = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(script_dir, "docs/project_header.md")) as fh: @@ -479,13 +542,13 @@ class Project(): with open(os.path.join(script_dir, "docs/project_preview.md")) as fh: doc_template = fh.read() - with open('datasheet.md', 'w') as fh: + with open("datasheet.md", "w") as fh: fh.write(doc_header) # handle pictures - yaml['picture_link'] = '' - if yaml['picture']: - picture_name = yaml['picture'] - yaml['picture_link'] = '![picture]({})'.format(picture_name) + yaml["picture_link"] = "" + if yaml["picture"]: + picture_name = yaml["picture"] + yaml["picture_link"] = "![picture]({})".format(picture_name) # now build the doc & print it try: @@ -495,17 +558,19 @@ class Project(): except IndexError: logging.warning("missing pins in info.yaml, skipping") - pdf_cmd = 'pandoc --pdf-engine=xelatex -i datasheet.md -o datasheet.pdf' + pdf_cmd = "pandoc --pdf-engine=xelatex -i datasheet.md -o datasheet.pdf" logging.info(pdf_cmd) p = subprocess.run(pdf_cmd, shell=True) if p.returncode != 0: logging.error("pdf command failed") - + # Read and return top-level GDS data from the final GDS file, using gdstk: def get_final_gds_top_cells(self): - gds = glob.glob(os.path.join(self.local_dir, 'runs/wokwi/results/final/gds/*gds')) + gds = glob.glob( + os.path.join(self.local_dir, "runs/wokwi/results/final/gds/*gds") + ) if self.args.openlane2: - gds = glob.glob(os.path.join(self.local_dir, 'runs/wokwi/final/gds/*gds')) + gds = glob.glob(os.path.join(self.local_dir, "runs/wokwi/final/gds/*gds")) library = gdstk.read_gds(gds[0]) top_cells = library.top_level() return top_cells[0] @@ -514,7 +579,7 @@ class Project(): # NOTE: This includes all standard GDS layers inc. text labels. def create_svg(self): top_cells = self.get_final_gds_top_cells() - top_cells.write_svg('gds_render.svg') + top_cells.write_svg("gds_render.svg") # Try various QUICK methods to create a more-compressed PNG render of the GDS, # and fall back to cairosvg if it doesn't work. This is designed for speed, @@ -522,177 +587,216 @@ class Project(): # For more info, see: # https://github.com/TinyTapeout/tt-gds-action/issues/8 def create_png(self): - - logging.info('Loading GDS data...') + logging.info("Loading GDS data...") top_cells = self.get_final_gds_top_cells() # Remove label layers (i.e. delete text), then generate SVG: sky130_label_layers = [ - (64,59), # 64/59 - pwell.label - (64,5), # 64/5 - nwell.label - (67,5), # 67/5 - li1.label - (68,5), # 68/5 - met1.label - (69,5), # 69/5 - met2.label - (70,5), # 70/5 - met3.label - (71,5), # 71/5 - met4.label + (64, 59), # 64/59 - pwell.label + (64, 5), # 64/5 - nwell.label + (67, 5), # 67/5 - li1.label + (68, 5), # 68/5 - met1.label + (69, 5), # 69/5 - met2.label + (70, 5), # 70/5 - met3.label + (71, 5), # 71/5 - met4.label ] top_cells.filter(sky130_label_layers) - svg = 'gds_render_preview.svg' - logging.info('Rendering SVG without text label layers: {}'.format(svg)) + svg = "gds_render_preview.svg" + logging.info("Rendering SVG without text label layers: {}".format(svg)) top_cells.write_svg(svg, pad=0) # We should now have gds_render_preview.svg # Try converting using 'rsvg-convert' command-line utility. # This should create gds_render_preview.png - png = 'gds_render_preview.png' - logging.info('Converting to PNG using rsvg-convert: {}'.format(png)) + png = "gds_render_preview.png" + logging.info("Converting to PNG using rsvg-convert: {}".format(png)) - cmd = 'rsvg-convert --unlimited {} -o {} --no-keep-image-data'.format(svg, png) + cmd = "rsvg-convert --unlimited {} -o {} --no-keep-image-data".format(svg, png) logging.debug(cmd) p = subprocess.run(cmd, shell=True, capture_output=True) if p.returncode == 127: - logging.warning('rsvg-convert not found; is package "librsvg2-bin" installed? Falling back to cairosvg. This might take a while...') + logging.warning( + 'rsvg-convert not found; is package "librsvg2-bin" installed? Falling back to cairosvg. This might take a while...' + ) # Fall back to cairosvg: cairosvg.svg2png(url=svg, write_to=png) - elif p.returncode != 0 and b'cannot load more than' in p.stderr: - logging.warning('Too many SVG elements ("{}"). Reducing complexity...'.format(p.stderr.decode().strip())) + elif p.returncode != 0 and b"cannot load more than" in p.stderr: + logging.warning( + 'Too many SVG elements ("{}"). Reducing complexity...'.format( + p.stderr.decode().strip() + ) + ) # Remove more layers that are hardly visible anyway: sky130_buried_layers = [ - (64,16), # 64/16 - nwell.pin - (65,44), # 65/44 - tap.drawing - (68,16), # 68/16 - met1.pin - (68,44), # 68/44 - via.drawing - (81,4 ), # 81/4 - areaid.standardc - (70,20), # 70/20 - met3.drawing + (64, 16), # 64/16 - nwell.pin + (65, 44), # 65/44 - tap.drawing + (68, 16), # 68/16 - met1.pin + (68, 44), # 68/44 - via.drawing + (81, 4), # 81/4 - areaid.standardc + (70, 20), # 70/20 - met3.drawing # Important: # (68,20), # 68/20 - met1.drawing # Less important, but keep for now: # (69,20), # 69/20 - met2.drawing ] top_cells.filter(sky130_buried_layers) - svg_alt = 'gds_render_preview_alt.svg' - logging.info('Rendering SVG with more layers removed: {}'.format(svg_alt)) + svg_alt = "gds_render_preview_alt.svg" + logging.info("Rendering SVG with more layers removed: {}".format(svg_alt)) top_cells.write_svg(svg_alt, pad=0) - logging.info('Converting to PNG using rsvg-convert: {}'.format(png)) + logging.info("Converting to PNG using rsvg-convert: {}".format(png)) - cmd = 'rsvg-convert --unlimited {} -o {} --no-keep-image-data'.format(svg_alt, png) + cmd = "rsvg-convert --unlimited {} -o {} --no-keep-image-data".format( + svg_alt, png + ) logging.debug(cmd) p = subprocess.run(cmd, shell=True, capture_output=True) if p.returncode != 0: - logging.warning('Still cannot convert to SVG ("{}"). Falling back to cairosvg. This might take a while...'.format(p.stderr.decode().strip())) + logging.warning( + 'Still cannot convert to SVG ("{}"). Falling back to cairosvg. This might take a while...'.format( + p.stderr.decode().strip() + ) + ) # Fall back to cairosvg, and since we're doing that, might as well use the original full-detail SVG anyway: cairosvg.svg2png(url=svg, write_to=png) # By now we should have gds_render_preview.png # Compress with pngquant: - final_png = 'gds_render.png' - if self.yaml['project']['tiles'] == '8x2': - quality = '0-10' # Compress more for 8x2 tiles. + final_png = "gds_render.png" + if self.yaml["project"]["tiles"] == "8x2": + quality = "0-10" # Compress more for 8x2 tiles. else: - quality = '0-30' - logging.info('Compressing PNG further with pngquant to: {}'.format(final_png)) + quality = "0-30" + logging.info("Compressing PNG further with pngquant to: {}".format(final_png)) - cmd = 'pngquant --quality {} --speed 1 --nofs --strip --force --output {} {}'.format(quality, final_png, png) + cmd = "pngquant --quality {} --speed 1 --nofs --strip --force --output {} {}".format( + quality, final_png, png + ) logging.debug(cmd) p = subprocess.run(cmd, shell=True, capture_output=True) - - if p.returncode == 127: - logging.warning('pngquant not found; is package "pngquant" installed? Using intermediate (uncompressed) PNG file') - os.rename(png, final_png) - elif p.returncode !=0: - logging.warning('pngquant error {} ("{}"). Using intermediate (uncompressed) PNG file'.format(p.returncode, p.stderr.decode().strip())) - os.rename(png, final_png) - logging.info('Final PNG is {} ({:,} bytes)'.format(final_png, os.path.getsize(final_png))) + if p.returncode == 127: + logging.warning( + 'pngquant not found; is package "pngquant" installed? Using intermediate (uncompressed) PNG file' + ) + os.rename(png, final_png) + elif p.returncode != 0: + logging.warning( + 'pngquant error {} ("{}"). Using intermediate (uncompressed) PNG file'.format( + p.returncode, p.stderr.decode().strip() + ) + ) + os.rename(png, final_png) + logging.info( + "Final PNG is {} ({:,} bytes)".format(final_png, os.path.getsize(final_png)) + ) def print_warnings(self): warnings: typing.List[str] = [] - synth_log = 'runs/wokwi/logs/synthesis/1-synthesis.log' + synth_log = "runs/wokwi/logs/synthesis/1-synthesis.log" if self.args.openlane2: - synth_log = 'runs/wokwi/05-yosys-synthesis/yosys-synthesis.log' + synth_log = "runs/wokwi/05-yosys-synthesis/yosys-synthesis.log" with open(os.path.join(self.local_dir, synth_log)) as f: for line in f.readlines(): - if line.startswith('Warning:'): + if line.startswith("Warning:"): # bogus warning https://github.com/YosysHQ/yosys/commit/bfacaddca8a2e113e4bc3d6177612ccdba1555c8 - if 'WIDTHLABEL' not in line: + if "WIDTHLABEL" not in line: warnings.append(line.strip()) if len(warnings): - print('# Synthesis warnings') + print("# Synthesis warnings") print() for warning in warnings: - print(f'* {warning}') + print(f"* {warning}") def print_stats(self): - print('# Routing stats') + print("# Routing stats") print() - print('| Utilisation (%) | Wire length (um) |') - print('|-------------|------------------|') - print('| {} | {} |'.format(self.metrics['OpenDP_Util'], self.metrics['wire_length'])) + print("| Utilisation (%) | Wire length (um) |") + print("|-------------|------------------|") + print( + "| {} | {} |".format( + self.metrics["OpenDP_Util"], self.metrics["wire_length"] + ) + ) # Print the summaries def summarize(self): cell_count = self.get_cell_counts_from_gl() script_dir = os.path.dirname(os.path.realpath(__file__)) - Categories = typing.TypedDict("Categories", {'categories': typing.List[str], 'map': typing.Dict[str, int]}) - with open(os.path.join(script_dir, 'categories.json')) as fh: + Categories = typing.TypedDict( + "Categories", {"categories": typing.List[str], "map": typing.Dict[str, int]} + ) + with open(os.path.join(script_dir, "categories.json")) as fh: categories: Categories = json.load(fh) - + defs = load_cells() # print all used cells, sorted by frequency total = 0 if self.args.print_cell_summary: - print('# Cell usage') + print("# Cell usage") print() - print('| Cell Name | Description | Count |') - print('|-----------|-------------|-------|') - for name, count in sorted(cell_count.items(), key=lambda item: item[1], reverse=True): + print("| Cell Name | Description | Count |") + print("|-----------|-------------|-------|") + for name, count in sorted( + cell_count.items(), key=lambda item: item[1], reverse=True + ): if count > 0: total += count - cell_link = f'{CELL_URL}{name}' - print(f'| [{name}]({cell_link}) | {defs[name]["description"]} |{count} |') + cell_link = f"{CELL_URL}{name}" + print( + f'| [{name}]({cell_link}) | {defs[name]["description"]} |{count} |' + ) - print(f'| | Total | {total} |') + print(f"| | Total | {total} |") if self.args.print_cell_category: - CategoryInfo = typing.TypedDict("CategoryInfo", {'count': int, 'examples': typing.List[str]}) + CategoryInfo = typing.TypedDict( + "CategoryInfo", {"count": int, "examples": typing.List[str]} + ) by_category: typing.Dict[str, CategoryInfo] = {} total = 0 for cell_name in cell_count: - cat_index = categories['map'][cell_name] - cat_name = categories['categories'][cat_index] + cat_index = categories["map"][cell_name] + cat_name = categories["categories"][cat_index] if cat_name in by_category: - by_category[cat_name]['count'] += cell_count[cell_name] - by_category[cat_name]['examples'].append(cell_name) + by_category[cat_name]["count"] += cell_count[cell_name] + by_category[cat_name]["examples"].append(cell_name) else: - by_category[cat_name] = {'count' : cell_count[cell_name], 'examples' : [cell_name]} + by_category[cat_name] = { + "count": cell_count[cell_name], + "examples": [cell_name], + } - if cat_name not in ['Fill', 'Tap']: + if cat_name not in ["Fill", "Tap"]: total += cell_count[cell_name] - print('# Cell usage by Category') + print("# Cell usage by Category") print() - print('| Category | Cells | Count |') - print('|---------------|----------|-------|') - for cat_name, cat_dict in sorted(by_category.items(), key=lambda x: x[1]['count'], reverse=True): - cell_links = [f'[{name}]({CELL_URL}{name})' for name in cat_dict['examples']] + print("| Category | Cells | Count |") + print("|---------------|----------|-------|") + for cat_name, cat_dict in sorted( + by_category.items(), key=lambda x: x[1]["count"], reverse=True + ): + cell_links = [ + f"[{name}]({CELL_URL}{name})" for name in cat_dict["examples"] + ] print(f'|{cat_name} | {" ".join(cell_links)} | {cat_dict["count"]}|') - print(f'## {total} total cells (excluding fill and tap cells)') + print(f"## {total} total cells (excluding fill and tap cells)") # get cell count from synth report def get_cell_count_from_synth(self): num_cells = 0 try: - yosys_report = f'{self.local_dir}/synthesis-stats.txt' + yosys_report = f"{self.local_dir}/synthesis-stats.txt" with open(yosys_report) as fh: for line in fh.readlines(): - m = re.search(r'Number of cells:\s+(\d+)', line) + m = re.search(r"Number of cells:\s+(\d+)", line) if m is not None: num_cells = int(m.group(1)) @@ -707,30 +811,30 @@ class Project(): total = 0 with open(self.get_gl_path()) as fh: for line in fh.readlines(): - m = re.search(r'sky130_(\S+)__(\S+)_(\d+)', line) + m = re.search(r"sky130_(\S+)__(\S+)_(\d+)", line) if m is not None: total += 1 cell_lib = m.group(1) cell_name = m.group(2) cell_drive = m.group(3) - assert cell_lib in ['fd_sc_hd', 'ef_sc_hd'] + assert cell_lib in ["fd_sc_hd", "ef_sc_hd"] assert int(cell_drive) > 0 try: cell_count[cell_name] += 1 except KeyError: cell_count[cell_name] = 1 - return (cell_count) + return cell_count # Load all the json defs and combine into one big dict, keyed by cellname # Needs access to the PDK def create_defs(self): # replace this with a find - json_files = glob.glob('sky130_fd_sc_hd/latest/cells/*/definition.json') + json_files = glob.glob("sky130_fd_sc_hd/latest/cells/*/definition.json") definitions: typing.Dict[str, Cell] = {} for json_file in json_files: with open(json_file) as fh: definition: Cell = json.load(fh) - definitions[definition['name']] = definition + definitions[definition["name"]] = definition - with open('cells.json', 'w') as fh: + with open("cells.json", "w") as fh: json.dump(definitions, fh) diff --git a/rom.py b/rom.py index 1f4b6ed..28e9c30 100644 --- a/rom.py +++ b/rom.py @@ -1,6 +1,8 @@ -from git.repo import Repo import os from urllib.parse import urlparse + +from git.repo import Repo + from config import Config MAX_ROM_TEXT_SIZE = 92 @@ -26,9 +28,11 @@ segment_font = { "t": 0b01111000, } + def segment_char(c: str): return segment_font[c] + class ROMFile: def __init__(self, config: Config): self.config = config @@ -53,7 +57,7 @@ class ROMFile: assert len(rom_text) < MAX_ROM_TEXT_SIZE, "ROM text too long" - rom[0:4] = map(segment_char, self.config['id']) + rom[0:4] = map(segment_char, self.config["id"]) rom[8:16] = map(segment_char, short_sha.upper()) rom[32 : 32 + len(rom_text)] = rom_text.encode("ascii") diff --git a/shuttle.py b/shuttle.py index 62fa246..eaccd00 100644 --- a/shuttle.py +++ b/shuttle.py @@ -1,10 +1,12 @@ +import glob +import json import logging import os -import yaml -import json -import glob import shutil + import git +import yaml + def copy_print(src: str, dest: str): os.makedirs(os.path.dirname(dest), exist_ok=True) @@ -17,8 +19,7 @@ def copy_print_glob(pattern: str, dest_dir: str): copy_print(file, os.path.join(dest_dir, os.path.basename(file))) -class ShuttleConfig(): - +class ShuttleConfig: def __init__(self, config, projects, modules_yaml_name: str): self.config = config self.projects = projects @@ -26,21 +27,27 @@ class ShuttleConfig(): self.modules_yaml_name = modules_yaml_name def configure_mux(self): - with open(self.modules_yaml_name, 'r') as modules_file: + with open(self.modules_yaml_name, "r") as modules_file: module_config = yaml.safe_load(modules_file) - configured_macros = set(map(lambda mod: mod['name'], module_config['modules'])) - logging.info(f"found {len(configured_macros)} preconfigured macros: {configured_macros}") + configured_macros = set( + map(lambda mod: mod["name"], module_config["modules"]) + ) + logging.info( + f"found {len(configured_macros)} preconfigured macros: {configured_macros}" + ) for project in self.projects: - tiles = project.yaml['project']['tiles'] - width, height = map(int, tiles.split('x')) + tiles = project.yaml["project"]["tiles"] + width, height = map(int, tiles.split("x")) if project.unprefixed_name not in configured_macros: - module_config['modules'].append({ - 'name': project.unprefixed_name, - 'width': width, - 'height': height, - }) + module_config["modules"].append( + { + "name": project.unprefixed_name, + "width": width, + "height": height, + } + ) - with open('tt-multiplexer/cfg/modules.yaml', 'w') as mux_modules_file: + with open("tt-multiplexer/cfg/modules.yaml", "w") as mux_modules_file: yaml.dump(module_config, mux_modules_file) res = os.system("make -C tt-multiplexer gensrc") @@ -50,29 +57,29 @@ class ShuttleConfig(): mux_index = {} mux_index_reverse = {} - with open('tt-multiplexer/cfg/modules_placed.yaml') as placed_modules_file: + with open("tt-multiplexer/cfg/modules_placed.yaml") as placed_modules_file: placed_modules = yaml.safe_load(placed_modules_file) - for module in placed_modules['modules']: - mux_address = (module['y'] << 5) + module['x'] - module_name = 'tt_um_' + module['name'] + for module in placed_modules["modules"]: + mux_address = (module["y"] << 5) + module["x"] + module_name = "tt_um_" + module["name"] project = next(p for p in self.projects if p.top_module == module_name) project.mux_address = mux_address mux_index[mux_address] = { "macro": module_name, - "x": module['x'], - "y": module['y'], + "x": module["x"], + "y": module["y"], "tiles": f"{module['width']}x{module['height']}", "repo": project.git_url, "commit": project.commit_id, } mux_index_reverse[module_name] = mux_address - + for project in self.projects: if project.top_module not in mux_index_reverse: logging.error(f"no placement found for {project}!") exit(1) project.mux_address = mux_index_reverse[project.top_module] - + repo = git.Repo(".") shuttle_index_data = { @@ -83,20 +90,23 @@ class ShuttleConfig(): "mux": mux_index, } - with open('shuttle_index.json', 'w') as shuttle_index_file: + with open("shuttle_index.json", "w") as shuttle_index_file: json.dump(shuttle_index_data, shuttle_index_file, indent=2) def list(self): for project in self.projects: logging.info(project) - def find_last_run(self, macro: str): runs = f"tt-multiplexer/ol2/{macro}/runs/" if macro == "tt_um_chip_rom": runs = "tt/rom/runs/" runlist = sorted( - [r for r in os.listdir(runs) if r.startswith("RUN_") and os.path.isdir(os.path.join(runs, r))] + [ + r + for r in os.listdir(runs) + if r.startswith("RUN_") and os.path.isdir(os.path.join(runs, r)) + ] ) if len(runlist) == 0: print(f"Error: no runs found for {macro}") @@ -104,25 +114,34 @@ class ShuttleConfig(): return os.path.join(runs, runlist[-1]) - def copy_macros(self): - logging.info('copying macros to tt_top:') + logging.info("copying macros to tt_top:") copy_print_glob("projects/*/*.gds", "tt-multiplexer/ol2/tt_top/gds") copy_print_glob("projects/*/*.lef", "tt-multiplexer/ol2/tt_top/lef") copy_print_glob("projects/*/*.v", "tt-multiplexer/ol2/tt_top/verilog") macros = ["tt_um_chip_rom", "tt_ctrl", "tt_mux"] for macro in macros: lastrun = self.find_last_run(macro) - copy_print(f"{lastrun}/final/gds/{macro}.gds", f"tt-multiplexer/ol2/tt_top/gds/{macro}.gds") - copy_print(f"{lastrun}/final/lef/{macro}.lef", f"tt-multiplexer/ol2/tt_top/lef/{macro}.lef") - copy_print(f"{lastrun}/final/nl/{macro}.nl.v", f"tt-multiplexer/ol2/tt_top/verilog/{macro}.v") - copy_print_glob(f"{lastrun}/final/spef/*/*.spef", "tt-multiplexer/ol2/tt_top/spef") - + copy_print( + f"{lastrun}/final/gds/{macro}.gds", + f"tt-multiplexer/ol2/tt_top/gds/{macro}.gds", + ) + copy_print( + f"{lastrun}/final/lef/{macro}.lef", + f"tt-multiplexer/ol2/tt_top/lef/{macro}.lef", + ) + copy_print( + f"{lastrun}/final/nl/{macro}.nl.v", + f"tt-multiplexer/ol2/tt_top/verilog/{macro}.v", + ) + copy_print_glob( + f"{lastrun}/final/spef/*/*.spef", "tt-multiplexer/ol2/tt_top/spef" + ) def copy_final_results(self): macros = ["tt_um_chip_rom", "tt_ctrl", "tt_mux", "tt_top"] - logging.info('copying final results:') + logging.info("copying final results:") for macro in macros: lastrun = self.find_last_run(macro) macro_name = macro if macro != "tt_top" else "user_project_wrapper" @@ -130,12 +149,22 @@ class ShuttleConfig(): logging.info(f" FROM {lastrun}") copy_print(f"{lastrun}/final/gds/{macro_name}.gds", f"gds/{macro_name}.gds") copy_print(f"{lastrun}/final/lef/{macro_name}.lef", f"lef/{macro_name}.lef") - copy_print(f"{lastrun}/final/pnl/{macro_name}.pnl.v", f"verilog/gl/{macro_name}.v") - copy_print(f"{lastrun}/final/nl/{macro_name}.nl.v", f"verilog/gl/{macro_name}.nl.v") + copy_print( + f"{lastrun}/final/pnl/{macro_name}.pnl.v", f"verilog/gl/{macro_name}.v" + ) + copy_print( + f"{lastrun}/final/nl/{macro_name}.nl.v", f"verilog/gl/{macro_name}.nl.v" + ) shutil.copytree( - f"{lastrun}/final/spef", f"spef/", copy_function=copy_print, dirs_exist_ok=True + f"{lastrun}/final/spef", + f"spef/", + copy_function=copy_print, + dirs_exist_ok=True, + ) + copy_print( + f"{lastrun}/final/spef/nom_/{macro_name}.nom_.spef", + f"spef/{macro_name}.spef", ) - copy_print(f"{lastrun}/final/spef/nom_/{macro_name}.nom_.spef", f"spef/{macro_name}.spef") def create_efabless_submission(self): logging.info("creating efabless submission directory:") @@ -143,6 +172,15 @@ class ShuttleConfig(): copy_print("README.md", "efabless/README.md") copy_print("shuttle_index.json", "efabless/shuttle_index.json") copy_print("verilog/rtl/user_defines.v", "efabless/verilog/rtl/user_defines.v") - copy_print(f"{lastrun}/final/pnl/user_project_wrapper.pnl.v", f"efabless/verilog/gl/user_project_wrapper.v") - copy_print(f"{lastrun}/final/gds/user_project_wrapper.gds", f"efabless/gds/user_project_wrapper.gds") - copy_print(f"{lastrun}/final/lef/user_project_wrapper.lef", f"efabless/lef/user_project_wrapper.lef") + copy_print( + f"{lastrun}/final/pnl/user_project_wrapper.pnl.v", + f"efabless/verilog/gl/user_project_wrapper.v", + ) + copy_print( + f"{lastrun}/final/gds/user_project_wrapper.gds", + f"efabless/gds/user_project_wrapper.gds", + ) + copy_print( + f"{lastrun}/final/lef/user_project_wrapper.lef", + f"efabless/lef/user_project_wrapper.lef", + ) diff --git a/testing/lib/testutils/truthtable.py b/testing/lib/testutils/truthtable.py index 2f222bd..8e8afed 100644 --- a/testing/lib/testutils/truthtable.py +++ b/testing/lib/testutils/truthtable.py @@ -1,17 +1,17 @@ -''' - TruthTable testing - Copyright (C) 2023 Pat Deegan, https://psychogenic.com +""" + TruthTable testing + Copyright (C) 2023 Pat Deegan, https://psychogenic.com - A simple system to take truth table data from yosys, like + A simple system to take truth table data from yosys, like - ui_in | uo_out - ---------- | ----------- - 8'11111000 | 8'xxxxxxx0 - 8'11111001 | 8'xxxxxxx1 - 8'11111010 | 8'xxxxxxx0 - ... - - or manually crafted combi- or clocked sequences, encapsulated in + ui_in | uo_out + ---------- | ----------- + 8'11111000 | 8'xxxxxxx0 + 8'11111001 | 8'xxxxxxx1 + 8'11111010 | 8'xxxxxxx0 + ... + + or manually crafted combi- or clocked sequences, encapsulated in markdown tables such as |IN: CBA RC | output | comment | @@ -24,479 +24,462 @@ | --- --- tc | -- 11100 - | locked | ... - and run it through cocotb harness. - + and run it through cocotb harness. + SAMPLE USAGE: - @cocotb.test() - async def truthTableCompare(dut): - i_bus = dut.ui_in - o_bus = dut.uo_out - tt = truthtable.loadSimpleTruthTable('test/truthtable.txt') - await tt.testAll(i_bus, o_bus, dut._log) - + @cocotb.test() + async def truthTableCompare(dut): + i_bus = dut.ui_in + o_bus = dut.uo_out + tt = truthtable.loadSimpleTruthTable('test/truthtable.txt') + await tt.testAll(i_bus, o_bus, dut._log) + SPDX-FileCopyrightText: © 2023 Pat Deegan, https://psychogenic.com SPDX-License-Identifier: Apache2.0 -''' +""" -from string import Template as StringTemplate -from cocotb.triggers import Timer -from cocotb.binary import BinaryValue +import logging import re -import logging +from string import Template as StringTemplate + +from cocotb.binary import BinaryValue +from cocotb.triggers import Timer log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) class SaneBinaryValue(BinaryValue): - def __init__(self, vstr:str): - super().__init__(vstr, n_bits=len(vstr), bigEndian=False) - numbits = len(vstr) - self.hardBit = [True]*numbits - for i in range(numbits): - c = vstr[i] - if c != '1' and c != '0': - self.hardBit[(numbits - 1) - i] = False + def __init__(self, vstr: str): + super().__init__(vstr, n_bits=len(vstr), bigEndian=False) + numbits = len(vstr) + self.hardBit = [True] * numbits + for i in range(numbits): + c = vstr[i] + if c != "1" and c != "0": + self.hardBit[(numbits - 1) - i] = False + class TruthMapping: - def __init__(self, resultingValue:str): - self.result = SaneBinaryValue(resultingValue) - - - def __str__(self): - return f'Expecting {self.result.binstr}' - - - - + def __init__(self, resultingValue: str): + self.result = SaneBinaryValue(resultingValue) + + def __str__(self): + return f"Expecting {self.result.binstr}" + + class TruthTable: - ''' - Holds the sequence/expected mapping of inputs -> outputs - and provides an async testAll() to perform the testing. - ''' - def __init__(self, interStepDelay:int=10, interStepTimeUnits:str='ns'): - self.mappings = [] - self.stepDelayTime = interStepDelay - self.stepDelayUnits = interStepTimeUnits - - - def addMapping(self, relationship:TruthMapping): - self.mappings.append(relationship) - - def getMapping(self, idx:int) -> TruthMapping: - if idx >= len(self.mappings): - raise IndexError('out of bounds on mapping') - - return self.mappings[idx] - - def numMappings(self): - return len(self.mappings) - - def __len__(self): - return self.numMappings() - - def __getitem__(self, idx:int): - return self.getMapping(idx) - - def __str__(self): - numsteps = len(self) - retStrs = [f"Truthtable with {numsteps} mappings"] - for i in range(numsteps): - step = self[i] - retStrs.append(f'{i + 1}\t{str(step)}') - - return '\n'.join(retStrs) - - def dump(self): - print(str(self)) - - - async def testAll(self, i_bus, o_bus, logger=None): - for i in range(len(self)): - - i_bus.value = self[i].state - await Timer(self.stepDelayTime, units=self.stepDelayUnits) # wait a tad - - logger.debug(f'Set i_bus to {i_bus.value} vs {self[i].state}') - for idx in range(8): - logger.debug(f'{i}-{idx}: i_bus[{idx}] == {i_bus[idx]} == {self[i].state[idx]}') - - expectedResult = self[i].result - if logger is not None: - logger.info(f'State {i}, setting input to {self[i].state}, expecting {expectedResult} (got {o_bus.value})') - - # this is so stupid that it's probably wrong... there _must_ be - # a good way to do this, otherwise _what_ is the point of having - # unknown and don't care... but anyway, - # will die if o_bus[n] is some defined value - # but we expect a don't know (x) or even a don't care (-), ugh - # manual style - for bit in range(len(o_bus)): - if expectedResult.hardBit[bit]: - # ok, safe to compare, duh - # logger.info(f'Doing bit {bit}: {o_bus[bit]} == {expectedResult[bit]}') - assert o_bus[bit] == expectedResult[bit] - - + """ + Holds the sequence/expected mapping of inputs -> outputs + and provides an async testAll() to perform the testing. + """ + + def __init__(self, interStepDelay: int = 10, interStepTimeUnits: str = "ns"): + self.mappings = [] + self.stepDelayTime = interStepDelay + self.stepDelayUnits = interStepTimeUnits + + def addMapping(self, relationship: TruthMapping): + self.mappings.append(relationship) + + def getMapping(self, idx: int) -> TruthMapping: + if idx >= len(self.mappings): + raise IndexError("out of bounds on mapping") + + return self.mappings[idx] + + def numMappings(self): + return len(self.mappings) + + def __len__(self): + return self.numMappings() + + def __getitem__(self, idx: int): + return self.getMapping(idx) + + def __str__(self): + numsteps = len(self) + retStrs = [f"Truthtable with {numsteps} mappings"] + for i in range(numsteps): + step = self[i] + retStrs.append(f"{i + 1}\t{str(step)}") + + return "\n".join(retStrs) + + def dump(self): + print(str(self)) + + async def testAll(self, i_bus, o_bus, logger=None): + for i in range(len(self)): + i_bus.value = self[i].state + await Timer(self.stepDelayTime, units=self.stepDelayUnits) # wait a tad + + logger.debug(f"Set i_bus to {i_bus.value} vs {self[i].state}") + for idx in range(8): + logger.debug( + f"{i}-{idx}: i_bus[{idx}] == {i_bus[idx]} == {self[i].state[idx]}" + ) + + expectedResult = self[i].result + if logger is not None: + logger.info( + f"State {i}, setting input to {self[i].state}, expecting {expectedResult} (got {o_bus.value})" + ) + + # this is so stupid that it's probably wrong... there _must_ be + # a good way to do this, otherwise _what_ is the point of having + # unknown and don't care... but anyway, + # will die if o_bus[n] is some defined value + # but we expect a don't know (x) or even a don't care (-), ugh + # manual style + for bit in range(len(o_bus)): + if expectedResult.hardBit[bit]: + # ok, safe to compare, duh + # logger.info(f'Doing bit {bit}: {o_bus[bit]} == {expectedResult[bit]}') + assert o_bus[bit] == expectedResult[bit] + + class OneToOneTruthMapping(TruthMapping): - def __init__(self, state:str, output:str): - super().__init__(output) - self.state = SaneBinaryValue(state) + def __init__(self, state: str, output: str): + super().__init__(output) + self.state = SaneBinaryValue(state) + def __str__(self): + return f"Set {self.state.binstr}, expect {self.result.binstr}" - def __str__(self): - return f'Set {self.state.binstr}, expect {self.result.binstr}' - - class TestTableEntryParser: - ''' - A little class that validates and cleans up - input and output specifications. - ''' - - def __init__(self, logger): - self.log = logger - self.optionalPrefix = r"(\d'|0b)?" - self.validInputValues = r'(\s*[01tTcCxX-]\s*){8}' # 0/1 hard-coded, t/T toggle, c/C clocking, x or - unchanged - self.validOutputValues = r'(\s*[01xX-]\s*){8}' # 0/1 hard-coded,x or - don't care - - - valueMatchReTpl = StringTemplate(r'\s*$prefix($valmatch)') - self.inputValueValidatorRe = re.compile( - valueMatchReTpl.substitute( - prefix=self.optionalPrefix, valmatch=self.validInputValues)) - self.outputValueValidatorRe = re.compile( - valueMatchReTpl.substitute( - prefix=self.optionalPrefix, valmatch=self.validOutputValues)) + """ + A little class that validates and cleans up + input and output specifications. + """ + def __init__(self, logger): + self.log = logger + self.optionalPrefix = r"(\d'|0b)?" + self.validInputValues = r"(\s*[01tTcCxX-]\s*){8}" # 0/1 hard-coded, t/T toggle, c/C clocking, x or - unchanged + self.validOutputValues = ( + r"(\s*[01xX-]\s*){8}" # 0/1 hard-coded,x or - don't care + ) + valueMatchReTpl = StringTemplate(r"\s*$prefix($valmatch)") + self.inputValueValidatorRe = re.compile( + valueMatchReTpl.substitute( + prefix=self.optionalPrefix, valmatch=self.validInputValues + ) + ) + self.outputValueValidatorRe = re.compile( + valueMatchReTpl.substitute( + prefix=self.optionalPrefix, valmatch=self.validOutputValues + ) + ) - def validateAndClean(self, val:str, usingRegex): - - if val is None or not len(val): - return None - - - mtch = usingRegex.match(val) - - - if mtch is None: - vcln = val.strip() - if not len(vcln): - # just whitespace - return None - - if vcln.find('#') >= 0 or vcln.find('//') >=0: - # just a comment - return None - - if vcln.find('1') >=0 or vcln.find('0') >=0 or vcln.find('x') >= 0: - self.log.error(f"Bad value passed ? '{val}'") - - return None - - return re.sub(r'\s+', '', mtch.group(0)).lower() + def validateAndClean(self, val: str, usingRegex): + if val is None or not len(val): + return None - def inputFrom(self, val:str): - return self.validateAndClean(val, self.inputValueValidatorRe) + mtch = usingRegex.match(val) - def outputFrom(self, val:str): - if val is None or not len(val): - return None - return self.validateAndClean(val, self.outputValueValidatorRe) + if mtch is None: + vcln = val.strip() + if not len(vcln): + # just whitespace + return None + + if vcln.find("#") >= 0 or vcln.find("//") >= 0: + # just a comment + return None + + if vcln.find("1") >= 0 or vcln.find("0") >= 0 or vcln.find("x") >= 0: + self.log.error(f"Bad value passed ? '{val}'") + + return None + + return re.sub(r"\s+", "", mtch.group(0)).lower() + + def inputFrom(self, val: str): + return self.validateAndClean(val, self.inputValueValidatorRe) + + def outputFrom(self, val: str): + if val is None or not len(val): + return None + return self.validateAndClean(val, self.outputValueValidatorRe) class TestTableParser: - ''' - Base class for tabular configuration values for input sequence and - expected outputs. - - This class handles maintaining the truth table and the state between - steps in a sequence. - - Basically does everything except handle the particularities of a - give table format (e.g. markdown). - - ''' - - def __init__(self, logger=None, numIOBits:int=8): - if logger is None: - logger = logging.getLogger(__name__) - self.log = logger - self.numIOBits = numIOBits - self.valueParser = TestTableEntryParser(self.log) - - - self._truthTable = None - self._textTable = [] - - self._currentInput = SaneBinaryValue('0'*numIOBits) - self._ignoreOutput = SaneBinaryValue('-'*numIOBits) - - - def reset(self): - self._currentInput = SaneBinaryValue('00000000') - self._truthTable = None - self._textTable = [] - - @property - def truthTable(self) -> TruthTable: - if self._truthTable is None: - self._truthTable = TruthTable() - - return self._truthTable - - - - def _inputStateFromBits(self, inputBits): - - newBits = ['0'] * self.numIOBits - - for i in range(self.numIOBits): - - if inputBits[i] == '0' or inputBits[i] == '1': - newBits[i] = inputBits[i] - self._currentInput[i] = int(inputBits[i]) - elif inputBits[i] == 't': - # toggled - self._currentInput[i] = ~self._currentInput[i] - newBits[i] = str(self._currentInput[i]) - else: - # unchanged - newBits[i] = str(self._currentInput[i]) - - - return ''.join(newBits) - - - def addStep(self, inputValue:str, outputValue:str=''): - inputBits = self.valueParser.inputFrom(inputValue) - - if inputBits is None or not len(inputBits): - self.log.info(f'Unable to parse input bits: {inputValue}') - return False - - - if inputBits.find('c') >= 0: - # clocking... - setupInputs = inputBits.replace('c', 'x') - self.log.debug(f'Have clocking in inputs {inputBits}, adding setup step {setupInputs}') - self.addStep(setupInputs) # setup - - # clock edge 1 - - clockEdges = inputBits.replace('t', 'x') # don't toggle twice! - - clockInAndHold = clockEdges.replace('c', 't') - - self.log.debug(f'Now clock {clockEdges} x2') - - self.addStep(clockInAndHold) # clock in - - # clock edge 2 - self.addStep(clockInAndHold) # clock out - - - inputBinVal = SaneBinaryValue(self._inputStateFromBits(inputBits)) - - outputBinVal = self._ignoreOutput - - outputBits = '' - if outputValue is not None and len(outputValue): - self.log.info(f'outval is "{outputValue}"') - outputBits = self.valueParser.outputFrom(outputValue) - if outputBits is None or not len(outputBits): - self.log.info(f'Unable to parse output bits: {outputBits}') - return False - - outputBinVal = SaneBinaryValue(outputBits) - - - - - self.log.info(f'Mapped {inputBits} -> {outputBits} ({inputBinVal} -> {outputBinVal})') - - self.truthTable.addMapping(OneToOneTruthMapping(inputBinVal.binstr, outputBinVal.binstr)) - self._textTable.append((inputBits, outputBits)) - return True - - def generateFrom(self, contents:str): - self.reset() # reset the table - self.log.error("Override generateFrom() in subclass") - return None - - - def __str__(self): - retStrs = [f'Truth table parser with {len(self._textTable)} steps added.'] - for i in range(len(self._textTable)): - step = self._textTable[i] - if step[1] is not None and len(step[1]): - retStrs.append(f'{i+1}\t{step[0]} -> {step[1]}') - else: - retStrs.append(f'{i+1}\t{step[0]}') - - return '\n'.join(retStrs) - - def dump(self): - print(str(self)) - - - - + """ + Base class for tabular configuration values for input sequence and + expected outputs. + + This class handles maintaining the truth table and the state between + steps in a sequence. + + Basically does everything except handle the particularities of a + give table format (e.g. markdown). + + """ + + def __init__(self, logger=None, numIOBits: int = 8): + if logger is None: + logger = logging.getLogger(__name__) + self.log = logger + self.numIOBits = numIOBits + self.valueParser = TestTableEntryParser(self.log) + + self._truthTable = None + self._textTable = [] + + self._currentInput = SaneBinaryValue("0" * numIOBits) + self._ignoreOutput = SaneBinaryValue("-" * numIOBits) + + def reset(self): + self._currentInput = SaneBinaryValue("00000000") + self._truthTable = None + self._textTable = [] + + @property + def truthTable(self) -> TruthTable: + if self._truthTable is None: + self._truthTable = TruthTable() + + return self._truthTable + + def _inputStateFromBits(self, inputBits): + newBits = ["0"] * self.numIOBits + + for i in range(self.numIOBits): + if inputBits[i] == "0" or inputBits[i] == "1": + newBits[i] = inputBits[i] + self._currentInput[i] = int(inputBits[i]) + elif inputBits[i] == "t": + # toggled + self._currentInput[i] = ~self._currentInput[i] + newBits[i] = str(self._currentInput[i]) + else: + # unchanged + newBits[i] = str(self._currentInput[i]) + + return "".join(newBits) + + def addStep(self, inputValue: str, outputValue: str = ""): + inputBits = self.valueParser.inputFrom(inputValue) + + if inputBits is None or not len(inputBits): + self.log.info(f"Unable to parse input bits: {inputValue}") + return False + + if inputBits.find("c") >= 0: + # clocking... + setupInputs = inputBits.replace("c", "x") + self.log.debug( + f"Have clocking in inputs {inputBits}, adding setup step {setupInputs}" + ) + self.addStep(setupInputs) # setup + + # clock edge 1 + + clockEdges = inputBits.replace("t", "x") # don't toggle twice! + + clockInAndHold = clockEdges.replace("c", "t") + + self.log.debug(f"Now clock {clockEdges} x2") + + self.addStep(clockInAndHold) # clock in + + # clock edge 2 + self.addStep(clockInAndHold) # clock out + + inputBinVal = SaneBinaryValue(self._inputStateFromBits(inputBits)) + + outputBinVal = self._ignoreOutput + + outputBits = "" + if outputValue is not None and len(outputValue): + self.log.info(f'outval is "{outputValue}"') + outputBits = self.valueParser.outputFrom(outputValue) + if outputBits is None or not len(outputBits): + self.log.info(f"Unable to parse output bits: {outputBits}") + return False + + outputBinVal = SaneBinaryValue(outputBits) + + self.log.info( + f"Mapped {inputBits} -> {outputBits} ({inputBinVal} -> {outputBinVal})" + ) + + self.truthTable.addMapping( + OneToOneTruthMapping(inputBinVal.binstr, outputBinVal.binstr) + ) + self._textTable.append((inputBits, outputBits)) + return True + + def generateFrom(self, contents: str): + self.reset() # reset the table + self.log.error("Override generateFrom() in subclass") + return None + + def __str__(self): + retStrs = [f"Truth table parser with {len(self._textTable)} steps added."] + for i in range(len(self._textTable)): + step = self._textTable[i] + if step[1] is not None and len(step[1]): + retStrs.append(f"{i+1}\t{step[0]} -> {step[1]}") + else: + retStrs.append(f"{i+1}\t{step[0]}") + + return "\n".join(retStrs) + + def dump(self): + print(str(self)) + + class MarkdownTestTableParser(TestTableParser): - ''' - A specialization of the TestTableParser that handles the - particulars of extracting test inputs/outputs from a markdown - table. - ''' - - def __init__(self, columnSeparatorRegex:str=r'\|', logger=None): - ''' - We want to support markdown tables. - These may have some header row, followed by a separator (which may - or may not include aligment info, i.e. ':'), finally followed - by our content of interest. E.g. - - | some name | another | <- column headers - |-----------|---------:| <- header sep - | CONTENT | CONTENT | <- actual content of interest - - CONTENT is 8 bits, MSB - [7 ... 0] - A B C D E F G H - - Where each bit is either: - 0 - 1 - X (or x) don't change/don't care - T (or t) toggle - C (or c) clock - ... - - - ''' - super().__init__(logger) - - self.columnSeperatorRe = columnSeparatorRegex - eol = r'(\r\n|\r|\n)' - - - - # the |-------|----- ... line - tableHeaderSepTpl = StringTemplate(r'\s*($sep\s*:?-{1,}:?\s*){2,}$sep$eol') - self.headerSepRe = re.compile(tableHeaderSepTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M) + """ + A specialization of the TestTableParser that handles the + particulars of extracting test inputs/outputs from a markdown + table. + """ - # each content line thereafter - # | CONTENT | CONTENT | - contentRowTpl = StringTemplate(r'^\s*(($sep\s*[^$sep\r\n]+){1,})$sep$eol') - self.contentRowRe = re.compile(contentRowTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M) - - - self.rowItemSplitterRe = re.compile(self.columnSeperatorRe) - - def parseMarkdownTable(self, contents:str): - - headerSepSearch = self.headerSepRe.search(contents) - if headerSepSearch is not None: - # did have a header - # dump it, and everything before it - headerSpan = headerSepSearch.span() - headerEndPos = headerSpan[1] - self.log.debug(f"Dumping header to {headerEndPos}\n{contents[:headerEndPos]}") - contents = contents[headerEndPos:] - - # extract valid rows in first pass - self.log.debug(f"Content row RE is {self.contentRowRe}") - contentRowMatches = self.contentRowRe.findall(contents) - if contentRowMatches is None or not len(contentRowMatches): - self.log.error('No valid rows found in markdown table') - return False - - - numSteps = 0 - for aRow in contentRowMatches: - self.log.debug(f'Matched row {aRow}') - entries = self.rowItemSplitterRe.split(aRow[0]) - - # note, we have entries like - # SEP AAA SEP BBB SEP - # so splitting on SEP gives us an empty entry at front of list - # hence the requirement for len == 2 for at least 1 entry, len == 3 - # for input and output. - - if len(entries) < 2: - self.log.info(f'Row has insufficient entries ? ("{aRow[0]}")') - continue - - inVal = entries[1] - outVal = None - - if len(entries) > 2 and len(entries[2]): - outVal = entries[2].strip() - - if self.addStep(inVal, outVal): - numSteps += 1 - - - return numSteps - - - + def __init__(self, columnSeparatorRegex: str = r"\|", logger=None): + """ + We want to support markdown tables. + These may have some header row, followed by a separator (which may + or may not include aligment info, i.e. ':'), finally followed + by our content of interest. E.g. - def generateFrom(self, contents:str): - - self.reset() - - tableParsed = self.parseMarkdownTable(contents) - - if tableParsed is None or not tableParsed: - self.log.error("Could not extract truthtable from contents") - return None - - return self.truthTable - - - + | some name | another | <- column headers + |-----------|---------:| <- header sep + | CONTENT | CONTENT | <- actual content of interest + + CONTENT is 8 bits, MSB + [7 ... 0] + A B C D E F G H + + Where each bit is either: + 0 + 1 + X (or x) don't change/don't care + T (or t) toggle + C (or c) clock + ... + """ + super().__init__(logger) -def parseSimpleTable(contents:str): - tt = TruthTable() - m = re.compile(r'''^\s*\d'(\d+)\s*\|\s*\d'([zZxX\d]+)''', re.M) - for match in m.findall(contents): - if match[1] != 'x': - # we skip anything where all results are don't care - # result = match[1].replace('x', '-') # now using the BinaryValue override - result = match[1] - tt.addMapping(OneToOneTruthMapping(match[0],result)) - - return tt + self.columnSeperatorRe = columnSeparatorRegex + eol = r"(\r\n|\r|\n)" -def parseMarkdownTable(contents:str, logger=None): - m = MarkdownTestTableParser(logger=logger) - table = m.generateFrom(contents) - return table - - -def loadSimpleTruthTable(filepath:str): - with open(filepath, 'r') as f: - contents = f.read() - return parseSimpleTable(contents) - - return None + # the |-------|----- ... line + tableHeaderSepTpl = StringTemplate(r"\s*($sep\s*:?-{1,}:?\s*){2,}$sep$eol") + self.headerSepRe = re.compile( + tableHeaderSepTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M + ) + + # each content line thereafter + # | CONTENT | CONTENT | + contentRowTpl = StringTemplate(r"^\s*(($sep\s*[^$sep\r\n]+){1,})$sep$eol") + self.contentRowRe = re.compile( + contentRowTpl.substitute(sep=self.columnSeperatorRe, eol=eol), re.M + ) + + self.rowItemSplitterRe = re.compile(self.columnSeperatorRe) + + def parseMarkdownTable(self, contents: str): + headerSepSearch = self.headerSepRe.search(contents) + if headerSepSearch is not None: + # did have a header + # dump it, and everything before it + headerSpan = headerSepSearch.span() + headerEndPos = headerSpan[1] + self.log.debug( + f"Dumping header to {headerEndPos}\n{contents[:headerEndPos]}" + ) + contents = contents[headerEndPos:] + + # extract valid rows in first pass + self.log.debug(f"Content row RE is {self.contentRowRe}") + contentRowMatches = self.contentRowRe.findall(contents) + if contentRowMatches is None or not len(contentRowMatches): + self.log.error("No valid rows found in markdown table") + return False + + numSteps = 0 + for aRow in contentRowMatches: + self.log.debug(f"Matched row {aRow}") + entries = self.rowItemSplitterRe.split(aRow[0]) + + # note, we have entries like + # SEP AAA SEP BBB SEP + # so splitting on SEP gives us an empty entry at front of list + # hence the requirement for len == 2 for at least 1 entry, len == 3 + # for input and output. + + if len(entries) < 2: + self.log.info(f'Row has insufficient entries ? ("{aRow[0]}")') + continue + + inVal = entries[1] + outVal = None + + if len(entries) > 2 and len(entries[2]): + outVal = entries[2].strip() + + if self.addStep(inVal, outVal): + numSteps += 1 + + return numSteps + + def generateFrom(self, contents: str): + self.reset() + + tableParsed = self.parseMarkdownTable(contents) + + if tableParsed is None or not tableParsed: + self.log.error("Could not extract truthtable from contents") + return None + + return self.truthTable -def loadMarkdownTruthTable(filepath:str, logger=None): - with open(filepath, 'r') as f: - contents = f.read() - return parseMarkdownTable(contents, logger) - - return None +def parseSimpleTable(contents: str): + tt = TruthTable() + m = re.compile(r"""^\s*\d'(\d+)\s*\|\s*\d'([zZxX\d]+)""", re.M) + for match in m.findall(contents): + if match[1] != "x": + # we skip anything where all results are don't care + # result = match[1].replace('x', '-') # now using the BinaryValue override + result = match[1] + tt.addMapping(OneToOneTruthMapping(match[0], result)) -TruthTableExample = ''' + return tt + + +def parseMarkdownTable(contents: str, logger=None): + m = MarkdownTestTableParser(logger=logger) + table = m.generateFrom(contents) + return table + + +def loadSimpleTruthTable(filepath: str): + with open(filepath, "r") as f: + contents = f.read() + return parseSimpleTable(contents) + + return None + + +def loadMarkdownTruthTable(filepath: str, logger=None): + with open(filepath, "r") as f: + contents = f.read() + return parseMarkdownTable(contents, logger) + + return None + + +TruthTableExample = """ \\ui_in | \\uo_out ---------- | ----------- 8'11111000 | 8'xxxxxxx0 @@ -508,10 +491,10 @@ TruthTableExample = ''' 8'11111110 | 8'xxxxxxx1 8'11111111 | 8'xxxxxxx0 -''' +""" # with comments and spacing -TruthTableMarkdownExample = ''' +TruthTableMarkdownExample = """ |IN: CBA RC | output | comment | |--------------|--------------|-----------| | 000 000 00 | -- ----- - | init | @@ -528,12 +511,12 @@ TruthTableMarkdownExample = ''' | --- 110 -c | -- 11111 - | success | | --- 000 -c | -- 11111 - | still open| | --- --- 1c | -- 11100 - | reset | -''' +""" if __name__ == "__main__": - m = MarkdownTestTableParser() - table = m.generateFrom(TruthTableMarkdownExample) - - m.dump() - table.dump() - #tt = parseSimpleTable(TruthTableExample) - #print(f'with state {tt[0].state} you get {tt[0].result}') + m = MarkdownTestTableParser() + table = m.generateFrom(TruthTableMarkdownExample) + + m.dump() + table.dump() + # tt = parseSimpleTable(TruthTableExample) + # print(f'with state {tt[0].state} you get {tt[0].result}') diff --git a/testing/src-tpl/test.py b/testing/src-tpl/test.py index 12a5d38..28c2b20 100644 --- a/testing/src-tpl/test.py +++ b/testing/src-tpl/test.py @@ -1,28 +1,27 @@ -''' +""" Wokwi TinyTapeout truth-table based auto test sequence. SPDX-FileCopyrightText: © 2023 Pat Deegan, https://psychogenic.com SPDX-License-Identifier: Apache2.0 -''' +""" -import os import cocotb import testutils.truthtable as truthtable + @cocotb.test() async def truthTableCompare(parentDUT): - ''' - truthTableCompare -- loads markdown truth table and, if load succeeded, actually performs the tests. - ''' + """ + truthTableCompare -- loads markdown truth table and, if load succeeded, actually performs the tests. + """ usermodule = parentDUT.dut i_bus = parentDUT.ui_in o_bus = parentDUT.uo_out - tt = truthtable.loadMarkdownTruthTable('truthtable.md', usermodule._log) + tt = truthtable.loadMarkdownTruthTable("truthtable.md", usermodule._log) if tt is None: - usermodule._log.info('No truth table loaded, no table compare test to run') - return - + usermodule._log.info("No truth table loaded, no table compare test to run") + return + usermodule._log.debug(str(tt)) await tt.testAll(i_bus, o_bus, usermodule._log) - diff --git a/tt_annotate.py b/tt_annotate.py index d8db13e..14faae8 100755 --- a/tt_annotate.py +++ b/tt_annotate.py @@ -2,31 +2,37 @@ from PIL import Image, ImageDraw, ImageFont -img = Image.open('pics/tinytapeout.png').convert('RGBA') -overlay = Image.new('RGBA', img.size, (0, 0, 0, 0)) +img = Image.open("pics/tinytapeout.png").convert("RGBA") +overlay = Image.new("RGBA", img.size, (0, 0, 0, 0)) draw = ImageDraw.Draw(overlay) font = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 30) # coordinate mapping xmap = ((88, 272), (2676, 1646)) # two pairs of (chip x coord, picture x coord) ymap = ((95, 1879), (3240, 213)) # two pairs of (chip y coord, picture y coord) -xshift, yshift = 40, -43 # text position relative to macro origin +xshift, yshift = 40, -43 # text position relative to macro origin -for line in open('openlane/user_project_wrapper/macro.cfg'): +for line in open("openlane/user_project_wrapper/macro.cfg"): macro, x, y, orient = line.split() - if macro == 'scan_controller': + if macro == "scan_controller": continue - elif macro.startswith('scanchain_'): + elif macro.startswith("scanchain_"): continue - name, num = macro.rsplit('_', 1) + name, num = macro.rsplit("_", 1) num, x, y = int(num), int(x), int(y) - x = int((x-xmap[0][0])/(xmap[1][0]-xmap[0][0])*(xmap[1][1]-xmap[0][1])+xmap[0][1]) - y = int((y-ymap[0][0])/(ymap[1][0]-ymap[0][0])*(ymap[1][1]-ymap[0][1])+ymap[0][1]) + x = int( + (x - xmap[0][0]) / (xmap[1][0] - xmap[0][0]) * (xmap[1][1] - xmap[0][1]) + + xmap[0][1] + ) + y = int( + (y - ymap[0][0]) / (ymap[1][0] - ymap[0][0]) * (ymap[1][1] - ymap[0][1]) + + ymap[0][1] + ) x += xshift y += yshift msg = str(num) w, h = draw.textsize(msg, font=font) - draw.text((x-w/2, y-h/2), msg, fill=(0, 0, 0, 255), font=font) + draw.text((x - w / 2, y - h / 2), msg, fill=(0, 0, 0, 255), font=font) -combined = Image.alpha_composite(img, overlay).convert('RGB') -combined.save('pics/tinytapeout_numbered.png') +combined = Image.alpha_composite(img, overlay).convert("RGB") +combined.save("pics/tinytapeout_numbered.png") diff --git a/tt_tool.py b/tt_tool.py index 3dad130..25bfc40 100755 --- a/tt_tool.py +++ b/tt_tool.py @@ -1,39 +1,111 @@ #!/usr/bin/env python3 -import sys -import logging import argparse +import logging +import sys + from project import Project -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser(description="TT setup") - parser.add_argument('--project-dir', help="location of the project", default='.') - parser.add_argument('--yaml', help="the project's yaml configuration file", default='info.yaml') - parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) - parser.add_argument('--openlane2', help="use openlane 2", action="store_const", const=True, default=False) + parser.add_argument("--project-dir", help="location of the project", default=".") + parser.add_argument( + "--yaml", help="the project's yaml configuration file", default="info.yaml" + ) + parser.add_argument( + "--debug", + help="debug logging", + action="store_const", + dest="loglevel", + const=logging.DEBUG, + default=logging.INFO, + ) + parser.add_argument( + "--openlane2", + help="use openlane 2", + action="store_const", + const=True, + default=False, + ) # reports & summaries - parser.add_argument('--print-cell-summary', help="print summary", action="store_const", const=True, default=False) - parser.add_argument('--print-cell-category', help="print category", action="store_const", const=True, default=False) - parser.add_argument('--print-stats', help="print some stats from the run", action="store_const", const=True) - parser.add_argument('--print-warnings', help="print any warnings", action="store_const", const=True) - parser.add_argument('--print-wokwi-id', help="prints the Wokwi project id", action="store_const", const=True) + parser.add_argument( + "--print-cell-summary", + help="print summary", + action="store_const", + const=True, + default=False, + ) + parser.add_argument( + "--print-cell-category", + help="print category", + action="store_const", + const=True, + default=False, + ) + parser.add_argument( + "--print-stats", + help="print some stats from the run", + action="store_const", + const=True, + ) + parser.add_argument( + "--print-warnings", help="print any warnings", action="store_const", const=True + ) + parser.add_argument( + "--print-wokwi-id", + help="prints the Wokwi project id", + action="store_const", + const=True, + ) # documentation - parser.add_argument('--check-docs', help="check the documentation part of the yaml", action="store_const", const=True) - parser.add_argument('--create-pdf', help="create a single page PDF", action="store_const", const=True) - parser.add_argument('--create-svg', help="create a svg of the GDS layout", action="store_const", const=True) - parser.add_argument('--create-png', help="create a png of the GDS layout", action="store_const", const=True) + parser.add_argument( + "--check-docs", + help="check the documentation part of the yaml", + action="store_const", + const=True, + ) + parser.add_argument( + "--create-pdf", + help="create a single page PDF", + action="store_const", + const=True, + ) + parser.add_argument( + "--create-svg", + help="create a svg of the GDS layout", + action="store_const", + const=True, + ) + parser.add_argument( + "--create-png", + help="create a png of the GDS layout", + action="store_const", + const=True, + ) # configure - parser.add_argument('--create-user-config', help="create the user_config.tcl file with top module and source files", action="store_const", const=True) - parser.add_argument('--harden', help="use a local OpenLane install to harden the project", action="store_const", const=True) + parser.add_argument( + "--create-user-config", + help="create the user_config.tcl file with top module and source files", + action="store_const", + const=True, + ) + parser.add_argument( + "--harden", + help="use a local OpenLane install to harden the project", + action="store_const", + const=True, + ) args = parser.parse_args() # setup log - log_format = logging.Formatter('%(asctime)s - %(module)-10s - %(levelname)-8s - %(message)s') + log_format = logging.Formatter( + "%(asctime)s - %(module)-10s - %(levelname)-8s - %(message)s" + ) # configure the client logging - log = logging.getLogger('') + log = logging.getLogger("") # has to be set to debug as is the root logger log.setLevel(args.loglevel) @@ -43,7 +115,7 @@ if __name__ == '__main__': ch.setFormatter(log_format) log.addHandler(ch) - project = Project(0, 'unknown', args.project_dir, args, is_user_project=True) + project = Project(0, "unknown", args.project_dir, args, is_user_project=True) project.post_clone_setup() # handle the options