From 97cfa3b15e546aa42377a4b8130be6704670de37 Mon Sep 17 00:00:00 2001 From: matt venn Date: Mon, 13 Mar 2023 18:05:56 +0100 Subject: [PATCH] lint and partially working --- caravel.py | 5 +++-- configure.py | 17 ++++++----------- documentation.py | 17 +++++++++-------- lint.sh | 2 +- orders.py | 15 ++++++--------- project.py | 45 ++++++++++----------------------------------- tt_tool.py | 2 +- 7 files changed, 36 insertions(+), 67 deletions(-) diff --git a/caravel.py b/caravel.py index 3660272..2692b4f 100644 --- a/caravel.py +++ b/caravel.py @@ -1,3 +1,6 @@ +import logging + + class CaravelConfig(): def __init__(self, projects, num_projects): @@ -267,5 +270,3 @@ class CaravelConfig(): def list(self): for project in self.projects: logging.info(project) - - diff --git a/configure.py b/configure.py index 808698c..cd4601f 100755 --- a/configure.py +++ b/configure.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -import re, glob, json, csv from orders import Orders import datetime import yaml -import subprocess -import argparse, logging, shutil, sys, os, collections -from git_utils import install_artifacts, get_latest_action_url, unique -import git +import argparse, logging, sys, os, collections from project import Project +from documentation import Docs +from caravel import CaravelConfig # pipe handling @@ -66,7 +64,6 @@ class Projects(): except KeyError: email = 'none' """ - git_url = self.project_config[index]['url'] filler = self.project_config[index]['fill'] @@ -126,7 +123,6 @@ class Projects(): all_gds_files = [project.get_macro_gds_filename() for project in self.projects if not project.is_fill()] self.assert_unique(all_gds_files) - def assert_unique(self, check): duplicates = [item for item, count in collections.Counter(check).items() if count > 1] if duplicates: @@ -175,7 +171,6 @@ class Projects(): if util < min_util: min_util = util - logging.info(f"build time for all projects {total_seconds / 3600} hrs") logging.info(f"total wire length {total_wire_length} um") logging.info(f"total cells {total_physical_cells}") @@ -230,8 +225,8 @@ if __name__ == '__main__': projects = Projects(config, args) -# docs = Docs(projects.projects, args=args) -# caravel = CaravelConfig(projects.projects, num_projects=args.limit_num_projects) + docs = Docs(projects.projects, args=args) + caravel = CaravelConfig(projects.projects, num_projects=args.limit_num_projects) if args.list: caravel.list() @@ -243,7 +238,7 @@ if __name__ == '__main__': orders = Orders(config) orders.fetch_orders() orders.update_project_list() - + if args.update_caravel: caravel.create_macro_config() caravel.instantiate() diff --git a/documentation.py b/documentation.py index ec16fad..2ead7e3 100644 --- a/documentation.py +++ b/documentation.py @@ -1,3 +1,5 @@ +import os, logging, subprocess, shutil, json + class Docs(): @@ -30,9 +32,9 @@ class Docs(): design = project.get_yaml() designs.append(design) - with open(args.dump_json, "w") as fh: + with open(self.args.dump_json, "w") as fh: fh.write(json.dumps(designs, indent=4)) - logging.info(f'wrote json to {args.dump_json}') + logging.info(f'wrote json to {self.args.dump_json}') def dump_markdown(self): @@ -51,7 +53,7 @@ class Docs(): with open("CREDITS.md") as fh: doc_credits = fh.read() - with open(args.dump_markdown, 'w') as fh: + with open(self.args.dump_markdown, 'w') as fh: fh.write(doc_header) for project in self.projects: @@ -86,17 +88,17 @@ class Docs(): fh.write("\n\pagebreak\n") fh.write(doc_credits) - logging.info(f'wrote markdown to {args.dump_markdown}') + logging.info(f'wrote markdown to {self.args.dump_markdown}') - if args.dump_pdf: - pdf_cmd = f'pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {args.dump_markdown} -o {args.dump_pdf}' + if self.args.dump_pdf: + pdf_cmd = f'pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {self.args.dump_markdown} -o {self.args.dump_pdf}' logging.info(pdf_cmd) p = subprocess.run(pdf_cmd, shell=True) if p.returncode != 0: logging.error("pdf command failed") def build_hugo_content(self): - hugo_root = args.build_hugo_content + hugo_root = self.args.build_hugo_content hugo_images = os.path.join(hugo_root, 'images') shutil.rmtree(hugo_root) os.makedirs(hugo_root) @@ -144,4 +146,3 @@ class Docs(): doc = doc_template.format(**yaml_data) with open(os.path.join(project_dir, '_index.md'), 'w') as pfh: pfh.write(doc) - diff --git a/lint.sh b/lint.sh index 983de92..b92498a 100755 --- a/lint.sh +++ b/lint.sh @@ -1 +1 @@ -flake8 --ignore E203,E123,E501,E221,E401,W605 documentation.py project.py tt_tool.py reports.py +flake8 --ignore E203,E123,E501,E221,E401,W605 configure.py project.py tt_tool.py orders.py documentation.py caravel.py diff --git a/orders.py b/orders.py index 476e143..1b0c3b8 100644 --- a/orders.py +++ b/orders.py @@ -1,8 +1,8 @@ -import stripe, os, json +import stripe, os from datetime import datetime -from collections import OrderedDict import yaml + class Orders(): def __init__(self, config): @@ -11,7 +11,6 @@ class Orders(): stripe.api_key = os.environ['STRIPE_TOKEN'] self.orders = [] - def fetch_orders(self): start_id = None after_open_date = True @@ -35,29 +34,27 @@ class Orders(): # pagination start_id = checkout['id'] - + # put in date order self.orders.reverse() return self.orders - def update_project_list(self): project_list = {} # first project is the filler test project index = 0 - project = { 'url': self.config['filler_project'], 'status': 'active', 'fill': False } + project = {'url': self.config['filler_project'], 'status': 'active', 'fill': False} project_list[index] = project # add all the real orders for index, order in enumerate(self.orders): - project = { 'url': order['git_url'], 'status': 'active', 'fill': False } + project = {'url': order['git_url'], 'status': 'active', 'fill': False} project_list[index + 1] = project # fill with the filler for index in range(len(project_list), self.config['num_projects']): - project = { 'url': self.config['filler_project'], 'status': 'active', 'fill': True } + project = {'url': self.config['filler_project'], 'status': 'active', 'fill': True} project_list[index] = project with open('projects.yaml', 'w') as fh: fh.write(yaml.dump(project_list, sort_keys=False)) - diff --git a/project.py b/project.py index 87edbb0..837ee95 100644 --- a/project.py +++ b/project.py @@ -1,22 +1,16 @@ import yaml -import logging -import requests +import glob import os -import subprocess import logging import re import subprocess -import os -import glob import gdstk import cairosvg -import os import csv import json -import re -import glob import git_utils import git +import shutil # can't find a google version CELL_URL = 'https://antmicro-skywater-pdk-docs.readthedocs.io/en/test-submodules-in-rtd/contents/libraries/sky130_fd_sc_ls/cells/' @@ -43,7 +37,6 @@ class Project(): except FileNotFoundError: self.metrics = {} - def check_ports(self): top = self.get_macro_name() sources = [os.path.join(self.local_dir, 'src', src) for src in self.src_files] @@ -69,7 +62,6 @@ class Project(): logging.error(f"{self} {port} doesn't have 8 bits") exit(1) - def check_num_cells(self): num_cells = self.get_cell_count_from_gl() if self.is_hdl(): @@ -79,16 +71,13 @@ class Project(): if num_cells < 11: logging.warning(f"{self} only has {num_cells} cells") - def is_fill(self): return self.fill - def is_wokwi(self): if self.wokwi_id != 0: return True - def is_hdl(self): return not self.is_wokwi() @@ -100,7 +89,6 @@ class Project(): logging.error(f"yaml file not found for {self} - do you need to --clone the project repos?") exit(1) - # find and save the location of the source files # get name of top module and the verilog module that contains the top def setup_source_files(self): @@ -125,13 +113,10 @@ class Project(): self.macro_instance = f"{self.top_module}_{self.index :03}" self.scanchain_instance = f"scanchain_{self.index :03}" - def get_wokwi_source(self): src_file = "user_module_{}.v".format(self.wokwi_id) return [src_file, 'cells.v'] - - def get_hdl_source(self): if 'source_files' not in self.yaml['project']: logging.error("source files must be provided if wokwi_id is set to 0") @@ -164,7 +149,6 @@ class Project(): return source_files - def get_yaml(self): return self.yaml @@ -224,18 +208,19 @@ class Project(): o = repo.remotes.origin o.pull() - def __str__(self): -# if self.args.log_email: -# return f"[{self.index:03} : {self.email} : {self.git_url}]" -# else: + """ + if self.args.log_email: + return f"[{self.index:03} : {self.email} : {self.git_url}]" + else: + """ return f"[{self.index:03} : {self.git_url}]" def fetch_gds(self): - install_artifacts(self.git_url, self.local_dir) + git_utils.install_artifacts(self.git_url, self.local_dir) def get_latest_action_url(self): - return get_latest_action_url(self.git_url, self.local_dir) + return git_utils.get_latest_action_url(self.git_url, self.local_dir) def get_macro_name(self): return self.top_module @@ -296,13 +281,12 @@ class Project(): logging.debug(f"copy {from_path} to {to_path}") shutil.copyfile(from_path, to_path) - def print_wokwi_id(self): print(self.wokwi_id) - def fetch_wokwi_files(self): logging.info("fetching wokwi files") + src_file = self.src_files[0] url = "https://wokwi.com/api/projects/{}/verilog".format(self.wokwi_id) git_utils.fetch_file(url, os.path.join(self.local_dir, "src", src_file)) @@ -311,7 +295,6 @@ class Project(): diagram_file = "wokwi_diagram.json" git_utils.fetch_file(url, os.path.join(self.local_dir, "src", diagram_file)) - def create_user_config(self): self.fetch_wokwi_files() @@ -326,7 +309,6 @@ class Project(): fh.write(' \\\n') fh.write('"\n') - """ def harden(self): logging.info(f"hardening {self}") @@ -392,7 +374,6 @@ class Project(): logging.error('Invalid format for discord username') exit(1) - # use pandoc to create a single page PDF preview def create_pdf(self): yaml = self.yaml @@ -430,7 +411,6 @@ class Project(): if p.returncode != 0: logging.error("pdf command failed") - # SVG and PNG renders of the GDS def create_svg(self): gds = glob.glob(os.path.join(self.local_dir, 'runs/wokwi/results/final/gds/*gds')) @@ -441,7 +421,6 @@ class Project(): if self.args.create_png: cairosvg.svg2png(url='gds_render.svg', write_to='gds_render.png') - def print_warnings(self): warnings = [] with open(os.path.join(self.local_dir, 'runs/wokwi/logs/synthesis/1-synthesis.log')) as f: @@ -456,7 +435,6 @@ class Project(): for warning in warnings: print(f'* {warning}') - def print_stats(self): print('# Routing stats') print() @@ -464,7 +442,6 @@ class Project(): print('|-------------|------------------|') print('| {} | {} |'.format(self.metrics['OpenDP_Util'], self.metrics['wire_length'])) - # Print the summaries def summarize(self): cell_count = self.get_cell_count_from_gl() @@ -509,7 +486,6 @@ class Project(): cell_links = [f'[{name}]({CELL_URL}{name})' for name in cat_dict['examples']] print(f'|{cat_name} | {" ".join(cell_links)} | {cat_dict["count"]}|') - # Parse the lib, cell and drive strength an OpenLane gate-level Verilog file def get_cell_count_from_gl(self): cell_count = {} @@ -531,7 +507,6 @@ class Project(): cell_count[cell_name] = 1 return (cell_count) - # Load all the json defs and combine into one big dict, keyed by cellname # Needs access to the PDK def create_defs(self): diff --git a/tt_tool.py b/tt_tool.py index b5a66d7..55f7ed3 100755 --- a/tt_tool.py +++ b/tt_tool.py @@ -60,7 +60,7 @@ if __name__ == '__main__': elif args.print_wokwi_id: project.print_wokwi_id() - + if args.create_user_config: project.create_user_config()