diff --git a/caravel.py b/caravel.py index 5eb4a03..979a1f6 100644 --- a/caravel.py +++ b/caravel.py @@ -1,81 +1,143 @@ import logging -import json import os +import yaml +import json +import glob +import shutil +import git + +def copy_print(src: str, dest: str): + os.makedirs(os.path.dirname(dest), exist_ok=True) + logging.info(f" -> {dest}") + shutil.copy2(src, dest) + + +def copy_print_glob(pattern: str, dest_dir: str): + for file in glob.glob(pattern): + copy_print(file, os.path.join(dest_dir, os.path.basename(file))) class CaravelConfig(): - def __init__(self, config, projects, num_projects): + def __init__(self, config, projects, modules_yaml_name: str): self.config = config self.projects = projects - self.num_projects = num_projects self.script_dir = os.path.dirname(os.path.realpath(__file__)) + self.modules_yaml_name = modules_yaml_name - # instantiate inside user_project_wrapper - def instantiate(self, extra_macros=[]): - # build the blackbox_project_includes.v file - used for blackboxing when building the GDS - with open('verilog/blackbox_project_includes.v', 'w') as fh: - for macro_name in extra_macros: - fh.write(f'`include "rtl/{macro_name}.v"\n') - for project in self.projects: - if not project.is_fill(): - fh.write(f'`include "gl/{project.get_gl_verilog_filename()}"\n') - - # build complete list of filenames for sim - with open('verilog/includes/includes.rtl.caravel_user_project', 'w') as fh: - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/user_project_wrapper.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/tt_top.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/tt_ctrl.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/tt_mux.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/tt_user_module.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_buf.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_dfrbp.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_diode.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_inv.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_mux4.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_tbuf_pol.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_tbuf.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_zbuf.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/cells.v\n') - for macro_name in extra_macros: - fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{macro_name}.v\n') - for project in self.projects: - if not project.is_fill(): - fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{project.get_top_verilog_filename()}\n') - with open('verilog/includes/includes.rtl.caravel_user_project.openlane2', 'w') as fh: - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/ol2/tt_top/user_project_wrapper.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/tt_top.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/tt_ctrl.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/tt_mux.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/tt_user_module.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_buf.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_dfrbp.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_diode.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_inv.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_mux4.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_tbuf_pol.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_tbuf.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_tie.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_zbuf.v\n') - fh.write('-v $(USER_PROJECT_VERILOG)/rtl/cells.v\n') - for macro_name in extra_macros: - fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{macro_name}.v\n') - for project in self.projects: - if not project.is_fill(): - fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{project.get_top_verilog_filename()}\n') - - # build GL includes + def write_gl_config(self): with open('verilog/includes/includes.gl.caravel_user_project', 'w') as fh: fh.write('-v $(USER_PROJECT_VERILOG)/gl/user_project_wrapper.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/gl/tt_ctrl.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/gl/tt_mux.v\n') - for macro_name in extra_macros: - fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{macro_name}.v\n') for project in self.projects: - if not project.is_fill(): - fh.write(f'-v $(USER_PROJECT_VERILOG)/gl/{project.get_gl_verilog_filename()}\n') + fh.write(f'-v $(USER_PROJECT_VERILOG)/../projects/{project.top_module}/{project.get_gl_verilog_filename()}\n') + + def configure_mux(self): + with open(self.modules_yaml_name, 'r') as modules_file: + module_config = yaml.safe_load(modules_file) + configured_macros = set(map(lambda mod: mod['name'], module_config['modules'])) + logging.info(f"found {len(configured_macros)} preconfigured macros: {configured_macros}") + for project in self.projects: + if project.unprefixed_name not in configured_macros: + module_config['modules'].append({'name': project.unprefixed_name}) + + with open('tt-multiplexer/cfg/modules.yaml', 'w') as mux_modules_file: + yaml.dump(module_config, mux_modules_file) + + res = os.system("make -C tt-multiplexer gensrc") + if res != 0: + logging.error("Failed to generate multiplexer placement configuration") + exit(1) + + mux_index = {} + mux_index_reverse = {} + with open('tt-multiplexer/cfg/modules_placed.yaml') as placed_modules_file: + placed_modules = yaml.safe_load(placed_modules_file) + for module in placed_modules['modules']: + mux_address = (module['y'] << 5) + module['x'] + module_name = 'tt_um_' + module['name'] + project = next(p for p in self.projects if p.top_module == module_name) + mux_index[mux_address] = { + "macro": module_name, + "x": module['x'], + "y": module['y'], + "repo": project.git_url, + "commit": project.commit_id, + }, + mux_index_reverse[module_name] = mux_address + + for project in self.projects: + if project.top_module not in mux_index_reverse: + logging.error(f"no placement found for {project}!") + exit(1) + project.mux_address = mux_index_reverse[project.top_module] + + repo = git.Repo(".") + + shuttle_index_data = { + "shuttle": self.config["name"], + "repo": list(repo.remotes[0].urls)[0], + "commit": repo.commit().hexsha, + "mux": mux_index, + } + + with open('shuttle_index.json', 'w') as shuttle_index_file: + json.dump(shuttle_index_data, shuttle_index_file, indent=2) def list(self): for project in self.projects: - if not project.is_fill(): - logging.info(project) + logging.info(project) + + + def find_last_run(self, macro: str): + runs = f"tt-multiplexer/ol2/{macro}/runs/" + runlist = sorted( + [r for r in os.listdir(runs) if r.startswith("RUN_") and os.path.isdir(os.path.join(runs, r))] + ) + if len(runlist) == 0: + print(f"Error: no runs found for {macro}") + exit(1) + + return os.path.join(runs, runlist[-1]) + + + def copy_macros(self): + logging.info('copying macros to tt_top:') + copy_print_glob("projects/*/*.gds", "tt-multiplexer/ol2/tt_top/gds") + copy_print_glob("projects/*/*.lef", "tt-multiplexer/ol2/tt_top/lef") + copy_print_glob("projects/*/*.v", "tt-multiplexer/ol2/tt_top/verilog") + macros = ["tt_ctrl", "tt_mux"] + for macro in macros: + lastrun = self.find_last_run(macro) + copy_print(f"{lastrun}/final/gds/{macro}.magic.gds", f"tt-multiplexer/ol2/tt_top/gds/{macro}.gds") + copy_print(f"{lastrun}/final/lef/{macro}.lef", f"tt-multiplexer/ol2/tt_top/lef/{macro}.lef") + + + def copy_final_results(self): + macros = ["tt_ctrl", "tt_mux", "tt_top"] + + logging.info('copying final results:') + for macro in macros: + lastrun = self.find_last_run(macro) + macro_name = macro if macro != "tt_top" else "user_project_wrapper" + logging.info(f"** {macro_name} **") + logging.info(f" FROM {lastrun}") + copy_print(f"{lastrun}/final/gds/{macro_name}.magic.gds", f"gds/{macro_name}.gds") + copy_print(f"{lastrun}/final/lef/{macro_name}.lef", f"lef/{macro_name}.lef") + copy_print(f"{lastrun}/final/pnl/{macro_name}.pnl.v", f"verilog/gl/{macro_name}.v") + copy_print(f"{lastrun}/final/nl/{macro_name}.nl.v", f"verilog/gl/{macro_name}.nl.v") + shutil.copytree( + f"{lastrun}/final/spef", f"spef/", copy_function=copy_print, dirs_exist_ok=True + ) + copy_print(f"{lastrun}/final/spef/nom_/{macro_name}.nom.spef", f"spef/{macro_name}.spef") + + def create_efabless_submission(self): + logging.info("creating efabless submission directory:") + lastrun = self.find_last_run("tt_top") + copy_print("README.md", "efabless/README.md") + copy_print("shuttle_index.json", "efabless/shuttle_index.json") + copy_print("verilog/rtl/user_defines.v", "efabless/verilog/rtl/user_defines.v") + copy_print(f"{lastrun}/final/pnl/user_project_wrapper.pnl.v", f"efabless/verilog/gl/user_project_wrapper.v") + copy_print(f"{lastrun}/final/gds/user_project_wrapper.magic.gds", f"efabless/gds/user_project_wrapper.gds") + copy_print(f"{lastrun}/final/lef/user_project_wrapper.lef", f"efabless/lef/user_project_wrapper.gds") diff --git a/configure.py b/configure.py index c6e90a8..77ad79c 100755 --- a/configure.py +++ b/configure.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -from orders import Orders import datetime import yaml +import json import argparse, logging, sys, os, collections from project import Project from documentation import Docs @@ -19,80 +19,30 @@ class Projects(): def __init__(self, config, args): self.args = args self.config = config - - if args.test: - self.projects_file = 'test_projects.yaml' - self.project_dir = config['test_project_dir'] - else: - self.projects_file = 'projects.yaml' - self.project_dir = config['project_dir'] + self.project_dir = config['project_dir'] if not os.path.exists(self.project_dir): os.makedirs(self.project_dir) - try: - with open(self.projects_file) as fh: - self.project_config = yaml.safe_load(fh) - except FileNotFoundError: - logging.error(f"{self.projects_file} not found, create it with --update-orders") - exit(1) - - """ - # useful for contacting people when I find problems in the repo - try: - with open('git_url_to_email.json') as fh: - self.git_url_to_email_map = json.load(fh) - except FileNotFoundError: - self.git_url_to_email_map = {} - """ - self.projects = [] - for index in range(args.limit_num_projects): - if args.single >= 0 and args.single != index: - continue - if index < args.start_from: - continue - if args.end_at != 0 and index > args.end_at: + project_list = [entry for entry in os.listdir(self.project_dir) if os.path.isdir(os.path.join(self.project_dir, entry))] + if args.test: + project_list = ['tt_um_factory_test'] + elif args.sta_projects: + project_list = ['tt_um_loopback'] + + for index, project_id in enumerate(project_list): + project_dir = os.path.join(self.project_dir, project_id) + + commit_id_file = os.path.join(project_dir, 'commit_id.json') + if not os.path.exists(commit_id_file): + logging.warning(f"no commit_id.json in {project_dir}, skipping") continue - """ - try: - email = self.git_url_to_email_map[git_url] - except KeyError: - email = 'none' - """ + commit_id_data = json.load(open(commit_id_file)) - git_url = self.project_config[index]['url'] - filler = self.project_config[index]['fill'] - - # not sure if this is a good idea - if self.project_config[index]['status'] == 'disable': - filler = True - - # fill projects don't have their own directory, reuse the first project which is always fill - if filler: - project_dir = os.path.join(os.path.join(self.project_dir, f'{0 :03}')) - else: - project_dir = os.path.join(os.path.join(self.project_dir, f'{index :03}')) - - project = Project(index, git_url, project_dir, args, fill=filler) - - # clone git repos locally & gds artifacts from action build - if args.clone_all: - if filler is False: - logging.info(f"cloning {project}") - project.clone() - - if args.fetch_gds: - if filler is False: - logging.info(f"fetching gds for {project}") - project.fetch_gds() - - if args.update_all: - if filler is False: - # only updates code, not gds artifacts - logging.info(f"git pull for {project}") - project.pull() + project = Project(index, commit_id_data['repo'], project_dir, args, is_user_project=False) + project.commit_id = commit_id_data['commit'] # projects should now be installed, so load all the data from the yaml files # fill projects will load from the fill project's directory @@ -101,34 +51,19 @@ class Projects(): logging.debug(project) if args.harden: - if filler is False: - project.create_user_config() - project.golden_harden() + project.create_user_config() + project.golden_harden() if args.update_caravel: - logging.debug("copying files to caravel") - if filler is False: - project.copy_files_to_caravel() - - # check all top level module ports are correct - project.check_ports() - project.check_num_cells() + project.check_ports() + project.check_num_cells() self.projects.append(project) - # now do some sanity checks - for project in self.projects: - if project.get_git_remote() != project.git_url: - logging.warning(f"{project} doesn't match remote: {project.get_git_remote()}") - exit(1) - all_macro_instances = [project.get_macro_instance() for project in self.projects] self.assert_unique(all_macro_instances) - all_top_files = [project.get_top_verilog_filename() for project in self.projects if not project.is_fill()] - self.assert_unique(all_top_files) - - all_gds_files = [project.get_macro_gds_filename() for project in self.projects if not project.is_fill()] + all_gds_files = [project.get_macro_gds_filename() for project in self.projects] self.assert_unique(all_gds_files) logging.info(f"loaded {len(self.projects)} projects") @@ -154,44 +89,43 @@ class Projects(): tags = [] for project in self.projects: - if not project.is_fill(): - try: - dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss') - except KeyError: - continue + try: + dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss') + except KeyError: + continue - delt = datetime.timedelta(hours=dt.hour, minutes=dt.minute, seconds=dt.second) - total_seconds += delt.total_seconds() + delt = datetime.timedelta(hours=dt.hour, minutes=dt.minute, seconds=dt.second) + total_seconds += delt.total_seconds() - total_wire_length += int(project.metrics['wire_length']) - total_wires_count += int(project.metrics['wires_count']) - util = float(project.metrics['OpenDP_Util']) - num_cells = project.get_cell_count_from_synth() - total_physical_cells += num_cells + total_wire_length += int(project.metrics['wire_length']) + total_wires_count += int(project.metrics['wires_count']) + util = float(project.metrics['OpenDP_Util']) + num_cells = project.get_cell_count_from_synth() + total_physical_cells += num_cells - yaml_data = project.get_project_doc_yaml() - lang = yaml_data['language'].lower() - if lang in languages: - languages[lang] += 1 - else: - languages[lang] = 1 + yaml_data = project.get_project_doc_yaml() + lang = yaml_data['language'].lower() + if lang in languages: + languages[lang] += 1 + else: + languages[lang] = 1 - if num_cells > max_cells: - max_cells = num_cells - max_cell_project = project - if num_cells < min_cells: - min_cells = num_cells + if num_cells > max_cells: + max_cells = num_cells + max_cell_project = project + if num_cells < min_cells: + min_cells = num_cells - if util > max_util: - max_util = util - max_util_project = project - if util < min_util: - min_util = util + if util > max_util: + max_util = util + max_util_project = project + if util < min_util: + min_util = util - try: - tags += yaml_data['tag'].split(',') - except KeyError: - pass + try: + tags += yaml_data['tag'].split(',') + except KeyError: + pass logging.info(f"build time for all projects {total_seconds / 3600} hrs") logging.info(f"total wire length {total_wire_length} um") @@ -226,18 +160,15 @@ if __name__ == '__main__': with open('config.yaml') as fh: config = yaml.safe_load(fh) - parser.add_argument('--update-orders', help="update the order config file", action="store_const", const=True) parser.add_argument('--list', help="list projects", action='store_const', const=True) - parser.add_argument('--clone-all', help="clone all projects", action="store_const", const=True) - parser.add_argument('--update-all', help="git pull all projects", action="store_const", const=True) - parser.add_argument('--fetch-gds', help="git fetch latest gds", action="store_const", const=True) parser.add_argument('--single', help="do action on single project", type=int, default=-1) - parser.add_argument('--start-from', help="do action on projects after this index", type=int, default=0) - parser.add_argument('--end-at', help="do action on projects before this index", type=int, default=0) parser.add_argument('--update-caravel', help='configure caravel for build', action='store_const', const=True) + parser.add_argument('--copy-macros', help='copy macros for building the tt_top project', action='store_const', const=True) + parser.add_argument('--copy-final-results', help='copy final project files to gds/lef directories', action='store_const', const=True) + parser.add_argument('--create-efabless-submission', help='create efabless submission files', action='store_const', const=True) parser.add_argument('--harden', help="harden project", action="store_const", const=True) - parser.add_argument('--limit-num-projects', help='only configure for the first n projects', type=int, default=int(config['num_projects'])) parser.add_argument('--test', help='use test projects', action='store_const', const=True) + parser.add_argument('--sta-projects', help='use sta projects', action='store_const', const=True) parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) parser.add_argument('--log-email', help="print persons email in messages", action="store_const", const=True) parser.add_argument('--update-image', help="update the image", action="store_const", const=True) @@ -246,8 +177,6 @@ if __name__ == '__main__': parser.add_argument('--dump-pdf', help="create pdf from the markdown") parser.add_argument('--build-hugo-content', help="directory to where to build hugo content") parser.add_argument('--metrics', help="print some project metrics", action="store_const", const=True) - parser.add_argument('--add-extra-project', help="specify a github url to add. Project is added to the extra_projects list specifed in config.yaml") - parser.add_argument('--sram-support', help="include sram LEF/GDS in caravel config", action="store_const", const=True) args = parser.parse_args() @@ -264,20 +193,17 @@ if __name__ == '__main__': ch.setFormatter(log_format) log.addHandler(ch) - if args.update_orders: - # don't instantiate orders unless we have to as it needs stripe token - orders = Orders(config) - orders.fetch_orders() - orders.update_project_list() - - if args.add_extra_project: - orders = Orders(config) - orders.add_extra_project(args.add_extra_project) - projects = Projects(config, args) + if args.test: + modules_yaml_name = "modules.test.yaml" + elif args.sta_projects: + modules_yaml_name = "modules.sta.yaml" + else: + modules_yaml_name = "modules.yaml" + docs = Docs(projects.projects, args=args) - caravel = CaravelConfig(config, projects.projects, num_projects=args.limit_num_projects) + caravel = CaravelConfig(config, projects.projects, modules_yaml_name) rom = ROMFile(config, projects.projects) if args.list: @@ -287,14 +213,22 @@ if __name__ == '__main__': projects.build_metrics() if args.update_caravel: - extra_macros = [] - if args.sram_support: - extra_macros = ["sky130_sram_2kbyte_1rw1r_32x512_8"] - caravel.instantiate(extra_macros) + caravel.configure_mux() + if args.test: + caravel.write_gl_config() rom.write_rom() if not args.test: docs.build_index() + if args.copy_macros: + caravel.copy_macros() + + if args.copy_final_results: + caravel.copy_final_results() + + if args.create_efabless_submission: + caravel.create_efabless_submission() + if args.update_image: docs.update_image() diff --git a/documentation.py b/documentation.py index df01b3e..6b4d06b 100644 --- a/documentation.py +++ b/documentation.py @@ -22,8 +22,7 @@ class Docs(): fh.write("| Index | Author | Title | Type | Git Repo |\n") fh.write("| ----- | ------ | ------| -----| ---------|\n") for project in self.projects: - if not project.is_fill(): - fh.write(project.get_index_row()) + fh.write(project.get_index_row()) def update_image(self): ruby = os.path.join(self.script_dir, 'caravel_template', 'dump_pic.rb') @@ -67,8 +66,6 @@ class Docs(): fh.write(doc_header) for project in self.projects: - if project.is_fill(): - continue yaml_data = project.get_project_doc_yaml() yaml_data['index'] = project.index @@ -134,27 +131,26 @@ class Docs(): fh.write("| ----- | ----- | -------|\n") for project in self.projects: logging.info(project) - if not project.is_fill(): - fh.write(project.get_hugo_row()) + fh.write(project.get_hugo_row()) - project_dir = os.path.join(hugo_root, f'{project.get_index() :03}') - project_image_dir = os.path.join(project_dir, 'images') - os.makedirs(project_dir) - os.makedirs(project_image_dir) - yaml_data = project.get_project_doc_yaml() - yaml_data['index'] = project.index - yaml_data['weight'] = project.index + 1 - yaml_data['git_action'] = project.get_latest_action_url() - yaml_data['picture_link'] = '' - if yaml_data['picture']: - picture_name = yaml_data['picture'] - picture_filename = os.path.join(project.local_dir, picture_name) - picture_basename = os.path.basename(picture_filename) - try: - shutil.copyfile(picture_filename, os.path.join(project_image_dir, picture_basename)) - yaml_data['picture_link'] = f'![picture](images/{picture_basename})' - except FileNotFoundError: - yaml_data['picture_link'] = 'Image path is broken' - doc = doc_template.format(**yaml_data) - with open(os.path.join(project_dir, '_index.md'), 'w') as pfh: - pfh.write(doc) + project_dir = os.path.join(hugo_root, f'{project.get_index() :03}') + project_image_dir = os.path.join(project_dir, 'images') + os.makedirs(project_dir) + os.makedirs(project_image_dir) + yaml_data = project.get_project_doc_yaml() + yaml_data['index'] = project.index + yaml_data['weight'] = project.index + 1 + yaml_data['git_action'] = project.get_latest_action_url() + yaml_data['picture_link'] = '' + if yaml_data['picture']: + picture_name = yaml_data['picture'] + picture_filename = os.path.join(project.local_dir, picture_name) + picture_basename = os.path.basename(picture_filename) + try: + shutil.copyfile(picture_filename, os.path.join(project_image_dir, picture_basename)) + yaml_data['picture_link'] = f'![picture](images/{picture_basename})' + except FileNotFoundError: + yaml_data['picture_link'] = 'Image path is broken' + doc = doc_template.format(**yaml_data) + with open(os.path.join(project_dir, '_index.md'), 'w') as pfh: + pfh.write(doc) diff --git a/git_utils.py b/git_utils.py index 1bbb5e9..8bc3fdf 100644 --- a/git_utils.py +++ b/git_utils.py @@ -3,21 +3,7 @@ from urllib.parse import urlparse import logging import requests import os -import zipfile -import io import errno -import sys - - -def unique(duplist): - unique_list = [] - # traverse for all elements - for x in duplist: - # check if exists in unique_list or not - if x not in unique_list: - unique_list.append(x) - return unique_list - def fetch_file(url, filename): logging.info("trying to download {}".format(url)) @@ -64,50 +50,6 @@ def headers_try_to_add_authorization_from_environment(headers: dict) -> bool: print("WARNING: No github token found from environment, trying public API requests without, see docs/INFO.md#instructions-to-build-gds", file=sys.stderr) return False -def fetch_file_from_git(git_url, path): - # get the basics - user_name, repo = split_git_url(git_url) - - headers = { - "Accept" : "application/vnd.github+json", - } - # authenticate for rate limiting - headers_try_to_add_authorization_from_environment(headers) - - api_url = 'https://api.github.com/repos/%s/%s/contents/%s' % (user_name, repo, path) - - logging.debug(api_url) - r = requests.get(api_url, headers=headers) - check_status(r) - requests_remaining = int(r.headers['X-RateLimit-Remaining']) - if requests_remaining == 0: - logging.error("no API requests remaining") - exit(1) - - logging.debug("API requests remaining %d" % requests_remaining) - - data = r.json() - if 'content' not in data: - return None - - file_content = data['content'] - - file_content_encoding = data.get('encoding') - if file_content_encoding == 'base64': - file_content = base64.b64decode(file_content) - - return file_content - - -# the latest artifact isn't necessarily the one related to the latest commit, as github -# could have taken longer to process an older commit than a newer one. -# so iterate through commits and return the artifact that matches -def get_most_recent_action_url(commits, artifacts): - release_sha_to_download_url = {artifact['workflow_run']['head_sha']: artifact['archive_download_url'] for artifact in artifacts} - for commit in commits: - if commit['sha'] in release_sha_to_download_url: - return release_sha_to_download_url[commit['sha']] - def get_most_recent_action_page(commits, runs): release_sha_to_page_url = {run['head_sha']: run['html_url'] for run in runs if run['name'] == 'gds'} @@ -127,69 +69,6 @@ def split_git_url(url): return user_name, repo -# download the artifact for each project to get the gds & lef -def install_artifacts(url, directory): - logging.debug(url) - user_name, repo = split_git_url(url) - - headers = { - "Accept" : "application/vnd.github+json", - } - # authenticate for rate limiting - headers_try_to_add_authorization_from_environment(headers) - - # first fetch the git commit history - api_url = f'https://api.github.com/repos/{user_name}/{repo}/commits' - r = requests.get(api_url, headers=headers) - check_status(r) - requests_remaining = int(r.headers['X-RateLimit-Remaining']) - if requests_remaining == 0: - logging.error("no API requests remaining") - exit(1) - - commits = r.json() - - # now get the artifacts - api_url = f'https://api.github.com/repos/{user_name}/{repo}/actions/artifacts' - r = requests.get(api_url, headers=headers, params={'per_page': 100}) - check_status(r) - data = r.json() - - # check there are some artifacts - if 'artifacts' not in data: - logging.error(f"no artifact found for {url}") - exit(1) - else: - # only get artifacts called GDS - artifacts = [a for a in data['artifacts'] if a['name'] == 'GDS'] - logging.debug(f"found {len(artifacts)} artifacts") - - if len(artifacts) == 0: - logging.error("no artifacts for this project") - exit(1) - - download_url = get_most_recent_action_url(commits, artifacts) - - # need actions access on the token to get the artifact - # won't work on a pull request because they won't have the token - attempts = 1 - max_attempts = 3 - while attempts < max_attempts: - try: - logging.debug(f"download url {download_url} attempt {attempts}") - r = requests.get(download_url, headers=headers) - check_status(r) - z = zipfile.ZipFile(io.BytesIO(r.content)) - z.extractall(directory) - break - except zipfile.BadZipFile: - attempts += 1 - logging.warning(f"problem with zipfile, retry {attempts}") - - if attempts == max_attempts: - logging.error("gave up downloading zipfile") - exit(1) - def get_latest_action_url(url, directory): logging.debug(url) diff --git a/orders.py b/orders.py deleted file mode 100644 index 09c53e3..0000000 --- a/orders.py +++ /dev/null @@ -1,137 +0,0 @@ -import stripe, os -import time -from datetime import datetime -import yaml -import requests -import pytz -import logging - - -class Orders(): - - def __init__(self, config): - self.config = config - self.open_date = datetime.strptime(self.config['start_date'], '%Y-%m-%d') - stripe.api_key = os.environ['STRIPE_TOKEN'] - self.orders = [] - - def fetch_orders(self): - orders = self.fetch_stripe_orders() + self.fetch_university_orders() + self.fetch_extra_orders() - logging.info(f"total = {len(orders)}") - self.orders = sorted(orders, key=lambda d: d['time']) - - def add_extra_project(self, git_url): - try: - with open(self.config['extra_projects']) as fh: - config = yaml.safe_load(fh) - except FileNotFoundError: - config = [] - - config.append({'project': {'url' : git_url, 'timestamp' : time.time()}}) - with open(self.config['extra_projects'], 'w') as fh: - yaml.safe_dump(config, fh) - logging.info(f"added {git_url} to {self.config['extra_projects']}") - - def fetch_extra_orders(self): - with open(self.config['extra_projects']) as fh: - config = yaml.safe_load(fh) - - orders = [] - for project in config: - orders.append({ - 'git_url' : project['project']['url'], - 'email' : None, - 'time' : datetime.fromtimestamp(project['project']['timestamp']).replace(tzinfo=pytz.UTC) - }) - - logging.info(f"fetched extra {len(orders)} orders") - return orders - - def fetch_university_orders(self): - name = self.config['name'] - url = f'https://app.tinytapeout.com/api/submissions?shuttle={name}' - r = requests.get(url) - if r.status_code != 200: - logging.warning("couldn't download {}".format(url)) - exit(1) - - orders = [] - for item in r.json()['items']: - order = { - 'git_url' : item['repo'], - 'email' : None, - 'time' : datetime.strptime(item['time'], '%Y-%m-%dT%H:%M:%S.%f%z').replace(tzinfo=pytz.UTC) - } - if order['time'] > self.open_date.replace(tzinfo=pytz.UTC): - orders.append(order) - logging.info(f"fetched university {len(orders)} orders") - return orders - - def fetch_stripe_orders(self): - start_id = None - after_open_date = True - orders = [] - refunded_orders = [] - - while after_open_date: - refunds = stripe.Refund.list(limit=10, starting_after=start_id) - for refund in refunds: - created = datetime.fromtimestamp(refund['created']) - if created < self.open_date: - after_open_date = False - - refunded_orders.append(refund['payment_intent']) - start_id = refund['id'] - - start_id = None - after_open_date = True - - while after_open_date: - checkouts = stripe.checkout.Session.list(limit=10, starting_after=start_id) - - for checkout in checkouts: - created = datetime.fromtimestamp(checkout['created']) - if created < self.open_date: - after_open_date = False - - if checkout['payment_status'] == 'paid': - if 'github' in checkout['metadata']: - git_url = checkout['metadata']['github'] - - if checkout['payment_intent'] in refunded_orders: - logging.warning(f"found a refunded order! {git_url}") - continue - - order = { - 'git_url' : git_url, - 'email' : checkout['customer_details']['email'], - 'time' : datetime.fromtimestamp(checkout['created']).replace(tzinfo=pytz.UTC), - } - orders.append(order) - - # pagination - start_id = checkout['id'] - - # put in date order - logging.info(f"fetched stripe {len(orders)} orders") - return orders - - def update_project_list(self): - project_list = {} - # first project is the filler test project - index = 0 - project = {'url': self.config['filler_project'], 'status': 'active', 'fill': False} - project_list[index] = project - - # add all the real orders - for index, order in enumerate(self.orders): - project = {'url': order['git_url'], 'status': 'active', 'fill': False} - project_list[index + 1] = project - - # fill with the filler - for index in range(len(project_list), self.config['num_projects']): - project = {'url': self.config['filler_project'], 'status': 'active', 'fill': True} - project_list[index] = project - - with open('projects.yaml', 'w') as fh: - fh.write(yaml.dump(project_list, sort_keys=False)) diff --git a/project.py b/project.py index b993503..c3245db 100644 --- a/project.py +++ b/project.py @@ -21,12 +21,13 @@ with open(os.path.join(os.path.dirname(__file__), 'tile_sizes.yaml'), 'r') as st class Project(): - def __init__(self, index, git_url, local_dir, args, fill=False): + def __init__(self, index, git_url, local_dir, args, is_user_project): self.git_url = git_url self.args = args self.index = index - self.fill = fill self.local_dir = local_dir + self.is_user_project = is_user_project + self.src_dir = os.path.join(self.local_dir, 'src') if self.is_user_project else self.local_dir def post_clone_setup(self): self.load_yaml() @@ -42,7 +43,7 @@ class Project(): def check_ports(self): top = self.get_macro_name() - sources = [os.path.join(self.local_dir, 'src', src) for src in self.src_files] + sources = [os.path.join(self.src_dir, src) for src in self.src_files] source_list = " ".join(sources) json_file = 'ports.json' @@ -85,9 +86,6 @@ class Project(): if num_cells < 11: logging.warning(f"{self} only has {num_cells} cells") - def is_fill(self): - return self.fill - def is_wokwi(self): if self.wokwi_id != 0: return True @@ -117,18 +115,25 @@ class Project(): if self.is_hdl(): self.top_module = self.yaml['project']['top_module'] - self.src_files = self.get_hdl_source() - self.top_verilog_filename = self.find_top_verilog() + if self.is_user_project: + self.src_files = self.get_hdl_source() + self.top_verilog_filename = self.find_top_verilog() else: self.top_module = f"tt_um_wokwi_{self.wokwi_id}" - self.src_files = self.get_wokwi_source() - self.top_verilog_filename = self.src_files[0] + if self.is_user_project: + self.src_files = self.get_wokwi_source() + self.top_verilog_filename = self.src_files[0] + + if not self.is_user_project: + self.src_files = [self.get_gl_verilog_filename()] + + self.unprefixed_name = re.sub('^tt_um_', '', self.top_module) if not self.top_module.startswith("tt_um_"): logging.error(f'top module name must start with "tt_um_" (current value: "{self.top_module}")') exit(1) - self.macro_instance = f"{self.top_module}_{self.index :03}" + self.macro_instance = f"{self.top_module}" def get_wokwi_source(self): src_file = "tt_um_wokwi_{}.v".format(self.wokwi_id) @@ -160,7 +165,7 @@ class Project(): if '*' in filename: logging.error("* not allowed, please specify each file") exit(1) - if not os.path.exists(os.path.join(self.local_dir, 'src', filename)): + if not os.path.exists(os.path.join(self.src_dir, filename)): logging.error(f"{filename} doesn't exist in the repo") exit(1) @@ -197,7 +202,7 @@ class Project(): rgx_mod = re.compile(r"(?:^|[\s])module[\s]{1,}([\w]+)") top_verilog = [] for src in self.src_files: - with open(os.path.join(self.local_dir, 'src', src)) as fh: + with open(os.path.join(self.src_dir, src)) as fh: for line in fh.readlines(): for match in rgx_mod.finditer(line): if match.group(1) == self.top_module: @@ -227,26 +232,6 @@ class Project(): if GITHUB_SERVER_URL and GITHUB_REPOSITORY and GITHUB_RUN_ID: return f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" - def clone(self): - if os.path.exists(self.local_dir): - git_remote = self.get_git_remote() - if self.git_url == git_remote: - logging.info("git repo already exists and is correct - skipping") - else: - logging.error("git repo exists and remote doesn't match - abort") - logging.error(f"{self.git_url} != {git_remote}") - exit(1) - else: - logging.debug("clone") - git.Repo.clone_from(self.git_url, self.local_dir, recursive=True) - - def pull(self): - repo = git.Repo(self.local_dir) - # reset - repo.git.reset('--hard') - o = repo.remotes.origin - o.pull() - def __str__(self): """ if self.args.log_email: @@ -255,9 +240,6 @@ class Project(): """ return f"[{self.index:03} : {self.git_url}]" - def fetch_gds(self): - git_utils.install_artifacts(self.git_url, self.local_dir) - def get_latest_action_url(self): return git_utils.get_latest_action_url(self.git_url, self.local_dir) @@ -274,7 +256,10 @@ class Project(): # metrics def get_metrics_path(self): - return os.path.join(self.local_dir, 'runs/wokwi/reports/metrics.csv') + if self.is_user_project: + return os.path.join(self.local_dir, 'runs/wokwi/reports/metrics.csv') + else: + return os.path.join(self.local_dir, 'metrics.csv') # name of the gds file def get_macro_gds_filename(self): @@ -307,25 +292,6 @@ class Project(): def get_git_url(self): return self.git_url - def copy_files_to_caravel(self): - files = [ - (f"runs/wokwi/results/final/gds/{self.get_macro_gds_filename()}", f"gds/{self.get_macro_gds_filename()}"), - (f"runs/wokwi/results/final/commit_id.json", f"gds/{self.get_macro_info_filename()}"), - (f"runs/wokwi/results/final/spef/{self.get_macro_spef_filename()}", f"spef/{self.get_macro_spef_filename()}"), - (f"runs/wokwi/results/final/lef/{self.get_macro_lef_filename()}", f"lef/{self.get_macro_lef_filename()}"), - (f"runs/wokwi/results/final/verilog/gl/{self.get_gl_verilog_filename()}", f"verilog/gl/{self.get_gl_verilog_filename()}"), - (f"src/{self.top_verilog_filename}", f"verilog/rtl/{self.get_top_verilog_filename()}"), - ] - # copy RTL verilog to openlane2 mux directory: - if os.path.isdir("tt-multiplexer/ol2/tt_top/verilog"): - files.append((f"runs/wokwi/results/final/verilog/gl/{self.get_gl_verilog_filename()}", f"tt-multiplexer/ol2/tt_top/verilog/{self.get_gl_verilog_filename()}")) - - logging.debug("copying files into position") - for from_path, to_path in files: - from_path = os.path.join(self.local_dir, from_path) - logging.debug(f"copy {from_path} to {to_path}") - shutil.copyfile(from_path, to_path) - def print_wokwi_id(self): print(self.wokwi_id) @@ -333,18 +299,18 @@ class Project(): logging.info("fetching wokwi files") src_file = self.src_files[0] url = f"https://wokwi.com/api/projects/{self.wokwi_id}/verilog" - git_utils.fetch_file(url, os.path.join(self.local_dir, "src", src_file)) + git_utils.fetch_file(url, os.path.join(self.src_dir, src_file)) # also fetch the wokwi diagram url = f"https://wokwi.com/api/projects/{self.wokwi_id}/diagram.json" diagram_file = "wokwi_diagram.json" - git_utils.fetch_file(url, os.path.join(self.local_dir, "src", diagram_file)) + git_utils.fetch_file(url, os.path.join(self.src_dir, diagram_file)) # attempt to download the *optional* truthtable for the project truthtable_file = "truthtable.md" url = f"https://wokwi.com/api/projects/{self.wokwi_id}/{truthtable_file}" try: - git_utils.fetch_file(url, os.path.join(self.local_dir, "src", truthtable_file)) + git_utils.fetch_file(url, os.path.join(self.src_dir, truthtable_file)) logging.info(f'Wokwi project {self.wokwi_id} has a truthtable included, will test!') self.install_wokwi_testing() except FileNotFoundError: @@ -383,7 +349,7 @@ class Project(): self.check_ports() logging.info("creating include file") filename = 'user_config.tcl' - with open(os.path.join(self.local_dir, 'src', filename), 'w') as fh: + with open(os.path.join(self.src_dir, filename), 'w') as fh: fh.write("set ::env(DESIGN_NAME) {}\n".format(self.top_module)) fh.write('set ::env(VERILOG_FILES) "\\\n') for line, source in enumerate(self.src_files): @@ -400,7 +366,7 @@ class Project(): def golden_harden(self): logging.info(f"hardening {self}") # copy golden config - shutil.copyfile('golden_config.tcl', os.path.join(self.local_dir, 'src', 'config.tcl')) + shutil.copyfile('golden_config.tcl', os.path.join(self.src_dir, 'config.tcl')) self.harden() def harden(self): @@ -568,7 +534,7 @@ class Project(): def get_cell_count_from_synth(self): num_cells = 0 try: - yosys_report = glob.glob(f'{self.local_dir}/runs/wokwi/reports/synthesis/1-synthesis.*0.stat.rpt')[0] # can't open a file with \ in the path + yosys_report = f'{self.local_dir}/synthesis-stats.txt' with open(yosys_report) as fh: for line in fh.readlines(): m = re.search(r'Number of cells:\s+(\d+)', line) diff --git a/tt_tool.py b/tt_tool.py index 364666d..7a56427 100755 --- a/tt_tool.py +++ b/tt_tool.py @@ -42,7 +42,7 @@ if __name__ == '__main__': ch.setFormatter(log_format) log.addHandler(ch) - project = Project(0, 'unknown', args.project_dir, args) + project = Project(0, 'unknown', args.project_dir, args, is_user_project=True) project.post_clone_setup() # handle the options