#!/usr/bin/env python3 import argparse import collections import datetime import json import logging import os import sys # pipe handling from signal import SIG_DFL, SIGPIPE, signal from typing import Dict, List, TypedDict import yaml from config import Config from documentation import Docs from project import Project from rom import ROMFile from shuttle import ShuttleConfig signal(SIGPIPE, SIG_DFL) class Projects: def __init__(self, config: Config, args): self.args = args self.config = config self.project_dir = config["project_dir"] if not os.path.exists(self.project_dir): os.makedirs(self.project_dir) only_projects = os.getenv("TT_ONLY_PROJECTS") if only_projects: only_projects = only_projects.split(",") self.projects: List[Project] = [] project_list = [ entry for entry in os.listdir(self.project_dir) if os.path.isdir(os.path.join(self.project_dir, entry)) ] if args.test: project_list = ["tt_um_chip_rom", "tt_um_factory_test"] elif args.sta_projects: project_list = ["tt_um_loopback"] for index, project_id in enumerate(project_list): project_dir = os.path.join(self.project_dir, project_id) commit_id_file = os.path.join(project_dir, "commit_id.json") if not os.path.exists(commit_id_file): logging.warning(f"no commit_id.json in {project_dir}, skippinggi") continue commit_id_data = json.load(open(commit_id_file)) if commit_id_data.get("skip", False): logging.warning(f"skipping {project_dir} (skip flag set)") continue if only_projects and project_id not in only_projects: continue project = Project( index, commit_id_data["repo"], project_dir, args, is_user_project=False, ) project.commit_id = commit_id_data["commit"] project.sort_id = commit_id_data["sort_id"] # projects should now be installed, so load all the data from the yaml files # fill projects will load from the fill project's directory logging.debug("post clone setup") project.post_clone_setup() logging.debug(project) if args.harden: project.create_user_config() project.golden_harden() if args.update_shuttle: project.check_ports(True) project.check_num_cells() self.projects.append(project) self.projects.sort(key=lambda x: x.sort_id) all_macro_instances = [project.get_macro_name() for project in self.projects] self.assert_unique(all_macro_instances) all_gds_files = [project.get_macro_gds_filename() for project in self.projects] self.assert_unique(all_gds_files) logging.info(f"loaded {len(self.projects)} projects") def assert_unique(self, check: List[str]): duplicates = [ item for item, count in collections.Counter(check).items() if count > 1 ] if duplicates: logging.error("duplicate projects: {}".format(duplicates)) exit(1) def build_metrics(self): total_seconds = 0.0 total_wire_length = 0 total_wires_count = 0 total_physical_cells = 0 max_cells = 0 min_cells = 1000 max_cell_project = None max_util = 0.0 min_util = 100.0 max_util_project = None languages: Dict[str, int] = {} for project in self.projects: try: dt = datetime.datetime.strptime( project.metrics["total_runtime"][:-3], "%Hh%Mm%Ss" ) except KeyError: continue if project.is_chip_rom(): continue delt = datetime.timedelta( hours=dt.hour, minutes=dt.minute, seconds=dt.second ) total_seconds += delt.total_seconds() cell_count = project.get_cell_counts_from_gl() script_dir = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(script_dir, "categories.json")) as fh: categories = json.load(fh) CategoryInfo = TypedDict( "CategoryInfo", {"count": int, "examples": List[str]} ) by_category: Dict[str, CategoryInfo] = {} total = 0 for cell_name in cell_count: cat_index = categories["map"][cell_name] cat_name = categories["categories"][cat_index] if cat_name in by_category: by_category[cat_name]["count"] += cell_count[cell_name] by_category[cat_name]["examples"].append(cell_name) else: by_category[cat_name] = { "count": cell_count[cell_name], "examples": [cell_name], } if cat_name not in ["Fill", "Tap", "Buffer", "Misc"]: total += cell_count[cell_name] if total < 10: del by_category["Fill"] del by_category["Tap"] if "Buffer" in by_category: del by_category["Buffer"] print(project.get_macro_name(), total, by_category) total_wire_length += int(project.metrics["wire_length"]) total_wires_count += int(project.metrics["wires_count"]) util = float(project.metrics["OpenDP_Util"]) num_cells = project.get_cell_count_from_synth() total_physical_cells += num_cells lang = project.info.language if lang in languages: languages[lang] += 1 else: languages[lang] = 1 if num_cells > max_cells: max_cells = num_cells max_cell_project = project if num_cells < min_cells: min_cells = num_cells if util > max_util: max_util = util max_util_project = project if util < min_util: min_util = util logging.info(f"build time for all projects {total_seconds / 3600} hrs") logging.info(f"total wire length {total_wire_length} um") logging.info(f"total cells {total_physical_cells}") logging.info(f"max cells {max_cells} for project {max_cell_project}") logging.info(f"min cells {min_cells}") logging.info(f"max util {max_util} for project {max_util_project}") logging.info(f"min util {min_util}") logging.info(f"languages {languages}") if __name__ == "__main__": parser = argparse.ArgumentParser(description="TinyTapeout configuration and docs") with open("config.yaml") as fh: config = yaml.safe_load(fh) parser.add_argument( "--list", help="list projects", action="store_const", const=True ) parser.add_argument( "--single", help="do action on single project", type=int, default=-1 ) parser.add_argument( "--update-shuttle", help="configure shuttle for build", action="store_const", const=True, ) parser.add_argument( "--copy-macros", help="copy macros for building the tt_top project", action="store_const", const=True, ) parser.add_argument( "--copy-final-results", help="copy final project files to gds/lef directories", action="store_const", const=True, ) parser.add_argument( "--create-efabless-submission", help="create efabless submission files", action="store_const", const=True, ) parser.add_argument( "--harden", help="harden project", action="store_const", const=True ) parser.add_argument( "--test", help="use test projects", action="store_const", const=True ) parser.add_argument( "--sta-projects", help="use sta projects", action="store_const", const=True ) parser.add_argument( "--debug", help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO, ) parser.add_argument( "--log-email", help="print persons email in messages", action="store_const", const=True, ) parser.add_argument( "--update-image", help="update the image", action="store_const", const=True ) parser.add_argument( "--dump-json", help="dump json of all project data to given file" ) parser.add_argument( "--dump-markdown", help="dump markdown of all project data to given file" ) parser.add_argument("--dump-pdf", help="create pdf from the markdown") parser.add_argument( "--build-hugo-content", help="directory to where to build hugo content" ) parser.add_argument( "--metrics", help="print some project metrics", action="store_const", const=True ) args = parser.parse_args() # setup log log_format = logging.Formatter("%(asctime)s - %(levelname)-8s - %(message)s") # configure the client logging log = logging.getLogger("") # has to be set to debug as is the root logger log.setLevel(args.loglevel) # create console handler and set level to info ch = logging.StreamHandler(sys.stdout) # create formatter for console ch.setFormatter(log_format) log.addHandler(ch) projects = Projects(config, args) if args.test: modules_yaml_name = "modules.test.yaml" elif args.sta_projects: modules_yaml_name = "modules.sta.yaml" else: modules_yaml_name = "modules.yaml" docs = Docs(config, projects.projects) shuttle = ShuttleConfig(config, projects.projects, modules_yaml_name) rom = ROMFile(config) if args.list: shuttle.list() if args.metrics: projects.build_metrics() if args.update_shuttle: shuttle.configure_mux() rom.write_rom() if not args.test: docs.build_index() if args.copy_macros: shuttle.copy_macros() if args.copy_final_results: shuttle.copy_final_results() if args.create_efabless_submission: shuttle.create_efabless_submission() if args.update_image: docs.update_image() if args.dump_markdown: shuttle.configure_mux() docs.write_datasheet(args.dump_markdown, args.dump_pdf) if args.build_hugo_content: shuttle.configure_mux() docs.build_hugo_content(args.build_hugo_content)