#!/usr/bin/env python3 from orders import Orders import datetime import yaml import argparse, logging, sys, os, collections from project import Project from documentation import Docs from caravel import CaravelConfig # pipe handling from signal import signal, SIGPIPE, SIG_DFL signal(SIGPIPE, SIG_DFL) class Projects(): def __init__(self, config, args): self.args = args self.config = config if args.test: self.projects_file = 'test_projects.yaml' self.project_dir = config['test_project_dir'] else: self.projects_file = 'projects.yaml' self.project_dir = config['project_dir'] if not os.path.exists(self.project_dir): os.makedirs(self.project_dir) try: with open(self.projects_file) as fh: self.project_config = yaml.safe_load(fh) except FileNotFoundError: logging.error(f"{self.projects_file} not found, create it with --update-orders") exit(1) """ # useful for contacting people when I find problems in the repo try: with open('git_url_to_email.json') as fh: self.git_url_to_email_map = json.load(fh) except FileNotFoundError: self.git_url_to_email_map = {} """ self.projects = [] for index in range(args.limit_num_projects): if args.single >= 0 and args.single != index: continue if index < args.start_from: continue if args.end_at != 0 and index > args.end_at: continue """ try: email = self.git_url_to_email_map[git_url] except KeyError: email = 'none' """ git_url = self.project_config[index]['url'] filler = self.project_config[index]['fill'] # not sure if this is a good idea if self.project_config[index]['status'] == 'disable': filler = True # fill projects don't have their own directory, reuse the first project which is always fill if filler: project_dir = os.path.join(os.path.join(self.project_dir, f'{0 :03}')) else: project_dir = os.path.join(os.path.join(self.project_dir, f'{index :03}')) project = Project(index, git_url, project_dir, args, fill=filler) # clone git repos locally & gds artifacts from action build if args.clone_all: if filler is False: logging.info(f"cloning {project}") project.clone() if args.fetch_gds: if filler is False: logging.info(f"fetching gds for {project}") project.fetch_gds() if args.update_all: if filler is False: # only updates code, not gds artifacts logging.info(f"git pull for {project}") project.pull() # projects should now be installed, so load all the data from the yaml files # fill projects will load from the fill project's directory logging.debug("post clone setup") project.post_clone_setup() logging.debug(project) if args.harden: if filler is False: project.create_user_config() project.golden_harden() if args.update_caravel: logging.debug("copying files to caravel") if filler is False: project.copy_files_to_caravel() # check all top level module ports are correct project.check_ports() project.check_num_cells() self.projects.append(project) # now do some sanity checks for project in self.projects: if project.get_git_remote() != project.git_url: logging.warning(f"{project} doesn't match remote: {project.get_git_remote()}") exit(1) all_macro_instances = [project.get_macro_instance() for project in self.projects] self.assert_unique(all_macro_instances) all_top_files = [project.get_top_verilog_filename() for project in self.projects if not project.is_fill()] self.assert_unique(all_top_files) all_gds_files = [project.get_macro_gds_filename() for project in self.projects if not project.is_fill()] self.assert_unique(all_gds_files) logging.info(f"loaded {len(self.projects)} projects") def assert_unique(self, check): duplicates = [item for item, count in collections.Counter(check).items() if count > 1] if duplicates: logging.error("duplicate projects: {}".format(duplicates)) exit(1) def build_metrics(self): total_seconds = 0 total_wire_length = 0 total_wires_count = 0 total_physical_cells = 0 max_cells = 0 min_cells = 1000 max_cell_project = None max_util = 0 min_util = 100 max_util_project = None languages = {} tags = [] for project in self.projects: if not project.is_fill(): try: dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss') except KeyError: continue delt = datetime.timedelta(hours=dt.hour, minutes=dt.minute, seconds=dt.second) total_seconds += delt.total_seconds() total_wire_length += int(project.metrics['wire_length']) total_wires_count += int(project.metrics['wires_count']) util = float(project.metrics['OpenDP_Util']) num_cells = project.get_cell_count_from_synth() total_physical_cells += num_cells yaml_data = project.get_project_doc_yaml() lang = yaml_data['language'].lower() if lang in languages: languages[lang] += 1 else: languages[lang] = 1 if num_cells > max_cells: max_cells = num_cells max_cell_project = project if num_cells < min_cells: min_cells = num_cells if util > max_util: max_util = util max_util_project = project if util < min_util: min_util = util try: tags += yaml_data['tag'].split(',') except KeyError: pass logging.info(f"build time for all projects {total_seconds / 3600} hrs") logging.info(f"total wire length {total_wire_length} um") logging.info(f"total cells {total_physical_cells}") logging.info(f"max cells {max_cells} for project {max_cell_project}") logging.info(f"min cells {min_cells}") logging.info(f"max util {max_util} for project {max_util_project}") logging.info(f"min util {min_util}") logging.info(f"languages {languages}") tags = [x.strip().lower() for x in tags] tags = list(filter(lambda a: a != "", tags)) def count_items(lst): freq = {} for item in lst: if item not in freq: freq[item] = 1 else: freq[item] += 1 sorted_items = sorted(freq.items(), key=lambda x: x[1], reverse=True) for item, count in sorted_items[0:9]: logging.info(f"{item:10}: {count}") logging.info("top 10 tags") count_items(tags) if __name__ == '__main__': parser = argparse.ArgumentParser(description="TinyTapeout configuration and docs") with open('config.yaml') as fh: config = yaml.safe_load(fh) parser.add_argument('--update-orders', help="update the order config file", action="store_const", const=True) parser.add_argument('--list', help="list projects", action='store_const', const=True) parser.add_argument('--clone-all', help="clone all projects", action="store_const", const=True) parser.add_argument('--update-all', help="git pull all projects", action="store_const", const=True) parser.add_argument('--fetch-gds', help="git fetch latest gds", action="store_const", const=True) parser.add_argument('--single', help="do action on single project", type=int, default=-1) parser.add_argument('--start-from', help="do action on projects after this index", type=int, default=0) parser.add_argument('--end-at', help="do action on projects before this index", type=int, default=0) parser.add_argument('--update-caravel', help='configure caravel for build', action='store_const', const=True) parser.add_argument('--harden', help="harden project", action="store_const", const=True) parser.add_argument('--limit-num-projects', help='only configure for the first n projects', type=int, default=int(config['num_projects'])) parser.add_argument('--test', help='use test projects', action='store_const', const=True) parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) parser.add_argument('--log-email', help="print persons email in messages", action="store_const", const=True) parser.add_argument('--update-image', help="update the image", action="store_const", const=True) parser.add_argument('--dump-json', help="dump json of all project data to given file") parser.add_argument('--dump-markdown', help="dump markdown of all project data to given file") parser.add_argument('--dump-pdf', help="create pdf from the markdown") parser.add_argument('--build-hugo-content', help="directory to where to build hugo content") parser.add_argument('--metrics', help="print some project metrics", action="store_const", const=True) parser.add_argument('--add-extra-project', help="specify a github url to add. Project is added to the extra_projects list specifed in config.yaml") parser.add_argument('--sram-support', help="include sram LEF/GDS in caravel config", action="store_const", const=True) args = parser.parse_args() # setup log log_format = logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s') # configure the client logging log = logging.getLogger('') # has to be set to debug as is the root logger log.setLevel(args.loglevel) # create console handler and set level to info ch = logging.StreamHandler(sys.stdout) # create formatter for console ch.setFormatter(log_format) log.addHandler(ch) if args.update_orders: # don't instantiate orders unless we have to as it needs stripe token orders = Orders(config) orders.fetch_orders() orders.update_project_list() if args.add_extra_project: orders = Orders(config) orders.add_extra_project(args.add_extra_project) projects = Projects(config, args) docs = Docs(projects.projects, args=args) caravel = CaravelConfig(config, projects.projects, num_projects=args.limit_num_projects) if args.list: caravel.list() if args.metrics: projects.build_metrics() if args.update_caravel: extra_macros = [] if args.sram_support: extra_macros = ["sky130_sram_2kbyte_1rw1r_32x512_8"] caravel.instantiate(extra_macros) if not args.test: docs.build_index() if args.update_image: docs.update_image() if args.dump_json: docs.dump_json() if args.dump_markdown: docs.dump_markdown() if args.build_hugo_content: docs.build_hugo_content()