wip

project.py

@@ -3,77 +3,141 @@ import logging
 import requests
 import os
 import subprocess
+import logging
+import re
+import subprocess
+import os
+import glob
+import gdstk
+import cairosvg
+import os
+import csv
+import json
+import re
+import glob
+import git_utils
+import git
+
+# can't find a google version
+CELL_URL = 'https://antmicro-skywater-pdk-docs.readthedocs.io/en/test-submodules-in-rtd/contents/libraries/sky130_fd_sc_ls/cells/'
 
 
-def load_yaml(args):
-    with open(args.yaml, "r") as stream:
-        return (yaml.safe_load(stream))
+class Project():
+
+    def __init__(self, index, git_url, local_dir, args, fill=False):
+        self.git_url = git_url
+        self.args = args
+        self.index = index
+        self.fill = fill
+        self.local_dir = local_dir
+
+    def post_clone_setup(self):
+        self.load_yaml()
+        self.setup_source_files()
+        self.load_metrics()
+
+    def load_metrics(self):
+        try:
+            with open(self.get_metrics_path()) as fh:
+                self.metrics = next(csv.DictReader(fh))
+        except FileNotFoundError:
+            self.metrics = {}
 
 
-def print_wokwi_id(yaml):
-    print(yaml['project']['wokwi_id'])
+    def check_ports(self):
+        top = self.get_macro_name()
+        sources = [os.path.join(self.local_dir, 'src', src) for src in self.src_files]
+        source_list = " ".join(sources)
+
+        json_file = 'ports.json'
+        yosys_cmd = f"yosys -qp 'read_verilog -lib -sv {source_list}; hierarchy -top {top} ; proc; write_json {json_file}'"
+        p = subprocess.run(yosys_cmd, shell=True)
+        if p.returncode != 0:
+            logging.error(f"yosys port read failed for {self}")
+            exit(1)
+
+        with open(json_file) as fh:
+            ports = json.load(fh)
+        os.unlink(json_file)
+
+        module_ports = ports['modules'][top]['ports']
+        for port in ['io_in', 'io_out']:
+            if port not in module_ports:
+                logging.error(f"{self} {port} not found in top")
+                exit(1)
+            if len(module_ports[port]['bits']) != 8:
+                logging.error(f"{self} {port} doesn't have 8 bits")
+                exit(1)
 
 
-def create_user_config(yaml):
-    logging.info("creating include file")
-    sources = get_project_source(yaml)
-    top_module = get_top_module(yaml)
-
-    if top_module == 'top':
-        logging.error("top module cannot be called top - prepend your repo name to make it unique")
-        exit(1)
-
-    filename = 'user_config.tcl'
-    with open(os.path.join('src', filename), 'w') as fh:
-        fh.write("set ::env(DESIGN_NAME) {}\n".format(top_module))
-        fh.write('set ::env(VERILOG_FILES) "\\\n')
-        for line, source in enumerate(sources):
-            fh.write("    $::env(DESIGN_DIR)/" + source)
-            if line != len(sources) - 1:
-                fh.write(' \\\n')
-        fh.write('"\n')
+    def check_num_cells(self):
+        num_cells = self.get_cell_count_from_gl()
+        if self.is_hdl():
+            if num_cells < 20:
+                logging.warning(f"{self} only has {num_cells} cells")
+        else:
+            if num_cells < 11:
+                logging.warning(f"{self} only has {num_cells} cells")
 
 
-def fetch_file(url, filename):
-    logging.info("trying to download {}".format(url))
-    r = requests.get(url)
-    if r.status_code != 200:
-        logging.warning("couldn't download {}".format(url))
-        exit(1)
-
-    with open(filename, 'wb') as fh:
-        logging.info("written to {}".format(filename))
-        fh.write(r.content)
+    def is_fill(self):
+        return self.fill
 
 
-def get_project_source(yaml):
-    # wokwi_id must be an int or 0
-    try:
-        wokwi_id = int(yaml['project']['wokwi_id'])
-    except ValueError:
-        logging.error("wokwi id must be an integer")
-        exit(1)
+    def is_wokwi(self):
+        if self.wokwi_id != 0:
+            return True
 
-    # it's a wokwi project
-    if wokwi_id != 0:
-        url = "https://wokwi.com/api/projects/{}/verilog".format(wokwi_id)
-        src_file = "user_module_{}.v".format(wokwi_id)
-        fetch_file(url, os.path.join("src", src_file))
 
-        # also fetch the wokwi diagram
-        url = "https://wokwi.com/api/projects/{}/diagram.json".format(wokwi_id)
-        diagram_file = "wokwi_diagram.json"
-        fetch_file(url, os.path.join("src", diagram_file))
+    def is_hdl(self):
+        return not self.is_wokwi()
 
+    def load_yaml(self):
+        try:
+            with open(os.path.join(self.local_dir, 'info.yaml')) as fh:
+                self.yaml = yaml.safe_load(fh)
+        except FileNotFoundError:
+            logging.error(f"yaml file not found for {self} - do you need to --clone the project repos?")
+            exit(1)
+
+
+    # find and save the location of the source files
+    # get name of top module and the verilog module that contains the top
+    def setup_source_files(self):
+        # wokwi_id must be an int or 0
+        try:
+            self.wokwi_id = int(self.yaml['project']['wokwi_id'])
+        except ValueError:
+            logging.error("wokwi id must be an integer")
+            exit(1)
+
+        self.yaml['project']['git_url'] = self.git_url
+
+        if self.is_hdl():
+            self.top_module             = self.yaml['project']['top_module']
+            self.src_files              = self.get_hdl_source()
+            self.top_verilog_filename   = self.find_top_verilog()
+        else:
+            self.top_module             = f"user_module_{self.wokwi_id}"
+            self.src_files              = self.get_wokwi_source()
+            self.top_verilog_filename   = self.src_files[0]
+
+        self.macro_instance         = f"{self.top_module}_{self.index :03}"
+        self.scanchain_instance     = f"scanchain_{self.index :03}"
+
+
+    def get_wokwi_source(self):
+        src_file = "user_module_{}.v".format(self.wokwi_id)
         return [src_file, 'cells.v']
 
-    # else it's HDL, so check source files
-    else:
-        if 'source_files' not in yaml['project']:
+
+
+    def get_hdl_source(self):
+        if 'source_files' not in self.yaml['project']:
             logging.error("source files must be provided if wokwi_id is set to 0")
             exit(1)
 
-        source_files = yaml['project']['source_files']
+        source_files = self.yaml['project']['source_files']
         if source_files is None:
             logging.error("must be more than 1 source file")
             exit(1)
@@ -82,34 +146,402 @@ def get_project_source(yaml):
             logging.error("must be more than 1 source file")
             exit(1)
 
-        if 'top_module' not in yaml['project']:
+        if 'top_module' not in self.yaml['project']:
             logging.error("must provide a top module name")
             exit(1)
 
+        if self.yaml['project']['top_module'] == 'top':
+            logging.error("top module cannot be called top - prepend your repo name to make it unique")
+            exit(1)
+
         for filename in source_files:
             if '*' in filename:
                 logging.error("* not allowed, please specify each file")
                 exit(1)
-            if not os.path.exists(os.path.join('src', filename)):
+            if not os.path.exists(os.path.join(self.local_dir, 'src', filename)):
                 logging.error(f"{filename} doesn't exist in the repo")
                 exit(1)
 
         return source_files
 
 
-def get_top_module(yaml):
-    wokwi_id = int(yaml['project']['wokwi_id'])
-    if wokwi_id != 0:
-        return "user_module_{}".format(wokwi_id)
-    else:
-        return yaml['project']['top_module']
+    def get_yaml(self):
+        return self.yaml
+
+    def get_hugo_row(self):
+        return f'| {self.index} | [{self.yaml["documentation"]["title"]}]({self.index :03}) | {self.yaml["documentation"]["author"]}|\n'
+
+    # docs stuff for index on README.md
+    def get_index_row(self):
+        return f'| {self.index} | {self.yaml["documentation"]["author"]} | {self.yaml["documentation"]["title"]} | {self.get_project_type_string()} | {self.git_url} |\n'
+
+    def get_project_type_string(self):
+        if self.is_wokwi():
+            return f"[Wokwi]({self.get_wokwi_url()})"
+        else:
+            return "HDL"
+
+    def get_project_doc_yaml(self):
+        # fstring dict support is limited to one level deep, so put the git url and wokwi url in the docs key
+        docs = self.yaml['documentation']
+        docs['project_type'] = self.get_project_type_string()
+        docs['git_url'] = self.git_url
+        return docs
+
+    def get_wokwi_url(self):
+        return f'https://wokwi.com/projects/{self.wokwi_id}'
+
+    # top module name is defined in one of the source files, which one?
+    def find_top_verilog(self):
+        rgx_mod  = re.compile(r"(?:^|[\W])module[\s]{1,}([\w]+)")
+        top_verilog = []
+        for src in self.src_files:
+            with open(os.path.join(self.local_dir, 'src', src)) as fh:
+                for line in fh.readlines():
+                    for match in rgx_mod.finditer(line):
+                        if match.group(1) == self.top_module:
+                            top_verilog.append(src)
+        assert len(top_verilog) == 1
+        return top_verilog[0]
+
+    def clone(self):
+        if os.path.exists(self.local_dir):
+            git_remote = list(git.Repo(self.local_dir).remotes[0].urls)[0]
+            if self.git_url == git_remote:
+                logging.info("git repo already exists and is correct - skipping")
+            else:
+                logging.error("git repo exists and remote doesn't match - abort")
+                logging.error(f"{self.git_url} != {git_remote}")
+                exit(1)
+        else:
+            logging.info("clone")
+            git.Repo.clone_from(self.git_url, self.local_dir, recursive=True)
+
+    def pull(self):
+        repo = git.Repo(self.local_dir)
+        # reset
+        repo.git.reset('--hard')
+        o = repo.remotes.origin
+        o.pull()
 
 
-def harden():
-    # requires PDK, PDK_ROOT, OPENLANE_ROOT & OPENLANE_IMAGE_NAME to be set in local environment
-    harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK=$PDK -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -tag wokwi"'
-    env = os.environ.copy()
-    p = subprocess.run(harden_cmd, shell=True, env=env)
-    if p.returncode != 0:
-        logging.error("harden failed")
-        exit(1)
+    def __str__(self):
+#        if self.args.log_email:
+#            return f"[{self.index:03} : {self.email} : {self.git_url}]"
+#        else:
+        return f"[{self.index:03} : {self.git_url}]"
+
+    def fetch_gds(self):
+        install_artifacts(self.git_url, self.local_dir)
+
+    def get_latest_action_url(self):
+        return get_latest_action_url(self.git_url, self.local_dir)
+
+    def get_macro_name(self):
+        return self.top_module
+
+    # instance name of the project, different for each id
+    def get_macro_instance(self):
+        return self.macro_instance
+
+    # instance name of the scanchain module, different for each id
+    def get_scanchain_instance(self):
+        return self.scanchain_instance
+
+    # unique id
+    def get_index(self):
+        return self.index
+
+    # metrics
+    def get_metrics_path(self):
+        return os.path.join(self.local_dir, 'runs/wokwi/reports/metrics.csv')
+
+    # name of the gds file
+    def get_macro_gds_filename(self):
+        return f"{self.top_module}.gds"
+
+    def get_macro_lef_filename(self):
+        return f"{self.top_module}.lef"
+
+    # for GL sims & blackboxing
+    def get_gl_verilog_filename(self):
+        return f"{self.top_module}.v"
+
+    # for simulations
+    def get_top_verilog_filename(self):
+        if self.is_hdl():
+            # make sure it's unique & without leading directories
+            # a few people use 'top.v', which is OK as long as the top module is called something more unique
+            # but then it needs to be made unique so the source can be found
+            filename = os.path.basename(self.top_verilog_filename)
+            return f'{self.index :03}_{filename}'
+        else:
+            return self.top_verilog_filename
+
+    def get_git_url(self):
+        return self.git_url
+
+    # todo add spef
+    def copy_files_to_caravel(self):
+        files = [
+            (f"runs/wokwi/results/final/gds/{self.get_macro_gds_filename()}", f"gds/{self.get_macro_gds_filename()}"),
+            (f"runs/wokwi/results/final/lef/{self.get_macro_lef_filename()}", f"lef/{self.get_macro_lef_filename()}"),
+            (f"runs/wokwi/results/final/verilog/gl/{self.get_gl_verilog_filename()}", f"verilog/gl/{self.get_gl_verilog_filename()}"),
+            (f"src/{self.top_verilog_filename}", f"verilog/rtl/{self.get_top_verilog_filename()}"),
+            ]
+
+        logging.debug("copying files into position")
+        for from_path, to_path in files:
+            from_path = os.path.join(self.local_dir, from_path)
+            logging.debug(f"copy {from_path} to {to_path}")
+            shutil.copyfile(from_path, to_path)
+
+
+    def print_wokwi_id(self):
+        print(self.wokwi_id)
+
+
+    def fetch_wokwi_files(self):
+        logging.info("fetching wokwi files")
+        url = "https://wokwi.com/api/projects/{}/verilog".format(self.wokwi_id)
+        git_utils.fetch_file(url, os.path.join(self.local_dir, "src", src_file))
+
+        # also fetch the wokwi diagram
+        url = "https://wokwi.com/api/projects/{}/diagram.json".format(self.wokwi_id)
+        diagram_file = "wokwi_diagram.json"
+        git_utils.fetch_file(url, os.path.join(self.local_dir, "src", diagram_file))
+
+
+    def create_user_config(self):
+        self.fetch_wokwi_files()
+
+        logging.info("creating include file")
+        filename = 'user_config.tcl'
+        with open(os.path.join(self.local_dir, 'src', filename), 'w') as fh:
+            fh.write("set ::env(DESIGN_NAME) {}\n".format(self.top_module))
+            fh.write('set ::env(VERILOG_FILES) "\\\n')
+            for line, source in enumerate(self.src_files):
+                fh.write("    $::env(DESIGN_DIR)/" + source)
+                if line != len(self.src_files) - 1:
+                    fh.write(' \\\n')
+            fh.write('"\n')
+
+
+    """
+    def harden(self):
+        logging.info(f"hardening {self}")
+
+        # copy golden config
+        shutil.copyfile('golden_config.tcl', os.path.join(self.local_dir, 'src', 'config.tcl'))
+
+        cwd = os.getcwd()
+        os.chdir(self.local_dir)
+
+        # setup user config, not including python fails on github action
+        if 'LOCAL' in os.environ:
+            configure_cmd = './configure.py --create-user-config'
+        else:
+            configure_cmd = 'python ./configure.py --create-user-config'
+        p = subprocess.run(configure_cmd, shell=True)
+        if p.returncode != 0:
+            logging.error(f"configure failed for {self}")
+            exit(1)
+
+        # requires PDK_ROOT, OPENLANE_ROOT & OPENLANE_IMAGE_NAME to be set in local environment
+        harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK=$PDK -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -tag wokwi"'
+        env = os.environ.copy()
+        p = subprocess.run(harden_cmd, shell=True, env=env)
+        if p.returncode != 0:
+            logging.error(f"harden failed for {self}")
+            exit(1)
+
+        os.chdir(cwd)
+    """
+
+    def harden(self):
+        cwd = os.getcwd()
+        os.chdir(self.local_dir)
+        # requires PDK, PDK_ROOT, OPENLANE_ROOT & OPENLANE_IMAGE_NAME to be set in local environment
+        harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK=$PDK -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -tag wokwi"'
+        logging.debug(harden_cmd)
+        env = os.environ.copy()
+        p = subprocess.run(harden_cmd, shell=True, env=env)
+        if p.returncode != 0:
+            logging.error("harden failed")
+            exit(1)
+
+        os.chdir(cwd)
+
+    # doc check
+    # makes sure that the basic info is present
+    def check_yaml_docs(self):
+        yaml = self.yaml
+        logging.info("checking docs")
+        for key in ['author', 'title', 'description', 'how_it_works', 'how_to_test', 'language', 'inputs', 'outputs']:
+            if key not in yaml['documentation']:
+                logging.error("missing key {} in documentation".format(key))
+                exit(1)
+            if yaml['documentation'][key] == "":
+                logging.error("missing value for {} in documentation".format(key))
+                exit(1)
+
+        # if provided, check discord handle is valid
+        if len(yaml['documentation']['discord']):
+            parts = yaml['documentation']['discord'].split('#')
+            if len(parts) != 2 or len(parts[0]) == 0 or not re.match('^[0-9]{4}$', parts[1]):
+                logging.error('Invalid format for discord username')
+                exit(1)
+
+
+    # use pandoc to create a single page PDF preview
+    def create_pdf(self):
+        yaml = self.yaml
+        yaml = yaml['documentation']
+        logging.info("creating pdf")
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        with open(os.path.join(script_dir, "docs/project_header.md")) as fh:
+            doc_header = fh.read()
+        with open(os.path.join(script_dir, "docs/project_preview.md")) as fh:
+            doc_template = fh.read()
+
+        with open('datasheet.md', 'w') as fh:
+            fh.write(doc_header)
+            # handle pictures
+            yaml['picture_link'] = ''
+            if yaml['picture']:
+                # skip SVG for now, not supported by pandoc
+                picture_name = yaml['picture']
+                if 'svg' not in picture_name:
+                    yaml['picture_link'] = '![picture]({})'.format(picture_name)
+                else:
+                    logging.warning("svg not supported")
+
+            # now build the doc & print it
+            try:
+                doc = doc_template.format(**yaml)
+                fh.write(doc)
+                fh.write("\n\pagebreak\n")
+            except IndexError:
+                logging.warning("missing pins in info.yaml, skipping")
+
+        pdf_cmd = 'pandoc --pdf-engine=xelatex -i datasheet.md -o datasheet.pdf'
+        logging.info(pdf_cmd)
+        p = subprocess.run(pdf_cmd, shell=True)
+        if p.returncode != 0:
+            logging.error("pdf command failed")
+
+
+    # SVG and PNG renders of the GDS
+    def create_svg(self):
+        gds = glob.glob(os.path.join(self.local_dir, 'runs/wokwi/results/final/gds/*gds'))
+        library = gdstk.read_gds(gds[0])
+        top_cells = library.top_level()
+        top_cells[0].write_svg('gds_render.svg')
+
+        if self.args.create_png:
+            cairosvg.svg2png(url='gds_render.svg', write_to='gds_render.png')
+
+
+    def print_warnings(self):
+        warnings = []
+        with open(os.path.join(self.local_dir, 'runs/wokwi/logs/synthesis/1-synthesis.log')) as f:
+            for line in f.readlines():
+                if line.startswith('Warning:'):
+                    # bogus warning https://github.com/YosysHQ/yosys/commit/bfacaddca8a2e113e4bc3d6177612ccdba1555c8
+                    if 'WIDTHLABEL' not in line:
+                        warnings.append(line.strip())
+        if len(warnings):
+            print('# Synthesis warnings')
+            print()
+            for warning in warnings:
+                print(f'* {warning}')
+
+
+    def print_stats(self):
+        print('# Routing stats')
+        print()
+        print('| Utilisation (%) | Wire length (um) |')
+        print('|-------------|------------------|')
+        print('| {} | {} |'.format(self.metrics['OpenDP_Util'], self.metrics['wire_length']))
+
+
+    # Print the summaries
+    def summarize(self):
+        cell_count = self.get_cell_count_from_gl()
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+
+        with open(os.path.join(script_dir, 'categories.json')) as fh:
+            categories = json.load(fh)
+        with open(os.path.join(script_dir, 'defs.json')) as fh:
+            defs = json.load(fh)
+
+        # print all used cells, sorted by frequency
+        total = 0
+        if self.args.print_cell_summary:
+            print('# Cell usage')
+            print()
+            print('| Cell Name | Description | Count |')
+            print('|-----------|-------------|-------|')
+            for name, count in sorted(cell_count.items(), key=lambda item: item[1], reverse=True):
+                if count > 0:
+                    total += count
+                    cell_link = f'{CELL_URL}{name}'
+                    print(f'| [{name}]({cell_link}) | {defs[name]["description"]} |{count} |')
+
+            print(f'| | Total | {total} |')
+
+        if self.args.print_cell_category:
+            by_category = {}
+            for cell_name in cell_count:
+                cat_index = categories['map'][cell_name]
+                cat_name = categories['categories'][cat_index]
+                if cat_name in by_category:
+                    by_category[cat_name]['count'] += cell_count[cell_name]
+                    by_category[cat_name]['examples'].append(cell_name)
+                else:
+                    by_category[cat_name] = {'count' : cell_count[cell_name], 'examples' : [cell_name]}
+
+            print('# Cell usage by Category')
+            print()
+            print('| Category | Cells | Count |')
+            print('|---------------|----------|-------|')
+            for cat_name, cat_dict in sorted(by_category.items(), key=lambda x: x[1]['count'], reverse=True):
+                cell_links = [f'[{name}]({CELL_URL}{name})' for name in cat_dict['examples']]
+                print(f'|{cat_name} | {" ".join(cell_links)} | {cat_dict["count"]}|')
+
+
+    # Parse the lib, cell and drive strength an OpenLane gate-level Verilog file
+    def get_cell_count_from_gl(self):
+        cell_count = {}
+        total = 0
+        gl_files = glob.glob(os.path.join(self.local_dir, 'runs/wokwi/results/final/verilog/gl/*.nl.v'))
+        with open(gl_files[0]) as fh:
+            for line in fh.readlines():
+                m = re.search(r'sky130_(\S+)__(\S+)_(\d+)', line)
+                if m is not None:
+                    total += 1
+                    cell_lib = m.group(1)
+                    cell_name = m.group(2)
+                    cell_drive = m.group(3)
+                    assert cell_lib in ['fd_sc_hd', 'ef_sc_hd']
+                    assert int(cell_drive) > 0
+                    try:
+                        cell_count[cell_name] += 1
+                    except KeyError:
+                        cell_count[cell_name] = 1
+        return (cell_count)
+
+
+    # Load all the json defs and combine into one big dict, keyed by cellname
+    # Needs access to the PDK
+    def create_defs(self):
+        # replace this with a find
+        json_files = glob.glob('sky130_fd_sc_hd/latest/cells/*/definition.json')
+        definitions = {}
+        for json_file in json_files:
+            with open(json_file) as fh:
+                definition = json.load(fh)
+                definitions[definition['name']] = definition
+
+        with open('defs.json', 'w') as fh:
+            json.dump(definitions, fh)