WiP TT04 update

This commit is contained in:
Uri Shaked
2023-07-20 16:51:20 +03:00
parent bb58ee1fae
commit 5847f76782
6 changed files with 140 additions and 504 deletions
+44 -22
View File
@@ -1,24 +1,21 @@
import logging import logging
import json
import os import os
import yaml
import json
class CaravelConfig(): class CaravelConfig():
def __init__(self, config, projects, num_projects): def __init__(self, config, projects, modules_yaml_name: str):
self.config = config self.config = config
self.projects = projects self.projects = projects
self.num_projects = num_projects
self.script_dir = os.path.dirname(os.path.realpath(__file__)) self.script_dir = os.path.dirname(os.path.realpath(__file__))
self.modules_yaml_name = modules_yaml_name
# instantiate inside user_project_wrapper # instantiate inside user_project_wrapper
def instantiate(self, extra_macros=[]): def instantiate(self):
# build the blackbox_project_includes.v file - used for blackboxing when building the GDS # build the blackbox_project_includes.v file - used for blackboxing when building the GDS
with open('verilog/blackbox_project_includes.v', 'w') as fh: with open('verilog/blackbox_project_includes.v', 'w') as fh:
for macro_name in extra_macros:
fh.write(f'`include "rtl/{macro_name}.v"\n')
for project in self.projects: for project in self.projects:
if not project.is_fill():
fh.write(f'`include "gl/{project.get_gl_verilog_filename()}"\n') fh.write(f'`include "gl/{project.get_gl_verilog_filename()}"\n')
# build complete list of filenames for sim # build complete list of filenames for sim
@@ -37,11 +34,8 @@ class CaravelConfig():
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_tbuf.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_tbuf.v\n')
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_zbuf.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/proto/prim_generic/tt_prim_zbuf.v\n')
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/cells.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/cells.v\n')
for macro_name in extra_macros: # for project in self.projects:
fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{macro_name}.v\n') # fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{project.get_top_verilog_filename()}\n')
for project in self.projects:
if not project.is_fill():
fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{project.get_top_verilog_filename()}\n')
with open('verilog/includes/includes.rtl.caravel_user_project.openlane2', 'w') as fh: with open('verilog/includes/includes.rtl.caravel_user_project.openlane2', 'w') as fh:
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/ol2/tt_top/user_project_wrapper.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/ol2/tt_top/user_project_wrapper.v\n')
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/tt_top.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/tt_top.v\n')
@@ -58,24 +52,52 @@ class CaravelConfig():
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_tie.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_tie.v\n')
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_zbuf.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/tt-multiplexer/rtl/prim_generic/tt_prim_zbuf.v\n')
fh.write('-v $(USER_PROJECT_VERILOG)/rtl/cells.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/rtl/cells.v\n')
for macro_name in extra_macros: # for project in self.projects:
fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{macro_name}.v\n') # fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{project.get_top_verilog_filename()}\n')
for project in self.projects:
if not project.is_fill():
fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{project.get_top_verilog_filename()}\n')
# build GL includes # build GL includes
with open('verilog/includes/includes.gl.caravel_user_project', 'w') as fh: with open('verilog/includes/includes.gl.caravel_user_project', 'w') as fh:
fh.write('-v $(USER_PROJECT_VERILOG)/gl/user_project_wrapper.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/gl/user_project_wrapper.v\n')
fh.write('-v $(USER_PROJECT_VERILOG)/gl/tt_ctrl.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/gl/tt_ctrl.v\n')
fh.write('-v $(USER_PROJECT_VERILOG)/gl/tt_mux.v\n') fh.write('-v $(USER_PROJECT_VERILOG)/gl/tt_mux.v\n')
for macro_name in extra_macros:
fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{macro_name}.v\n')
for project in self.projects: for project in self.projects:
if not project.is_fill():
fh.write(f'-v $(USER_PROJECT_VERILOG)/gl/{project.get_gl_verilog_filename()}\n') fh.write(f'-v $(USER_PROJECT_VERILOG)/gl/{project.get_gl_verilog_filename()}\n')
with open(self.modules_yaml_name, 'r') as modules_file:
module_config = yaml.safe_load(modules_file)
configured_macros = set(map(lambda mod: mod['name'], module_config['modules']))
logging.info(f"found {len(configured_macros)} preconfigured macros: {configured_macros}")
for project in self.projects:
if project.unprefixed_name not in configured_macros:
module_config['modules'].append({'name': project.unprefixed_name})
with open('tt-multiplexer/cfg/modules.yaml', 'w') as mux_modules_file:
yaml.dump(module_config, mux_modules_file)
res = os.system("make -C tt-multiplexer gensrc")
if res != 0:
logging.error("Failed to generate multiplexer placement configuration")
exit(1)
mux_index = {}
mux_index_reverse = {}
with open('tt-multiplexer/cfg/modules_placed.yaml') as placed_modules_file:
placed_modules = yaml.safe_load(placed_modules_file)
for module in placed_modules['modules']:
mux_address = (module['y'] << 5) + module['x']
module_name = 'tt_um_' + module['name']
mux_index[mux_address] = module_name
mux_index_reverse[module_name] = mux_address
for project in self.projects:
if project.top_module not in mux_index_reverse:
logging.error(f"no placement found for {project}!")
exit(1)
project.mux_address = mux_index_reverse[project.top_module]
with open('mux_index.json', 'w') as mux_index_file:
json.dump(mux_index, mux_index_file)
def list(self): def list(self):
for project in self.projects: for project in self.projects:
if not project.is_fill():
logging.info(project) logging.info(project)
+22 -103
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from orders import Orders
import datetime import datetime
import yaml import yaml
import json
import argparse, logging, sys, os, collections import argparse, logging, sys, os, collections
from project import Project from project import Project
from documentation import Docs from documentation import Docs
@@ -21,78 +21,29 @@ class Projects():
self.config = config self.config = config
if args.test: if args.test:
self.projects_file = 'test_projects.yaml'
self.project_dir = config['test_project_dir'] self.project_dir = config['test_project_dir']
else: else:
self.projects_file = 'projects.yaml'
self.project_dir = config['project_dir'] self.project_dir = config['project_dir']
if not os.path.exists(self.project_dir): if not os.path.exists(self.project_dir):
os.makedirs(self.project_dir) os.makedirs(self.project_dir)
try:
with open(self.projects_file) as fh:
self.project_config = yaml.safe_load(fh)
except FileNotFoundError:
logging.error(f"{self.projects_file} not found, create it with --update-orders")
exit(1)
"""
# useful for contacting people when I find problems in the repo
try:
with open('git_url_to_email.json') as fh:
self.git_url_to_email_map = json.load(fh)
except FileNotFoundError:
self.git_url_to_email_map = {}
"""
self.projects = [] self.projects = []
for index in range(args.limit_num_projects): project_list = [entry for entry in os.listdir(self.project_dir) if os.path.join(self.project_dir, entry)]
if args.single >= 0 and args.single != index: if args.sta_projects:
continue project_list = ['tt_um_loopback']
if index < args.start_from:
continue for index, project_id in enumerate(project_list):
if args.end_at != 0 and index > args.end_at: project_dir = os.path.join(self.project_dir, project_id)
commit_id_file = os.path.join(project_dir, 'commit_id.json')
if not os.path.exists(commit_id_file):
logging.warning(f"no commit_id.json in {project_dir}, skipping")
continue continue
""" commit_id_data = json.load(open(commit_id_file))
try:
email = self.git_url_to_email_map[git_url]
except KeyError:
email = 'none'
"""
git_url = self.project_config[index]['url'] project = Project(index, commit_id_data['repo'], project_dir, args)
filler = self.project_config[index]['fill']
# not sure if this is a good idea
if self.project_config[index]['status'] == 'disable':
filler = True
# fill projects don't have their own directory, reuse the first project which is always fill
if filler:
project_dir = os.path.join(os.path.join(self.project_dir, f'{0 :03}'))
else:
project_dir = os.path.join(os.path.join(self.project_dir, f'{index :03}'))
project = Project(index, git_url, project_dir, args, fill=filler)
# clone git repos locally & gds artifacts from action build
if args.clone_all:
if filler is False:
logging.info(f"cloning {project}")
project.clone()
if args.fetch_gds:
if filler is False:
logging.info(f"fetching gds for {project}")
project.fetch_gds()
if args.update_all:
if filler is False:
# only updates code, not gds artifacts
logging.info(f"git pull for {project}")
project.pull()
# projects should now be installed, so load all the data from the yaml files # projects should now be installed, so load all the data from the yaml files
# fill projects will load from the fill project's directory # fill projects will load from the fill project's directory
@@ -101,34 +52,19 @@ class Projects():
logging.debug(project) logging.debug(project)
if args.harden: if args.harden:
if filler is False:
project.create_user_config() project.create_user_config()
project.golden_harden() project.golden_harden()
if args.update_caravel: if args.update_caravel:
logging.debug("copying files to caravel")
if filler is False:
project.copy_files_to_caravel()
# check all top level module ports are correct
project.check_ports() project.check_ports()
project.check_num_cells() project.check_num_cells()
self.projects.append(project) self.projects.append(project)
# now do some sanity checks
for project in self.projects:
if project.get_git_remote() != project.git_url:
logging.warning(f"{project} doesn't match remote: {project.get_git_remote()}")
exit(1)
all_macro_instances = [project.get_macro_instance() for project in self.projects] all_macro_instances = [project.get_macro_instance() for project in self.projects]
self.assert_unique(all_macro_instances) self.assert_unique(all_macro_instances)
all_top_files = [project.get_top_verilog_filename() for project in self.projects if not project.is_fill()] all_gds_files = [project.get_macro_gds_filename() for project in self.projects]
self.assert_unique(all_top_files)
all_gds_files = [project.get_macro_gds_filename() for project in self.projects if not project.is_fill()]
self.assert_unique(all_gds_files) self.assert_unique(all_gds_files)
logging.info(f"loaded {len(self.projects)} projects") logging.info(f"loaded {len(self.projects)} projects")
@@ -154,7 +90,6 @@ class Projects():
tags = [] tags = []
for project in self.projects: for project in self.projects:
if not project.is_fill():
try: try:
dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss') dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss')
except KeyError: except KeyError:
@@ -226,18 +161,12 @@ if __name__ == '__main__':
with open('config.yaml') as fh: with open('config.yaml') as fh:
config = yaml.safe_load(fh) config = yaml.safe_load(fh)
parser.add_argument('--update-orders', help="update the order config file", action="store_const", const=True)
parser.add_argument('--list', help="list projects", action='store_const', const=True) parser.add_argument('--list', help="list projects", action='store_const', const=True)
parser.add_argument('--clone-all', help="clone all projects", action="store_const", const=True)
parser.add_argument('--update-all', help="git pull all projects", action="store_const", const=True)
parser.add_argument('--fetch-gds', help="git fetch latest gds", action="store_const", const=True)
parser.add_argument('--single', help="do action on single project", type=int, default=-1) parser.add_argument('--single', help="do action on single project", type=int, default=-1)
parser.add_argument('--start-from', help="do action on projects after this index", type=int, default=0)
parser.add_argument('--end-at', help="do action on projects before this index", type=int, default=0)
parser.add_argument('--update-caravel', help='configure caravel for build', action='store_const', const=True) parser.add_argument('--update-caravel', help='configure caravel for build', action='store_const', const=True)
parser.add_argument('--harden', help="harden project", action="store_const", const=True) parser.add_argument('--harden', help="harden project", action="store_const", const=True)
parser.add_argument('--limit-num-projects', help='only configure for the first n projects', type=int, default=int(config['num_projects']))
parser.add_argument('--test', help='use test projects', action='store_const', const=True) parser.add_argument('--test', help='use test projects', action='store_const', const=True)
parser.add_argument('--sta-projects', help='use sta projects', action='store_const', const=True)
parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO)
parser.add_argument('--log-email', help="print persons email in messages", action="store_const", const=True) parser.add_argument('--log-email', help="print persons email in messages", action="store_const", const=True)
parser.add_argument('--update-image', help="update the image", action="store_const", const=True) parser.add_argument('--update-image', help="update the image", action="store_const", const=True)
@@ -246,8 +175,6 @@ if __name__ == '__main__':
parser.add_argument('--dump-pdf', help="create pdf from the markdown") parser.add_argument('--dump-pdf', help="create pdf from the markdown")
parser.add_argument('--build-hugo-content', help="directory to where to build hugo content") parser.add_argument('--build-hugo-content', help="directory to where to build hugo content")
parser.add_argument('--metrics', help="print some project metrics", action="store_const", const=True) parser.add_argument('--metrics', help="print some project metrics", action="store_const", const=True)
parser.add_argument('--add-extra-project', help="specify a github url to add. Project is added to the extra_projects list specifed in config.yaml")
parser.add_argument('--sram-support', help="include sram LEF/GDS in caravel config", action="store_const", const=True)
args = parser.parse_args() args = parser.parse_args()
@@ -264,20 +191,15 @@ if __name__ == '__main__':
ch.setFormatter(log_format) ch.setFormatter(log_format)
log.addHandler(ch) log.addHandler(ch)
if args.update_orders:
# don't instantiate orders unless we have to as it needs stripe token
orders = Orders(config)
orders.fetch_orders()
orders.update_project_list()
if args.add_extra_project:
orders = Orders(config)
orders.add_extra_project(args.add_extra_project)
projects = Projects(config, args) projects = Projects(config, args)
if args.sta_projects:
modules_yaml_name = "modules_sta.yaml"
else:
modules_yaml_name = "modules.yaml"
docs = Docs(projects.projects, args=args) docs = Docs(projects.projects, args=args)
caravel = CaravelConfig(config, projects.projects, num_projects=args.limit_num_projects) caravel = CaravelConfig(config, projects.projects, modules_yaml_name)
rom = ROMFile(config, projects.projects) rom = ROMFile(config, projects.projects)
if args.list: if args.list:
@@ -287,10 +209,7 @@ if __name__ == '__main__':
projects.build_metrics() projects.build_metrics()
if args.update_caravel: if args.update_caravel:
extra_macros = [] caravel.instantiate()
if args.sram_support:
extra_macros = ["sky130_sram_2kbyte_1rw1r_32x512_8"]
caravel.instantiate(extra_macros)
rom.write_rom() rom.write_rom()
if not args.test: if not args.test:
docs.build_index() docs.build_index()
-4
View File
@@ -22,7 +22,6 @@ class Docs():
fh.write("| Index | Author | Title | Type | Git Repo |\n") fh.write("| Index | Author | Title | Type | Git Repo |\n")
fh.write("| ----- | ------ | ------| -----| ---------|\n") fh.write("| ----- | ------ | ------| -----| ---------|\n")
for project in self.projects: for project in self.projects:
if not project.is_fill():
fh.write(project.get_index_row()) fh.write(project.get_index_row())
def update_image(self): def update_image(self):
@@ -67,8 +66,6 @@ class Docs():
fh.write(doc_header) fh.write(doc_header)
for project in self.projects: for project in self.projects:
if project.is_fill():
continue
yaml_data = project.get_project_doc_yaml() yaml_data = project.get_project_doc_yaml()
yaml_data['index'] = project.index yaml_data['index'] = project.index
@@ -134,7 +131,6 @@ class Docs():
fh.write("| ----- | ----- | -------|\n") fh.write("| ----- | ----- | -------|\n")
for project in self.projects: for project in self.projects:
logging.info(project) logging.info(project)
if not project.is_fill():
fh.write(project.get_hugo_row()) fh.write(project.get_hugo_row())
project_dir = os.path.join(hugo_root, f'{project.get_index() :03}') project_dir = os.path.join(hugo_root, f'{project.get_index() :03}')
-121
View File
@@ -3,21 +3,7 @@ from urllib.parse import urlparse
import logging import logging
import requests import requests
import os import os
import zipfile
import io
import errno import errno
import sys
def unique(duplist):
unique_list = []
# traverse for all elements
for x in duplist:
# check if exists in unique_list or not
if x not in unique_list:
unique_list.append(x)
return unique_list
def fetch_file(url, filename): def fetch_file(url, filename):
logging.info("trying to download {}".format(url)) logging.info("trying to download {}".format(url))
@@ -64,50 +50,6 @@ def headers_try_to_add_authorization_from_environment(headers: dict) -> bool:
print("WARNING: No github token found from environment, trying public API requests without, see docs/INFO.md#instructions-to-build-gds", file=sys.stderr) print("WARNING: No github token found from environment, trying public API requests without, see docs/INFO.md#instructions-to-build-gds", file=sys.stderr)
return False return False
def fetch_file_from_git(git_url, path):
# get the basics
user_name, repo = split_git_url(git_url)
headers = {
"Accept" : "application/vnd.github+json",
}
# authenticate for rate limiting
headers_try_to_add_authorization_from_environment(headers)
api_url = 'https://api.github.com/repos/%s/%s/contents/%s' % (user_name, repo, path)
logging.debug(api_url)
r = requests.get(api_url, headers=headers)
check_status(r)
requests_remaining = int(r.headers['X-RateLimit-Remaining'])
if requests_remaining == 0:
logging.error("no API requests remaining")
exit(1)
logging.debug("API requests remaining %d" % requests_remaining)
data = r.json()
if 'content' not in data:
return None
file_content = data['content']
file_content_encoding = data.get('encoding')
if file_content_encoding == 'base64':
file_content = base64.b64decode(file_content)
return file_content
# the latest artifact isn't necessarily the one related to the latest commit, as github
# could have taken longer to process an older commit than a newer one.
# so iterate through commits and return the artifact that matches
def get_most_recent_action_url(commits, artifacts):
release_sha_to_download_url = {artifact['workflow_run']['head_sha']: artifact['archive_download_url'] for artifact in artifacts}
for commit in commits:
if commit['sha'] in release_sha_to_download_url:
return release_sha_to_download_url[commit['sha']]
def get_most_recent_action_page(commits, runs): def get_most_recent_action_page(commits, runs):
release_sha_to_page_url = {run['head_sha']: run['html_url'] for run in runs if run['name'] == 'gds'} release_sha_to_page_url = {run['head_sha']: run['html_url'] for run in runs if run['name'] == 'gds'}
@@ -127,69 +69,6 @@ def split_git_url(url):
return user_name, repo return user_name, repo
# download the artifact for each project to get the gds & lef
def install_artifacts(url, directory):
logging.debug(url)
user_name, repo = split_git_url(url)
headers = {
"Accept" : "application/vnd.github+json",
}
# authenticate for rate limiting
headers_try_to_add_authorization_from_environment(headers)
# first fetch the git commit history
api_url = f'https://api.github.com/repos/{user_name}/{repo}/commits'
r = requests.get(api_url, headers=headers)
check_status(r)
requests_remaining = int(r.headers['X-RateLimit-Remaining'])
if requests_remaining == 0:
logging.error("no API requests remaining")
exit(1)
commits = r.json()
# now get the artifacts
api_url = f'https://api.github.com/repos/{user_name}/{repo}/actions/artifacts'
r = requests.get(api_url, headers=headers, params={'per_page': 100})
check_status(r)
data = r.json()
# check there are some artifacts
if 'artifacts' not in data:
logging.error(f"no artifact found for {url}")
exit(1)
else:
# only get artifacts called GDS
artifacts = [a for a in data['artifacts'] if a['name'] == 'GDS']
logging.debug(f"found {len(artifacts)} artifacts")
if len(artifacts) == 0:
logging.error("no artifacts for this project")
exit(1)
download_url = get_most_recent_action_url(commits, artifacts)
# need actions access on the token to get the artifact
# won't work on a pull request because they won't have the token
attempts = 1
max_attempts = 3
while attempts < max_attempts:
try:
logging.debug(f"download url {download_url} attempt {attempts}")
r = requests.get(download_url, headers=headers)
check_status(r)
z = zipfile.ZipFile(io.BytesIO(r.content))
z.extractall(directory)
break
except zipfile.BadZipFile:
attempts += 1
logging.warning(f"problem with zipfile, retry {attempts}")
if attempts == max_attempts:
logging.error("gave up downloading zipfile")
exit(1)
def get_latest_action_url(url, directory): def get_latest_action_url(url, directory):
logging.debug(url) logging.debug(url)
-137
View File
@@ -1,137 +0,0 @@
import stripe, os
import time
from datetime import datetime
import yaml
import requests
import pytz
import logging
class Orders():
def __init__(self, config):
self.config = config
self.open_date = datetime.strptime(self.config['start_date'], '%Y-%m-%d')
stripe.api_key = os.environ['STRIPE_TOKEN']
self.orders = []
def fetch_orders(self):
orders = self.fetch_stripe_orders() + self.fetch_university_orders() + self.fetch_extra_orders()
logging.info(f"total = {len(orders)}")
self.orders = sorted(orders, key=lambda d: d['time'])
def add_extra_project(self, git_url):
try:
with open(self.config['extra_projects']) as fh:
config = yaml.safe_load(fh)
except FileNotFoundError:
config = []
config.append({'project': {'url' : git_url, 'timestamp' : time.time()}})
with open(self.config['extra_projects'], 'w') as fh:
yaml.safe_dump(config, fh)
logging.info(f"added {git_url} to {self.config['extra_projects']}")
def fetch_extra_orders(self):
with open(self.config['extra_projects']) as fh:
config = yaml.safe_load(fh)
orders = []
for project in config:
orders.append({
'git_url' : project['project']['url'],
'email' : None,
'time' : datetime.fromtimestamp(project['project']['timestamp']).replace(tzinfo=pytz.UTC)
})
logging.info(f"fetched extra {len(orders)} orders")
return orders
def fetch_university_orders(self):
name = self.config['name']
url = f'https://app.tinytapeout.com/api/submissions?shuttle={name}'
r = requests.get(url)
if r.status_code != 200:
logging.warning("couldn't download {}".format(url))
exit(1)
orders = []
for item in r.json()['items']:
order = {
'git_url' : item['repo'],
'email' : None,
'time' : datetime.strptime(item['time'], '%Y-%m-%dT%H:%M:%S.%f%z').replace(tzinfo=pytz.UTC)
}
if order['time'] > self.open_date.replace(tzinfo=pytz.UTC):
orders.append(order)
logging.info(f"fetched university {len(orders)} orders")
return orders
def fetch_stripe_orders(self):
start_id = None
after_open_date = True
orders = []
refunded_orders = []
while after_open_date:
refunds = stripe.Refund.list(limit=10, starting_after=start_id)
for refund in refunds:
created = datetime.fromtimestamp(refund['created'])
if created < self.open_date:
after_open_date = False
refunded_orders.append(refund['payment_intent'])
start_id = refund['id']
start_id = None
after_open_date = True
while after_open_date:
checkouts = stripe.checkout.Session.list(limit=10, starting_after=start_id)
for checkout in checkouts:
created = datetime.fromtimestamp(checkout['created'])
if created < self.open_date:
after_open_date = False
if checkout['payment_status'] == 'paid':
if 'github' in checkout['metadata']:
git_url = checkout['metadata']['github']
if checkout['payment_intent'] in refunded_orders:
logging.warning(f"found a refunded order! {git_url}")
continue
order = {
'git_url' : git_url,
'email' : checkout['customer_details']['email'],
'time' : datetime.fromtimestamp(checkout['created']).replace(tzinfo=pytz.UTC),
}
orders.append(order)
# pagination
start_id = checkout['id']
# put in date order
logging.info(f"fetched stripe {len(orders)} orders")
return orders
def update_project_list(self):
project_list = {}
# first project is the filler test project
index = 0
project = {'url': self.config['filler_project'], 'status': 'active', 'fill': False}
project_list[index] = project
# add all the real orders
for index, order in enumerate(self.orders):
project = {'url': order['git_url'], 'status': 'active', 'fill': False}
project_list[index + 1] = project
# fill with the filler
for index in range(len(project_list), self.config['num_projects']):
project = {'url': self.config['filler_project'], 'status': 'active', 'fill': True}
project_list[index] = project
with open('projects.yaml', 'w') as fh:
fh.write(yaml.dump(project_list, sort_keys=False))
+13 -56
View File
@@ -21,11 +21,10 @@ with open(os.path.join(os.path.dirname(__file__), 'tile_sizes.yaml'), 'r') as st
class Project(): class Project():
def __init__(self, index, git_url, local_dir, args, fill=False): def __init__(self, index, git_url, local_dir, args):
self.git_url = git_url self.git_url = git_url
self.args = args self.args = args
self.index = index self.index = index
self.fill = fill
self.local_dir = local_dir self.local_dir = local_dir
def post_clone_setup(self): def post_clone_setup(self):
@@ -42,7 +41,7 @@ class Project():
def check_ports(self): def check_ports(self):
top = self.get_macro_name() top = self.get_macro_name()
sources = [os.path.join(self.local_dir, 'src', src) for src in self.src_files] sources = [os.path.join(self.local_dir, src) for src in self.src_files]
source_list = " ".join(sources) source_list = " ".join(sources)
json_file = 'ports.json' json_file = 'ports.json'
@@ -85,9 +84,6 @@ class Project():
if num_cells < 11: if num_cells < 11:
logging.warning(f"{self} only has {num_cells} cells") logging.warning(f"{self} only has {num_cells} cells")
def is_fill(self):
return self.fill
def is_wokwi(self): def is_wokwi(self):
if self.wokwi_id != 0: if self.wokwi_id != 0:
return True return True
@@ -117,18 +113,21 @@ class Project():
if self.is_hdl(): if self.is_hdl():
self.top_module = self.yaml['project']['top_module'] self.top_module = self.yaml['project']['top_module']
self.src_files = self.get_hdl_source() self.src_files = [self.get_gl_verilog_filename()]
self.top_verilog_filename = self.find_top_verilog() #self.src_files = self.get_hdl_source()
#self.top_verilog_filename = self.find_top_verilog()
else: else:
self.top_module = f"tt_um_wokwi_{self.wokwi_id}" self.top_module = f"tt_um_wokwi_{self.wokwi_id}"
self.src_files = self.get_wokwi_source() #self.src_files = self.get_wokwi_source()
self.top_verilog_filename = self.src_files[0] #self.top_verilog_filename = self.src_files[0]
self.unprefixed_name = re.sub('^tt_um_', '', self.top_module)
if not self.top_module.startswith("tt_um_"): if not self.top_module.startswith("tt_um_"):
logging.error(f'top module name must start with "tt_um_" (current value: "{self.top_module}")') logging.error(f'top module name must start with "tt_um_" (current value: "{self.top_module}")')
exit(1) exit(1)
self.macro_instance = f"{self.top_module}_{self.index :03}" self.macro_instance = f"{self.top_module}"
def get_wokwi_source(self): def get_wokwi_source(self):
src_file = "tt_um_wokwi_{}.v".format(self.wokwi_id) src_file = "tt_um_wokwi_{}.v".format(self.wokwi_id)
@@ -197,7 +196,7 @@ class Project():
rgx_mod = re.compile(r"(?:^|[\s])module[\s]{1,}([\w]+)") rgx_mod = re.compile(r"(?:^|[\s])module[\s]{1,}([\w]+)")
top_verilog = [] top_verilog = []
for src in self.src_files: for src in self.src_files:
with open(os.path.join(self.local_dir, 'src', src)) as fh: with open(os.path.join(self.local_dir, src)) as fh:
for line in fh.readlines(): for line in fh.readlines():
for match in rgx_mod.finditer(line): for match in rgx_mod.finditer(line):
if match.group(1) == self.top_module: if match.group(1) == self.top_module:
@@ -227,26 +226,6 @@ class Project():
if GITHUB_SERVER_URL and GITHUB_REPOSITORY and GITHUB_RUN_ID: if GITHUB_SERVER_URL and GITHUB_REPOSITORY and GITHUB_RUN_ID:
return f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" return f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}"
def clone(self):
if os.path.exists(self.local_dir):
git_remote = self.get_git_remote()
if self.git_url == git_remote:
logging.info("git repo already exists and is correct - skipping")
else:
logging.error("git repo exists and remote doesn't match - abort")
logging.error(f"{self.git_url} != {git_remote}")
exit(1)
else:
logging.debug("clone")
git.Repo.clone_from(self.git_url, self.local_dir, recursive=True)
def pull(self):
repo = git.Repo(self.local_dir)
# reset
repo.git.reset('--hard')
o = repo.remotes.origin
o.pull()
def __str__(self): def __str__(self):
""" """
if self.args.log_email: if self.args.log_email:
@@ -255,9 +234,6 @@ class Project():
""" """
return f"[{self.index:03} : {self.git_url}]" return f"[{self.index:03} : {self.git_url}]"
def fetch_gds(self):
git_utils.install_artifacts(self.git_url, self.local_dir)
def get_latest_action_url(self): def get_latest_action_url(self):
return git_utils.get_latest_action_url(self.git_url, self.local_dir) return git_utils.get_latest_action_url(self.git_url, self.local_dir)
@@ -274,7 +250,7 @@ class Project():
# metrics # metrics
def get_metrics_path(self): def get_metrics_path(self):
return os.path.join(self.local_dir, 'runs/wokwi/reports/metrics.csv') return os.path.join(self.local_dir, 'metrics.csv')
# name of the gds file # name of the gds file
def get_macro_gds_filename(self): def get_macro_gds_filename(self):
@@ -307,25 +283,6 @@ class Project():
def get_git_url(self): def get_git_url(self):
return self.git_url return self.git_url
def copy_files_to_caravel(self):
files = [
(f"runs/wokwi/results/final/gds/{self.get_macro_gds_filename()}", f"gds/{self.get_macro_gds_filename()}"),
(f"runs/wokwi/results/final/commit_id.json", f"gds/{self.get_macro_info_filename()}"),
(f"runs/wokwi/results/final/spef/{self.get_macro_spef_filename()}", f"spef/{self.get_macro_spef_filename()}"),
(f"runs/wokwi/results/final/lef/{self.get_macro_lef_filename()}", f"lef/{self.get_macro_lef_filename()}"),
(f"runs/wokwi/results/final/verilog/gl/{self.get_gl_verilog_filename()}", f"verilog/gl/{self.get_gl_verilog_filename()}"),
(f"src/{self.top_verilog_filename}", f"verilog/rtl/{self.get_top_verilog_filename()}"),
]
# copy RTL verilog to openlane2 mux directory:
if os.path.isdir("tt-multiplexer/ol2/tt_top/verilog"):
files.append((f"runs/wokwi/results/final/verilog/gl/{self.get_gl_verilog_filename()}", f"tt-multiplexer/ol2/tt_top/verilog/{self.get_gl_verilog_filename()}"))
logging.debug("copying files into position")
for from_path, to_path in files:
from_path = os.path.join(self.local_dir, from_path)
logging.debug(f"copy {from_path} to {to_path}")
shutil.copyfile(from_path, to_path)
def print_wokwi_id(self): def print_wokwi_id(self):
print(self.wokwi_id) print(self.wokwi_id)
@@ -568,7 +525,7 @@ class Project():
def get_cell_count_from_synth(self): def get_cell_count_from_synth(self):
num_cells = 0 num_cells = 0
try: try:
yosys_report = glob.glob(f'{self.local_dir}/runs/wokwi/reports/synthesis/1-synthesis.*0.stat.rpt')[0] # can't open a file with \ in the path yosys_report = f'{self.local_dir}/synthesis-stats.txt'
with open(yosys_report) as fh: with open(yosys_report) as fh:
for line in fh.readlines(): for line in fh.readlines():
m = re.search(r'Number of cells:\s+(\d+)', line) m = re.search(r'Number of cells:\s+(\d+)', line)