feat(tt_tool): scramble stdcell layers in png render for IHP & GF (#129)

* feat(tt_tool): scramble stdcell layers in png render for IHP & GF

* fix(tt_tool): resolve mypy errors

* refactor(tt_tool): separate svg & png rendering into render_utils.py

project.py

@@ -9,9 +9,7 @@ import shutil
 import subprocess
 import typing
 
-import cairosvg  # type: ignore
 import chevron
-import gdstk  # type: ignore
 import klayout.db as pya
 import yaml
 from git.repo import Repo
@@ -21,6 +19,7 @@ from config_utils import read_config, write_config
 from doc_utils import DocsHelper
 from markdown_utils import limit_markdown_headings
 from project_info import ProjectInfo, ProjectYamlError
+from render_utils import render_png, render_svg
 from tech import TechName, tech_map
 
 PINOUT_KEYS = [
@@ -300,7 +299,7 @@ class Project:
         repo = Repo(os.path.join(self.local_dir, "tt"))
         if repo.head.is_detached:
             ref = next(
-                (ref.name for ref in repo.refs if ref.commit == repo.head.commit),
+                (ref.name for ref in repo.refs if ref.commit == repo.head.commit),  # type: ignore[attr-defined] # see mypy issue #8085
                 "(detached)",
             )
         else:
@@ -742,122 +741,29 @@ class Project:
 
         DocsHelper.compile("./docs/doc.typ")
 
-    # Read and return top-level GDS data from the final GDS file, using gdstk:
+    # Return final GDS used for create_svg and create_png
-    def get_final_gds_top_cells(self):
+    def get_final_gds(self):
         if "GDS_PATH" in os.environ:
-            gds_path = os.environ["GDS_PATH"]
+            return os.environ["GDS_PATH"]
         else:
-            gds_path = glob.glob(
+            return glob.glob(
                 os.path.join(self.local_dir, "runs/wokwi/final/gds/*.gds")
             )[0]
-        library = gdstk.read_gds(gds_path)
-        top_cells = library.top_level()
-        return top_cells[0]
 
-    # SVG render of the GDS.
+    # SVG render of the GDS
-    # NOTE: This includes all standard GDS layers inc. text labels.
     def create_svg(self):
-        top_cells = self.get_final_gds_top_cells()
+        render_svg(self.get_final_gds(), scramble_cells=self.tech.scramble_cells)
-        top_cells.write_svg("gds_render.svg")
 
-    # Try various QUICK methods to create a more-compressed PNG render of the GDS,
-    # and fall back to cairosvg if it doesn't work. This is designed for speed,
-    # and in particular for use by the GitHub Actions.
-    # For more info, see:
-    # https://github.com/TinyTapeout/tt-gds-action/issues/8
     def create_png(self):
-        logging.info("Loading GDS data...")
-        top_cells = self.get_final_gds_top_cells()
-
-        # Remove label layers (i.e. delete text), then generate SVG:
-        label_layers = self.tech.label_layers
-        top_cells.filter(label_layers)
-        for subcell in top_cells.dependencies(True):
-            subcell.filter(label_layers)
-        svg = "gds_render_preview.svg"
-        logging.info("Rendering SVG without text label layers: {}".format(svg))
-        top_cells.write_svg(svg, pad=0)
-        # We should now have gds_render_preview.svg
-
-        # Try converting using 'rsvg-convert' command-line utility.
-        # This should create gds_render_preview.png
-        png = "gds_render_preview.png"
-        logging.info("Converting to PNG using rsvg-convert: {}".format(png))
-
-        cmd = "rsvg-convert --unlimited {} -o {} --no-keep-image-data".format(svg, png)
-        logging.debug(cmd)
-        p = subprocess.run(cmd, shell=True, capture_output=True)
-
-        if p.returncode == 127:
-            logging.warning(
-                'rsvg-convert not found; is package "librsvg2-bin" installed? Falling back to cairosvg. This might take a while...'
-            )
-            # Fall back to cairosvg:
-            cairosvg.svg2png(url=svg, write_to=png)
-
-        elif p.returncode != 0 and b"cannot load more than" in p.stderr:
-            logging.warning(
-                'Too many SVG elements ("{}"). Reducing complexity...'.format(
-                    p.stderr.decode().strip()
-                )
-            )
-
-            # Remove more layers that are hardly visible anyway:
-            buried_layers = self.tech.buried_layers
-            top_cells.filter(buried_layers)
-            for subcell in top_cells.dependencies(True):
-                subcell.filter(buried_layers)
-            svg_alt = "gds_render_preview_alt.svg"
-            logging.info("Rendering SVG with more layers removed: {}".format(svg_alt))
-            top_cells.write_svg(svg_alt, pad=0)
-            logging.info("Converting to PNG using rsvg-convert: {}".format(png))
-
-            cmd = "rsvg-convert --unlimited {} -o {} --no-keep-image-data".format(
-                svg_alt, png
-            )
-            logging.debug(cmd)
-            p = subprocess.run(cmd, shell=True, capture_output=True)
-
-            if p.returncode != 0:
-                logging.warning(
-                    'Still cannot convert to SVG ("{}"). Falling back to cairosvg. This might take a while...'.format(
-                        p.stderr.decode().strip()
-                    )
-                )
-
-                # Fall back to cairosvg, and since we're doing that, might as well use the original full-detail SVG anyway:
-                cairosvg.svg2png(url=svg, write_to=png)
-
-        # By now we should have gds_render_preview.png
-
-        # Compress with pngquant:
-        final_png = "gds_render.png"
         if self.info.tiles == "8x2":
             quality = "0-10"  # Compress more for 8x2 tiles.
         else:
             quality = "0-30"
-        logging.info("Compressing PNG further with pngquant to: {}".format(final_png))
+        render_png(
-
+            self.get_final_gds(),
-        cmd = "pngquant --quality {} --speed 1 --nofs --strip --force --output {} {}".format(
+            scramble_cells=self.tech.scramble_cells,
-            quality, final_png, png
+            buried_layers=self.tech.buried_layers,
-        )
+            quality=quality,
-        logging.debug(cmd)
-        p = subprocess.run(cmd, shell=True, capture_output=True)
-
-        if p.returncode == 127:
-            logging.warning(
-                'pngquant not found; is package "pngquant" installed? Using intermediate (uncompressed) PNG file'
-            )
-            os.rename(png, final_png)
-        elif p.returncode != 0:
-            logging.warning(
-                'pngquant error {} ("{}"). Using intermediate (uncompressed) PNG file'.format(
-                    p.returncode, p.stderr.decode().strip()
-                )
-            )
-            os.rename(png, final_png)
-        logging.info(
-            "Final PNG is {} ({:,} bytes)".format(final_png, os.path.getsize(final_png))
         )
 
     def print_warnings(self):

render_utils.py

@@ -0,0 +1,147 @@
+import logging
+import os
+import random
+import re
+import subprocess
+
+import cairosvg  # type: ignore
+import gdstk  # type: ignore
+
+
+# Custom exception for convert_svg_to_png()
+class SVGTooComplexError(Exception):
+    pass
+
+
+# Helper function to randomize layer order of polygons in a subcell in a deterministic way
+def scramble_polygons(polygons, seed):
+    # group polygons by layer/datatype
+    groups: dict[tuple[int, int], list] = {}
+    for p in polygons:
+        groups.setdefault((p.layer, p.datatype), []).append(p)
+    # shuffle the layer list
+    layers = sorted(groups)
+    random.Random(seed).shuffle(layers)
+    # return polygons according to the shuffled layer order
+    return [p for layer in layers for p in groups[layer]]
+
+
+# SVG render of the GDS
+def render_svg(
+    gds,
+    svg="gds_render.svg",
+    pad="5%",
+    filter_text=False,
+    filter_layers=None,
+    scramble_cells=None,
+):
+    library = gdstk.read_gds(gds)
+    top_cells = library.top_level()
+    assert len(top_cells) == 1
+    top_cell = top_cells[0]
+    assert isinstance(top_cell, gdstk.Cell)
+    cells = [top_cell] + [
+        cell for cell in top_cell.dependencies(True) if isinstance(cell, gdstk.Cell)
+    ]
+    for cell in cells:
+        if filter_text:
+            cell.remove(*cell.labels)
+        if filter_layers:
+            cell.filter(filter_layers)
+        if scramble_cells is not None:
+            if re.match(scramble_cells, cell.name):
+                polygons = cell.polygons
+                paths = cell.paths
+                cell.remove(*polygons, *paths)
+                path_polygons = [pp for p in paths for pp in p.to_polygons()]
+                cell.add(*scramble_polygons(polygons, cell.name))
+                cell.add(*scramble_polygons(path_polygons, cell.name))
+    top_cell.write_svg(svg, pad=pad)
+
+
+# Convert SVG to PNG using rsvg-convert or cairosvg
+def convert_svg_to_png(svg, png, force_cairo=False):
+    if force_cairo:
+        logging.warning("Falling back to cairosvg. This might take a while...")
+        cairosvg.svg2png(url=svg, write_to=png)
+    else:
+        cmd = f"rsvg-convert --unlimited {svg} -o {png} --no-keep-image-data"
+        logging.debug(cmd)
+        p = subprocess.run(cmd, shell=True, capture_output=True)
+        if p.returncode == 127:
+            logging.warning(
+                'rsvg-convert not found; is package "librsvg2-bin" installed?'
+            )
+            # fall back to cairosvg
+            convert_svg_to_png(svg, png, force_cairo=True)
+        elif p.returncode != 0:
+            if b"cannot load more than" in p.stderr:
+                logging.warning(
+                    f'Too many SVG elements ("{p.stderr.decode().strip()}")'
+                )
+                raise SVGTooComplexError()
+            else:
+                logging.warning(
+                    f'rsvg-convert returned an error ("{p.stderr.decode().strip()}")'
+                )
+                # fall back to cairosvg
+                convert_svg_to_png(svg, png, force_cairo=True)
+
+
+# Try various QUICK methods to create a more-compressed PNG render of the GDS,
+# and fall back to cairosvg if it doesn't work. This is designed for speed,
+# and in particular for use by the GitHub Actions.
+# For more info, see:
+# https://github.com/TinyTapeout/tt-gds-action/issues/8
+def render_png(
+    gds,
+    svg="gds_render_preview.svg",
+    svg_alt="gds_render_preview_alt.svg",
+    png="gds_render_preview.png",
+    final_png="gds_render.png",
+    scramble_cells=None,
+    buried_layers=None,
+    quality="0-30",
+):
+    logging.info(f"Rendering SVG without text labels: {svg}")
+    render_svg(gds=gds, svg=svg, pad=0, filter_text=True, scramble_cells=scramble_cells)
+    try:
+        logging.info(f"Converting to PNG using rsvg-convert: {png}")
+        convert_svg_to_png(svg, png)
+    except SVGTooComplexError:
+        logging.info(f"Rendering SVG without text labels or buried layers: {svg_alt}")
+        render_svg(
+            gds=gds,
+            svg=svg_alt,
+            pad=0,
+            filter_text=True,
+            filter_layers=buried_layers,
+            scramble_cells=scramble_cells,
+        )
+        try:
+            logging.info(f"Converting to PNG using rsvg-convert: {png}")
+            convert_svg_to_png(svg_alt, png)
+        except SVGTooComplexError:
+            # Fall back to cairosvg, and since we're doing that, might as well use the original full-detail SVG anyway:
+            convert_svg_to_png(svg, png, force_cairo=True)
+
+    # By now we should have gds_render_preview.png
+
+    # Compress with pngquant:
+    logging.info(f"Compressing PNG further with pngquant to: {final_png}")
+
+    cmd = f"pngquant --quality {quality} --speed 1 --nofs --strip --force --output {final_png} {png}"
+    logging.debug(cmd)
+    p = subprocess.run(cmd, shell=True, capture_output=True)
+
+    if p.returncode == 127:
+        logging.warning(
+            'pngquant not found; is package "pngquant" installed? Using intermediate (uncompressed) PNG file'
+        )
+        os.rename(png, final_png)
+    elif p.returncode != 0:
+        logging.warning(
+            f'pngquant error {p.returncode} ("{p.stderr.decode().strip()}"). Using intermediate (uncompressed) PNG file'
+        )
+        os.rename(png, final_png)
+    logging.info(f"Final PNG is {final_png} ({os.path.getsize(final_png):,} bytes)")

tech.py

@@ -1,6 +1,6 @@
 import json
 import os
-from typing import Dict, List, Literal, Protocol, Tuple, TypedDict, Union
+from typing import Dict, List, Literal, Pattern, Protocol, Tuple, TypedDict, Union
 
 from git.repo import Repo
 
@@ -33,6 +33,8 @@ class Tech(Protocol):
     label_layers: List[Tuple[int, int]]
     """ These layers are hardly visible and will also be removed from the SVG render of the layout """
     buried_layers: List[Tuple[int, int]]
+    """ Cells with names matching this regex will have their layers shuffled in the SVG render """
+    scramble_cells: None | str | Pattern
     """ Default name of the mux config YAML file. """
     mux_config_yaml_name: str
     """ Lists all of the analog switch / power gate macros that need to be copied to the tt_top directory. """
@@ -81,6 +83,7 @@ class Sky130Tech(Tech):
         (81, 4),  # areaid.standardc
         (70, 20),  # met3.drawing
     ]
+    scramble_cells = None
     mux_config_yaml_name = "sky130.yaml"
     mux_macros = [
         "pg/sky130/tt_pg_1v8_hp_1",
@@ -129,6 +132,7 @@ class IHPTech(Tech):
         (51, 0),  # HeatTrans.drawing
         (189, 4),  # prBoundary.boundary
     ]
+    scramble_cells = "sg13g2_"
     mux_config_yaml_name = "ihp-sg13g2.yaml"
     mux_macros = [
         "pg/ihp-sg13g2/tt_pg_1v5_hp_1",
@@ -204,6 +208,7 @@ class GF180MCUDTech(Tech):
     buried_layers = [
         # TODO: add buried layers
     ]
+    scramble_cells = "gf180mcu_fd_sc_"
     mux_config_yaml_name = "gf180mcuD.yaml"
     mux_macros = [
         "pg/gf180mcuD/tt_pg_1v8_hp_1",