jlcparts-based parts library (#374)

A set of parts libraries based on data available from
https://github.com/yaqwsx/jlcparts

JLC no longer (well, hasn't for a few years now) makes their entire
parts library available by CSV, and the current parts table is starting
to get outdated with parts no longer available or basic parts. jlcparts
seems to be able to get an up-to-date library as well as do some parsing
of parametric parts.

Since the data here isn't deterministic, this is strictly opt-in; all
test boards will continue to be based on the included CSV. By nature
this won't be as well-tested.

To use these parts:
1. clone jlcparts, download the cached parts list and build its tables
(according to its readme)
2. in the top-level board design file, add
`JlcPartsBase.config_root_dir("/path/to/jlcparts-repo/web/public/data/")`
3. in the top-level design, add JlcPartsRefinements to the superclass
list before (takes priority over) other refinements

so if this was done on the SMU example, it would look like:
```py
import ...
JlcPartsBase.config_root_dir(".../jlcparts/web/public/data/")
...
class UsbSourceMeasure(JlcPartsRefinements, JlcBoardTop):
  def contents(self) -> None:
    ...
```

Other changes
- Add green-yellow to TableLed, different than a more intense green I
guess
- Add TableLed
- Internal semantics: these are now exact even when no tolerance is
specified: FerriteBead.DC_RESISTANCE, Fet.RDS_ON,
Inductor.DC_RESISTANCE,
- Pin dependency versions in requirements.txt

.github/workflows/pr-python.yml

@@ -43,7 +43,8 @@ jobs:
 
     - name: install mypy
       run: |
-        pip install mypy mypy-protobuf types-protobuf types-Deprecated
+        pip install -r requirements.txt
+        pip install mypy mypy-protobuf
         mypy --version
     - name: mypy
       run: mypy --install-types .
@@ -94,7 +95,8 @@ jobs:
 
       - name: install mypy
         run: |
-          pip install mypy mypy-protobuf types-protobuf types-Deprecated
+          pip install -r requirements.txt
+          pip install mypy mypy-protobuf
           mypy --version
       - name: mypy
         run: mypy --install-types .

edg/__init__.py

@@ -4,6 +4,7 @@
 from .electronics_model import *
 from .abstract_parts import *
 from .parts import *
+from .jlcparts import *
 
 from .BoardTop import BoardTop, SimpleBoardTop, JlcBoardTop
 

edg/abstract_parts/AbstractLed.py

@@ -1,6 +1,8 @@
 from ..electronics_model import *
 from .Categories import *
 from .AbstractResistor import Resistor
+from .PartsTable import PartsTableColumn, PartsTableRow
+from .PartsTablePart import PartsTableFootprintSelector
 from .StandardFootprint import StandardFootprint
 
 LedColor = str  # type alias
@@ -12,6 +14,7 @@ class Led(DiscreteSemiconductor):
   # Common color definitions
   Red: LedColor = "red"
   Green: LedColor = "green"
+  GreenYellow: LedColor = "greenyellow"  # more a mellow green
   Blue: LedColor = "blue"
   Yellow: LedColor = "yellow"
   White: LedColor = "white"
@@ -60,6 +63,24 @@ class LedStandardFootprint(Led, StandardFootprint[Led]):
   }
 
 
+@non_library
+class TableLed(LedStandardFootprint, PartsTableFootprintSelector):
+  COLOR = PartsTableColumn(str)
+
+  @init_in_parent
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.generator_param(self.color)
+
+  def _row_filter(self, row: PartsTableRow) -> bool:
+    return super()._row_filter(row) and \
+      (not self.get(self.color) or row[self.COLOR] == self.get(self.color))
+
+  def _row_generate(self, row: PartsTableRow) -> None:
+    super()._row_generate(row)
+    self.assign(self.actual_color, row[self.COLOR])
+
+
 @abstract_block
 class RgbLedCommonAnode(DiscreteSemiconductor):
   def __init__(self):

edg/abstract_parts/__init__.py

@@ -44,7 +44,7 @@
 from .AbstractDiodes import BaseDiode, Diode, BaseDiodeStandardFootprint, TableDiode
 from .AbstractDiodes import ZenerDiode, TableZenerDiode, ProtectionZenerDiode, AnalogClampZenerDiode
 from .AbstractTvsDiode import TvsDiode, ProtectionTvsDiode, DigitalTvsDiode
-from .AbstractLed import Led, LedStandardFootprint, RgbLedCommonAnode, LedColor, LedColorLike
+from .AbstractLed import Led, LedStandardFootprint, TableLed, RgbLedCommonAnode, LedColor, LedColorLike
 from .AbstractLed import IndicatorLed, IndicatorSinkLed, IndicatorSinkLedResistor, VoltageIndicatorLed, IndicatorSinkRgbLed
 from .AbstractLed import IndicatorSinkPackedRgbLed
 from .AbstractLed import IndicatorLedArray, IndicatorSinkLedArray

edg/electronics_model/PartParserUtil.py

@@ -90,7 +90,7 @@ def parse_abs_tolerance(cls, value: str, center: float, units: str) -> Range:
     elif value.endswith('ppm'):
       value = value.removesuffix('ppm').rstrip()
       return Range.from_tolerance(center, float(value) * 1e-6)
-    elif value.endswith(units):
+    elif units and value.endswith(units):
       return Range.from_abs_tolerance(center, cls.parse_value(value, units))
 
     raise cls.ParseError(f"Unknown tolerance '{value}'")

edg/jlcparts/JlcPartsBase.py

@@ -0,0 +1,170 @@
+import sys
+from typing import Any, Optional, Dict, List, TypeVar, Type
+
+from pydantic import BaseModel, RootModel, Field
+import gzip
+import os
+
+from ..abstract_parts import *
+from ..parts.JlcPart import JlcPart
+
+kTableFilenamePostfix = ".json.gz"
+kStockFilenamePostfix = ".stock.json"
+
+
+class JlcPartsFile(BaseModel):
+    category: str
+    components: list[list[Any]]  # index-matched with schema
+    jlcpart_schema: list[str] = Field(..., alias='schema')
+
+
+class JlcPartsAttributeEntry(BaseModel):
+    # format: Optional[str] = None  # unused, no idea why this exists
+    # primary: Optional[str] = None  # unused, no idea why this exists
+    values: dict[str, tuple[Any, str]]
+
+
+ParsedType = TypeVar('ParsedType')  # can't be inside the class or it gets confused as a pydantic model entry
+
+class JlcPartsAttributes(RootModel):
+    root: dict[str, JlcPartsAttributeEntry]
+
+    def get(self, key: str, expected_type: Type[ParsedType], default: Optional[ParsedType] = None, sub='default') -> ParsedType:
+        """Utility function that gets an attribute of the specified name, checking that it is the expected type
+        or returning some default (if specified)."""
+        if key not in self.root:
+            if default is not None:
+                return default
+            else:
+                raise KeyError
+        value = self.root[key].values[sub][0]
+        if not isinstance(value, expected_type):
+            if default is not None:
+                return default
+            else:
+                raise TypeError
+        return value
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.root
+
+
+class JlcPartsPriceEntry(BaseModel):
+    price: float
+    qFrom: int
+    qTo: Optional[int]  # None = top bucket
+
+
+class JlcPartsPrice(RootModel):
+    root: list[JlcPartsPriceEntry]
+
+    def for_min_qty(self) -> float:
+        min_seen_price = (sys.maxsize, float(sys.maxsize))  # return ridiculously high if not specified
+
+        for bucket in self.root:
+            if bucket.qFrom <= 1 or bucket.qFrom is None:  # short circuit for qty=1
+                return bucket.price
+            if bucket.qFrom < min_seen_price[0]:
+                min_seen_price = (bucket.qFrom, bucket.price)
+        return min_seen_price[1]
+
+
+class JlcPartsStockFile(RootModel):
+    root: dict[str, int]  # LCSC to stock level
+
+
+class JlcPartsBase(JlcPart, PartsTableSelector, PartsTableFootprint):
+    """Base class parsing parts from https://github.com/yaqwsx/jlcparts"""
+    _config_parts_root_dir: Optional[str] = None
+    _config_min_stock: int = 250
+
+    # overrides from PartsTableBase
+    PART_NUMBER_COL = PartsTableColumn(str)
+    MANUFACTURER_COL = PartsTableColumn(str)
+    DESCRIPTION_COL = PartsTableColumn(str)
+    DATASHEET_COL = PartsTableColumn(str)
+
+    # new columns here
+    LCSC_COL = PartsTableColumn(str)
+    BASIC_PART_COL = PartsTableColumn(bool)
+    COST_COL = PartsTableColumn(str)
+
+    @staticmethod
+    def config_root_dir(root_dir: str):
+        """Configures the root dir that contains the data files from jlcparts, eg
+        CapacitorsMultilayer_Ceramic_Capacitors_MLCC___SMDakaSMT.json.gz
+        This setting is on a JlcPartsBase-wide basis."""
+        assert JlcPartsBase._config_parts_root_dir is None, \
+            f"attempted to reassign configure_root_dir, was {JlcPartsBase._config_parts_root_dir}, new {root_dir}"
+        JlcPartsBase._config_parts_root_dir = root_dir
+
+    _JLC_PARTS_FILE_NAMES: List[str]  # set by subclass
+    _cached_table: Optional[PartsTable] = None  # set on a per-class basis
+
+    @classmethod
+    def _make_table(cls) -> PartsTable:
+        """Return the table, cached if possible"""
+        if cls._cached_table is None:
+            cls._cached_table = cls._parse_table()
+        return cls._cached_table
+
+    @classmethod
+    def _entry_to_table_row(cls, row_dict: Dict[PartsTableColumn, Any], filename: str, package: str, attributes: JlcPartsAttributes)\
+            -> Optional[Dict[PartsTableColumn, Any]]:
+        """Given an entry from jlcparts and row pre-populated with metadata, adds category-specific data to the row
+        (in-place), and returns the row (or None, if it failed to parse and the row should be discarded)."""
+        raise NotImplementedError
+
+    @classmethod
+    def _parse_table(cls) -> PartsTable:
+        """Parses the file to a PartsTable"""
+        assert cls._config_parts_root_dir is not None, "must configure_root_dir with jlcparts data folder"
+
+        rows: List[PartsTableRow] = []
+
+        for filename in cls._JLC_PARTS_FILE_NAMES:
+            with gzip.open(os.path.join(cls._config_parts_root_dir, filename + kTableFilenamePostfix), 'r') as f:
+                data = JlcPartsFile.model_validate_json(f.read())
+            with open(os.path.join(cls._config_parts_root_dir, filename + kStockFilenamePostfix), 'r') as f:
+                stocking = JlcPartsStockFile.model_validate_json(f.read())
+
+            lcsc_index = data.jlcpart_schema.index("lcsc")
+            part_number_index = data.jlcpart_schema.index("mfr")
+            description_index = data.jlcpart_schema.index("description")
+            datasheet_index = data.jlcpart_schema.index("datasheet")
+            attributes_index = data.jlcpart_schema.index("attributes")
+            price_index = data.jlcpart_schema.index("price")
+
+            for component in data.components:
+                row_dict: Dict[PartsTableColumn, Any] = {}
+
+                row_dict[cls.LCSC_COL] = lcsc = component[lcsc_index]
+                if stocking.root.get(lcsc, 0) < cls._config_min_stock:
+                    continue
+
+                row_dict[cls.PART_NUMBER_COL] = component[part_number_index]
+                row_dict[cls.DESCRIPTION_COL] = component[description_index]
+                row_dict[cls.DATASHEET_COL] = component[datasheet_index]
+                row_dict[cls.COST_COL] = JlcPartsPrice(component[price_index]).for_min_qty()
+
+                attributes = JlcPartsAttributes(**component[attributes_index])
+                if attributes.get("Status", str) in ["Discontinued"]:
+                    continue
+                row_dict[cls.BASIC_PART_COL] = attributes.get("Basic/Extended", str) == "Basic"
+                row_dict[cls.MANUFACTURER_COL] = attributes.get("Manufacturer", str)
+
+                package = attributes.get("Package", str)
+                row_dict_opt = cls._entry_to_table_row(row_dict, filename, package, attributes)
+                if row_dict_opt is not None:
+                    rows.append(PartsTableRow(row_dict_opt))
+
+        return PartsTable(rows)
+
+    @classmethod
+    def _row_sort_by(cls, row: PartsTableRow) -> Any:
+        return [row[cls.BASIC_PART_COL], row[cls.KICAD_FOOTPRINT], row[cls.COST_COL]]
+
+    def _row_generate(self, row: PartsTableRow) -> None:
+        super()._row_generate(row)
+        self.assign(self.lcsc_part, row[self.LCSC_COL])
+        self.assign(self.actual_basic_part, row[self.BASIC_PART_COL])

edg/jlcparts/JlcPartsBjt.py

@@ -0,0 +1,32 @@
+from typing import Any, Optional, Dict
+from ..abstract_parts import *
+from ..parts.JlcBjt import JlcBjt
+from .JlcPartsBase import JlcPartsBase, JlcPartsAttributes
+
+
+class JlcPartsBjt(TableBjt, JlcPartsBase):
+    _JLC_PARTS_FILE_NAMES = ["TransistorsBipolar_Transistors___BJT"]
+    _CHANNEL_MAP = {
+        'NPN': 'NPN',
+        'PNP': 'PNP',
+    }
+
+    @classmethod
+    def _entry_to_table_row(cls, row_dict: Dict[PartsTableColumn, Any], filename: str, package: str, attributes: JlcPartsAttributes) \
+            -> Optional[Dict[PartsTableColumn, Any]]:
+        try:
+            row_dict[cls.KICAD_FOOTPRINT] = JlcBjt.PACKAGE_FOOTPRINT_MAP[package]
+
+            row_dict[cls.CHANNEL] = cls._CHANNEL_MAP[attributes.get("Transistor type", str)]
+            row_dict[cls.VCE_RATING] = Range.zero_to_upper(PartParserUtil.parse_value(
+                attributes.get("Collector-emitter breakdown voltage (vceo)", str), 'V'))
+            row_dict[cls.ICE_RATING] = Range.zero_to_upper(PartParserUtil.parse_value(
+                attributes.get("Collector current (ic)", str), 'A'))
+            row_dict[cls.GAIN] = Range.exact(PartParserUtil.parse_value(
+                attributes.get("Dc current gain (hfe@ic,vce)", str).split('@')[0], ''))
+            row_dict[cls.POWER_RATING] = Range.zero_to_upper(
+                attributes.get("Power dissipation (pd)", float, sub='power'))
+
+            return row_dict
+        except (KeyError, TypeError, PartParserUtil.ParseError):
+            return None

edg/jlcparts/JlcPartsBoardTop.py

@@ -0,0 +1,31 @@
+from ..parts import *
+
+from .JlcPartsResistorSmd import JlcPartsResistorSmd
+from .JlcPartsMlcc import JlcPartsMlcc
+from .JlcPartsInductor import JlcPartsInductor
+from .JlcPartsFerriteBead import JlcPartsFerriteBead
+from .JlcPartsDiode import JlcPartsDiode, JlcPartsZenerDiode
+from .JlcPartsLed import JlcPartsLed
+from .JlcPartsBjt import JlcPartsBjt
+from .JlcPartsFet import JlcPartsFet, JlcPartsSwitchFet
+from .JlcPartsPptcFuse import JlcPartsPptcFuse
+
+
+class JlcPartsRefinements(DesignTop):
+  """List of refinements that use JlcParts - mix this into a BoardTop"""
+  def refinements(self) -> Refinements:
+    return super().refinements() + Refinements(
+      class_refinements=[
+        (Resistor, JlcPartsResistorSmd),
+        (Capacitor, JlcPartsMlcc),
+        (Inductor, JlcPartsInductor),
+        (Diode, JlcPartsDiode),
+        (ZenerDiode, JlcPartsZenerDiode),
+        (Led, JlcPartsLed),
+        (Bjt, JlcPartsBjt),
+        (Fet, JlcPartsFet),
+        (SwitchFet, JlcPartsSwitchFet),
+        (PptcFuse, JlcPartsPptcFuse),
+        (FerriteBead, JlcPartsFerriteBead)
+      ]
+    )

edg/jlcparts/JlcPartsDiode.py

@@ -0,0 +1,70 @@
+from typing import Any, Optional, Dict
+from ..abstract_parts import *
+from ..parts.JlcDiode import JlcDiode
+from .JlcPartsBase import JlcPartsBase, JlcPartsAttributes
+
+
+class JlcPartsDiode(TableDiode, JlcPartsBase):
+    _JLC_PARTS_FILE_NAMES = [
+        "DiodesSchottky_Barrier_Diodes__SBD_",
+        "DiodesDiodes___Fast_Recovery_Rectifiers",
+        "DiodesDiodes___General_Purpose",
+        "DiodesSwitching_Diode",
+    ]
+
+    @classmethod
+    def _entry_to_table_row(cls, row_dict: Dict[PartsTableColumn, Any], filename: str, package: str, attributes: JlcPartsAttributes) \
+            -> Optional[Dict[PartsTableColumn, Any]]:
+        try:
+            row_dict[cls.KICAD_FOOTPRINT] = JlcDiode.PACKAGE_FOOTPRINT_MAP[package]
+
+            row_dict[cls.VOLTAGE_RATING] = Range.zero_to_upper(PartParserUtil.parse_value(
+                attributes.get("Reverse voltage (vr)", str), 'V'))
+            row_dict[cls.CURRENT_RATING] = Range.zero_to_upper(PartParserUtil.parse_value(
+                attributes.get("Average rectified current (io)", str), 'A'))
+            row_dict[cls.FORWARD_VOLTAGE] = Range.zero_to_upper(PartParserUtil.parse_value(
+                attributes.get("Forward voltage (vf@if)", str).split('@')[0], 'V'))
+
+            try:  # sometimes '-'
+                reverse_recovery = Range.exact(PartParserUtil.parse_value(
+                    attributes.get("Reverse recovery time (trr)", str), 's'))
+            except (KeyError, PartParserUtil.ParseError):
+                if filename == "DiodesDiodes___Fast_Recovery_Rectifiers":
+                    reverse_recovery = Range(0, 500e-9)  # arbitrary <500ns
+                else:
+                    reverse_recovery = Range.all()
+            row_dict[cls.REVERSE_RECOVERY] = reverse_recovery
+
+            return row_dict
+        except (KeyError, TypeError, PartParserUtil.ParseError):
+            return None
+
+
+class JlcPartsZenerDiode(TableZenerDiode, JlcPartsBase):
+    _JLC_PARTS_FILE_NAMES = ["DiodesZener_Diodes"]
+
+    @classmethod
+    def _entry_to_table_row(cls, row_dict: Dict[PartsTableColumn, Any], filename: str, package: str, attributes: JlcPartsAttributes) \
+            -> Optional[Dict[PartsTableColumn, Any]]:
+        try:
+            row_dict[cls.KICAD_FOOTPRINT] = JlcDiode.PACKAGE_FOOTPRINT_MAP[package]
+
+            if "Zener voltage (range)" in attributes:  # note, some devices have range='-'
+                zener_voltage_split = attributes.get("Zener voltage (range)", str).split('~')
+                zener_voltage = Range(
+                    PartParserUtil.parse_value(zener_voltage_split[0], 'V'),
+                    PartParserUtil.parse_value(zener_voltage_split[1], 'V')
+                )
+            else:  # explicit tolerance
+                zener_voltage = PartParserUtil.parse_abs_tolerance(
+                    attributes.get("Tolerance", str),
+                    PartParserUtil.parse_value(attributes.get("Zener voltage (nom)", str), 'V'),
+                    '')
+            row_dict[cls.ZENER_VOLTAGE] = zener_voltage
+
+            row_dict[cls.POWER_RATING] = Range.zero_to_upper(PartParserUtil.parse_value(
+                attributes.get("Power dissipation", str), 'W'))
+
+            return row_dict
+        except (KeyError, TypeError, PartParserUtil.ParseError):
+            return None