From 01dedaad72c61c386c5a87e4b2dc9b6724e5560a Mon Sep 17 00:00:00 2001 From: Sheng Yu Date: Fri, 12 Apr 2024 12:22:25 -0400 Subject: [PATCH] feat(grammar)!: generalized grammar type class (#19) --- craft_grammar/models.py | 248 ++++++++++++-------------------------- tests/unit/test_models.py | 134 ++++---------------- 2 files changed, 102 insertions(+), 280 deletions(-) diff --git a/craft_grammar/models.py b/craft_grammar/models.py index 93f0b2e..5d86039 100644 --- a/craft_grammar/models.py +++ b/craft_grammar/models.py @@ -1,6 +1,6 @@ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # -# Copyright 2022 Canonical Ltd. +# Copyright 2022-2024 Canonical Ltd. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,9 +18,11 @@ import abc import re -from typing import Any, Dict, List, Union +from typing import Any, Generic, List, TypeVar, get_args, get_origin from overrides import overrides +from pydantic import BaseConfig, PydanticTypeError +from pydantic.validators import find_validators _on_pattern = re.compile(r"^on\s+(.+)\s*$") _to_pattern = re.compile(r"^to\s+(.+)\s*$") @@ -31,7 +33,7 @@ _TRY = "try" -_GrammarType = Dict[str, Any] +T = TypeVar("T") class _GrammarBase(abc.ABC): @@ -53,196 +55,102 @@ def _grammar_append(cls, entry: List, item: Any) -> None: _mark_and_append(entry, {key: cls.validate(value)}) -# Public types for grammar-enabled attributes -class GrammarBool(_GrammarBase): - """Grammar-enabled bool field.""" +def _format_type_error(type_: type, entry: Any) -> str: + """Format a type error message.""" + origin = get_origin(type_) + args = get_args(type_) - __root__: Union[bool, _GrammarType] + # handle primitive types which origin is None + if not origin: + origin = type_ - @classmethod - @overrides - def validate(cls, entry): - # GrammarBool entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of bool: {entry!r}") - return new_entry - - if isinstance(entry, bool): - return entry - - raise TypeError(f"value must be a bool: {entry!r}") - - -class GrammarInt(_GrammarBase): - """Grammar-enabled integer field.""" - - __root__: Union[int, _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarInt entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of integer: {entry!r}") - return new_entry + if issubclass(origin, list): + if args: + return f"value must be a list of {args[0].__name__}: {entry!r}" + return f"value must be a list: {entry!r}" - if isinstance(entry, int): - return entry + if issubclass(origin, dict): + if len(args) == 2: + return f"value must be a dict of {args[0].__name__} to {args[1].__name__}: {entry!r}" + return f"value must be a dict: {entry!r}" - raise TypeError(f"value must be a integer: {entry!r}") + return f"value must be a {type_.__name__}: {entry!r}" -class GrammarFloat(_GrammarBase): - """Grammar-enabled float field.""" +class GrammarMetaClass(type): + """Grammar type metaclass. - __root__: Union[float, _GrammarType] + Allows to use GrammarType[T] to define a grammar-aware type. + """ - @classmethod - @overrides - def validate(cls, entry): - # GrammarFloat entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of float: {entry!r}") - return new_entry + # Define __getitem__ method to be able to use index + def __getitem__(cls, type_): + class GrammarScalar(_GrammarBase): + """Grammar scalar class. - if isinstance(entry, (int, float)): - return float(entry) + Dynamically generated class to handle grammar-aware types. + """ - raise TypeError(f"value must be a float: {entry!r}") + @classmethod + @overrides + def validate(cls, entry): + # Grammar[T] entry can be a list if it contains clauses + if isinstance(entry, list): + # Check if the type_ supposed to be a list + sub_type = get_args(type_) + # handle typed list + if sub_type: + sub_type = sub_type[0] + if sub_type is Any: + sub_type = None -class GrammarStr(_GrammarBase): - """Grammar-enabled string field.""" + new_entry = [] + for item in entry: + # Check if the item is a valid grammar clause + if _is_grammar_clause(item): + cls._grammar_append(new_entry, item) + else: + # Check if the item is a valid type if not a grammar clause + if sub_type and isinstance(item, sub_type): + new_entry.append(item) + else: + raise TypeError(_format_type_error(type_, entry)) - __root__: Union[str, _GrammarType] + return new_entry - @classmethod - @overrides - def validate(cls, entry): - # GrammarStr entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a string: {entry!r}") - return new_entry + # Not a valid grammar, check if it is a dict + if isinstance(entry, dict): + # Check if the type_ supposed to be a dict + if get_origin(type_) is not dict: + raise TypeError(_format_type_error(type_, entry)) - if isinstance(entry, str): - return entry + # we do not care about the dict contents type, other models will handle it + return entry - raise TypeError(f"value must be a string: {entry!r}") + # handle primitive types with pydantic validators + try: + for validator in find_validators(type_, BaseConfig): + # we do not need the return value of the validator + validator(entry) + except PydanticTypeError as err: + raise TypeError(_format_type_error(type_, entry)) from err + return entry -class GrammarStrList(_GrammarBase): - """Grammar-enabled list of strings field.""" - - __root__: Union[List[Union[str, _GrammarType]], _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarStrList will always be a list - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - elif isinstance(item, str): - new_entry.append(item) - else: - raise TypeError(f"value must be a list of string: {entry!r}") - return new_entry + return GrammarScalar - raise TypeError(f"value must be a list of string: {entry!r}") +class Grammar(Generic[T], metaclass=GrammarMetaClass): + """Grammar aware type. -class GrammarSingleEntryDictList(_GrammarBase): - """Grammar-enabled list of dictionaries field.""" + Allows to use Grammar[T] to define a grammar-aware type. - __root__: Union[List[Dict[str, Any]], _GrammarType] + Grammar[int] + Grammar[list[str]] + Grammar[dict[str, int]] - @classmethod - @overrides - def validate(cls, entry): - # GrammarSingleEntryDictList will always be a list - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - elif isinstance(item, dict) and len(item) == 1: - new_entry.append(item) - else: - raise TypeError( - f"value must be a list of single-entry dictionaries: {entry!r}" - ) - return new_entry - - raise TypeError(f"value must be a list of single-entry dictionaries: {entry!r}") - - -class GrammarDict(_GrammarBase): - """Grammar-enabled dictionary field.""" - - __root__: Union[Dict[str, Any], _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarDict entry can be a list if it contains clauses - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - else: - raise TypeError(f"value must be a list of dictionaries: {entry!r}") - return new_entry - - if isinstance(entry, dict): - return entry - - raise TypeError(f"value must be a dictionary: {entry!r}") - - -class GrammarDictList(_GrammarBase): - """Grammar-enabled list of dictionary field.""" - - __root__: Union[List[Dict[str, Any]], _GrammarType] - - @classmethod - @overrides - def validate(cls, entry): - # GrammarDictList will always be a list - if isinstance(entry, list): - new_entry = [] - for item in entry: - if _is_grammar_clause(item): - cls._grammar_append(new_entry, item) - elif isinstance(item, dict): - new_entry.append(item) - else: - raise TypeError(f"value must be a list of dictionaries: {entry!r}") - return new_entry - - raise TypeError(f"value must be a list of dictionary: {entry!r}") + """ def _ensure_selector_valid(selector: str, *, clause: str) -> None: diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 868cc6c..9a914aa 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -1,6 +1,6 @@ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # -# Copyright 2022 Canonical Ltd. +# Copyright 2022-2024 Canonical Ltd. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,35 +16,26 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import textwrap +from typing import Any, Dict, List import pydantic import pytest import yaml -from craft_grammar.models import ( - GrammarBool, - GrammarDict, - GrammarDictList, - GrammarFloat, - GrammarInt, - GrammarSingleEntryDictList, - GrammarStr, - GrammarStrList, -) +from craft_grammar.models import Grammar class ValidationTest(pydantic.BaseModel): """A test model containing all types of grammar-aware types.""" control: str - grammar_bool: GrammarBool - grammar_int: GrammarInt - grammar_float: GrammarFloat - grammar_str: GrammarStr - grammar_strlist: GrammarStrList - grammar_dict: GrammarDict - grammar_single_entry_dictlist: GrammarSingleEntryDictList - grammar_dictlist: GrammarDictList + grammar_bool: Grammar[bool] + grammar_int: Grammar[int] + grammar_float: Grammar[float] + grammar_str: Grammar[str] + grammar_strlist: Grammar[List[str]] + grammar_dict: Grammar[Dict[str, Any]] + grammar_dictlist: Grammar[List[Dict]] def test_validate_grammar_trivial(): @@ -63,9 +54,6 @@ def test_validate_grammar_trivial(): grammar_dict: key: value other_key: other_value - grammar_single_entry_dictlist: - - key: value - - other_key: other_value grammar_dictlist: - key: value other_key: other_value @@ -83,10 +71,6 @@ def test_validate_grammar_trivial(): assert v.grammar_str == "another string" assert v.grammar_strlist == ["a", "string", "list"] assert v.grammar_dict == {"key": "value", "other_key": "other_value"} - assert v.grammar_single_entry_dictlist == [ - {"key": "value"}, - {"other_key": "other_value"}, - ] assert v.grammar_dictlist == [ {"key": "value", "other_key": "other_value"}, {"key2": "value", "other_key2": "other_value"}, @@ -121,11 +105,6 @@ def test_validate_grammar_simple(): key: value other_key: other_value - else fail - grammar_single_entry_dictlist: - - on arch: - - key: value - - other_key: other_value - - else fail grammar_dictlist: - on arch: - key: value @@ -163,10 +142,6 @@ def test_validate_grammar_simple(): {"*on amd64": {"key": "value", "other_key": "other_value"}}, "*else fail", ] - assert v.grammar_single_entry_dictlist == [ - {"*on arch": [{"key": "value"}, {"other_key": "other_value"}]}, - "*else fail", - ] assert v.grammar_dictlist == [ { "*on arch": [ @@ -231,16 +206,6 @@ def test_validate_grammar_recursive(): yet_another_key: yet_another_value - else fail - else fail - grammar_single_entry_dictlist: - - on arch,other_arch: - - on other_arch: - - to yet_another_arch: - - key: value - - other_key: other_value - - else fail - - else: - - yet_another_key: yet_another_value - - else fail grammar_dictlist: - on arch,other_arch: - on other_arch: @@ -321,26 +286,6 @@ def test_validate_grammar_recursive(): }, ] - assert v.grammar_single_entry_dictlist == [ - { - "*on arch,other_arch": [ - { - "*on other_arch": [ - { - "*to yet_another_arch": [ - {"key": "value"}, - {"other_key": "other_value"}, - ] - }, - "*else fail", - ] - }, - {"*else": [{"yet_another_key": "yet_another_value"}]}, - ] - }, - "*else fail", - ] - assert v.grammar_dictlist == [ { "*on arch,other_arch": [ @@ -373,13 +318,13 @@ def test_validate_grammar_recursive(): @pytest.mark.parametrize( "value", - [23, True, ["foo"], {"x"}, [{"a": "b"}]], + [["foo"], {"x"}, [{"a": "b"}]], ) def test_grammar_str_error(value): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: Grammar[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=value) # type: ignore @@ -388,7 +333,7 @@ class GrammarValidation(pydantic.BaseModel): assert len(err) == 1 assert err[0]["loc"] == ("x",) assert err[0]["type"] == "type_error" - assert err[0]["msg"] == f"value must be a string: {value!r}" + assert err[0]["msg"] == f"value must be a str: {value!r}" @pytest.mark.parametrize( @@ -399,7 +344,7 @@ def test_grammar_strlist_error(value): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStrList + x: Grammar[List[str]] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=value) # type: ignore @@ -408,55 +353,33 @@ class GrammarValidation(pydantic.BaseModel): assert len(err) == 1 assert err[0]["loc"] == ("x",) assert err[0]["type"] == "type_error" - assert err[0]["msg"] == f"value must be a list of string: {value!r}" - - -@pytest.mark.parametrize( - "value", - [23, "string", [{"a": 42}, "foo"], [{"a": 42, "b": 43}]], -) -def test_grammar_single_entry_dictlist_error(value): - class GrammarValidation(pydantic.BaseModel): - """Test validation of grammar-enabled types.""" - - x: GrammarSingleEntryDictList - - with pytest.raises(pydantic.ValidationError) as raised: - GrammarValidation(x=value) # type: ignore - - err = raised.value.errors() - assert len(err) == 1 - assert err[0]["loc"] == ("x",) - assert err[0]["type"] == "type_error" - assert err[0]["msg"] == ( - f"value must be a list of single-entry dictionaries: {value!r}" - ) + assert err[0]["msg"] == f"value must be a list of str: {value!r}" def test_grammar_nested_error(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: Grammar[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation( x=[ - {"on arm64,amd64": [{"on arm64": "foo"}, {"else": 35}]}, + {"on arm64,amd64": [{"on arm64": "foo"}, {"else": [35]}]}, ] # type: ignore ) err = raised.value.errors() assert len(err) == 1 assert err[0]["loc"] == ("x",) assert err[0]["type"] == "type_error" - assert err[0]["msg"] == "value must be a string: 35" + assert err[0]["msg"] == "value must be a str: [35]" def test_grammar_str_elsefail(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: Grammar[str] GrammarValidation(x=[{"on arch": "foo"}, "else fail"]) # type: ignore @@ -465,25 +388,16 @@ def test_grammar_strlist_elsefail(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStrList + x: Grammar[List[str]] GrammarValidation(x=[{"on arch": ["foo"]}, "else fail"]) # type: ignore -def test_grammar_single_entry_dictlist_elsefail(): - class GrammarValidation(pydantic.BaseModel): - """Test validation of grammar-enabled types.""" - - x: GrammarSingleEntryDictList - - GrammarValidation(x=[{"on arch": [{"foo": "bar"}]}, "else fail"]) # type: ignore - - def test_grammar_try(): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: Grammar[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=[{"try": "foo"}]) # type: ignore @@ -498,13 +412,13 @@ class GrammarValidation(pydantic.BaseModel): @pytest.mark.parametrize( "clause,err_msg", [ - ("on", "value must be a string: [{'on': 'foo'}]"), + ("on", "value must be a str: [{'on': 'foo'}]"), ("on ,", "syntax error in 'on' selector"), ("on ,arch", "syntax error in 'on' selector"), ("on arch,", "syntax error in 'on' selector"), ("on arch,,arch", "syntax error in 'on' selector"), ("on arch, arch", "spaces are not allowed in 'on' selector"), - ("to", "value must be a string: [{'to': 'foo'}]"), + ("to", "value must be a str: [{'to': 'foo'}]"), ("to ,", "syntax error in 'to' selector"), ("to ,arch", "syntax error in 'to' selector"), ("to arch,", "syntax error in 'to' selector"), @@ -526,7 +440,7 @@ def test_grammar_errors(clause, err_msg): class GrammarValidation(pydantic.BaseModel): """Test validation of grammar-enabled types.""" - x: GrammarStr + x: Grammar[str] with pytest.raises(pydantic.ValidationError) as raised: GrammarValidation(x=[{clause: "foo"}]) # type: ignore