diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5f8d4a03..5f7ddcdd 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,10 +17,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.11 - name: Install the package and run the tests run: | diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 2c94eff7..047b79ec 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -17,13 +17,13 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies - run: pip install nomenclature-iamc==0.9.0 + run: pip install nomenclature-iamc>=0.12.0 - name: Run the nomenclature project validation run: nomenclature validate-project . --dimensions "['region', 'scenario', 'variable']" diff --git a/definitions/scenario/scenarios.yaml b/definitions/scenario/scenarios.yaml index 0b25d12f..7abced80 100644 --- a/definitions/scenario/scenarios.yaml +++ b/definitions/scenario/scenarios.yaml @@ -99,3 +99,35 @@ - WP5 EastWest-1150 - WP5 EastWestTech-650 - WP5 EastWestTech-1150 + +# ECEMF - Work package 5 task 5.3 +- WP5 OPT-MIX +- WP5 RAP-MIX +- WP5 OPT-CP +- WP5 RAP-CP +- WP5 OPT-REG +- WP5 RAP-REG +- WP5 OPT-MIX-LimBio +- WP5 RAP-MIX-LimBio +- WP5 OPT-CP-LimBio +- WP5 RAP-CP-LimBio +- WP5 OPT-REG-LimBio +- WP5 RAP-REG-LimBio +- WP5 OPT-MIX-LimCCS +- WP5 RAP-MIX-LimCCS +- WP5 OPT-CP-LimCCS +- WP5 RAP-CP-LimCCS +- WP5 OPT-REG-LimCCS +- WP5 RAP-REG-LimCCS +- WP5 OPT-MIX-LimNuc +- WP5 RAP-MIX-LimNuc +- WP5 OPT-CP-LimNuc +- WP5 RAP-CP-LimNuc +- WP5 OPT-REG-LimNuc +- WP5 RAP-REG-LimNuc +- WP5 OPT-MIX-LimRES +- WP5 RAP-MIX-LimRES +- WP5 OPT-CP-LimRES +- WP5 RAP-CP-LimRES +- WP5 OPT-REG-LimRES +- WP5 RAP-REG-LimRES diff --git a/setup.cfg b/setup.cfg index 42145738..7ae33600 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,8 +13,7 @@ packages = openentrance install_requires = setuptools >= 41 pyyaml - nomenclature-iamc >= 0.9, <0.9.1 - pyam-iamc >= 1.0 # the pyam package is released on pypi under this name + nomenclature-iamc >= 0.12.0 iam-units >= 2021.11.12 setup_requires = setuptools >= 41 diff --git a/tests/test_validate.py b/tests/test_validate.py index eb154b21..3ef1b1dc 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -10,49 +10,46 @@ from workflow import main as workflow -TEST_DF = pd.DataFrame( +TEST_DATA = pd.DataFrame( [ ["model_a", "scen_a", "Europe", "Primary Energy", "EJ/yr", 1, 6.0], ], columns=["model", "scenario", "region", "variable", "unit", 2005, 2010], ) -df = IamDataFrame(TEST_DF) - - -def validate(df): - try: - workflow(df) - return True - except ValueError as e: - print(e) - return False +TEST_DF = IamDataFrame(TEST_DATA) def test_validate(): # test simple validation - assert validate(df) + workflow(TEST_DF) def test_validate_fail(): # test that simple validation fails on variable and region dimension - assert not (validate(df.rename(variable={"Primary Energy": "foo"}))) - assert not (validate(df.rename(region={"Europe": "foo"}))) + + with pytest.raises(ValueError): + workflow(TEST_DF.rename(variable={"Primary Energy": "foo"})) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(region={"Europe": "foo"})) def test_validate_directional(): # test that validation works as expected with directional data - assert validate(df.rename(region={"Europe": "Austria>Germany"})) - assert not validate(df.rename(region={"Europe": "Austria>foo"})) + workflow(TEST_DF.rename(region={"Europe": "Austria>Germany"})) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(region={"Europe": "Austria>foo"})) # test that directional data with more than one `>` fails - assert not validate(df.rename(region={"Europe": "Austria>Italy>France"})) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(region={"Europe": "Austria>Italy>France"})) def test_validate_subannual_months(): # test that validation works as expected with months # (and representative timeslices generally) - assert validate(IamDataFrame(TEST_DF, subannual="January")) - assert not validate(IamDataFrame(TEST_DF, subannual="foo")) + workflow(IamDataFrame(TEST_DATA, subannual="January")) + with pytest.raises(ValueError): + workflow(IamDataFrame(TEST_DATA, subannual="foo")) @pytest.mark.parametrize( @@ -66,7 +63,11 @@ def test_validate_subannual_months(): ) def test_validate_subannual_datetime(subannual, status): # test that validation works as expected with continuous time as subannual - assert validate(IamDataFrame(TEST_DF, subannual=subannual)) == status + if status: + workflow(IamDataFrame(TEST_DATA, subannual=subannual)) + else: + with pytest.raises(ValueError): + workflow(IamDataFrame(TEST_DATA, subannual=subannual)) @pytest.mark.parametrize( @@ -80,12 +81,17 @@ def test_validate_subannual_datetime(subannual, status): def test_validate_time_entry(rename_mapping, status): # test that validation works as expected with datetime-domain _df = IamDataFrame( - IamDataFrame(TEST_DF) + IamDataFrame(TEST_DATA) .data.rename(columns={"year": "time"}) .replace(rename_mapping) ) - assert validate(_df) == status + if status: + workflow(_df) + else: + with pytest.raises(ValueError): + workflow(_df) def test_validate_unit_entry(): - assert not (validate(df.rename(unit={"EJ/yr": "MWh"}))) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(unit={"EJ/yr": "MWh"})) diff --git a/workflow.py b/workflow.py index bbc4c60e..af91f593 100755 --- a/workflow.py +++ b/workflow.py @@ -51,6 +51,10 @@ def main(df: pyam.IamDataFrame, dimensions=["region", "variable"]) -> pyam.IamDa # validate the region and variable dimensions, apply region processing df = process(df, definition, dimensions=dimensions, processor=processor) + # this is a quick-fix for https://github.com/IAMconsortium/nomenclature/issues/283 + if invalid_regions := definition.region.validate_items(df.region): + raise ValueError(f"Invalid regions: {invalid_regions}") + # convert to subannual format if data provided in datetime format if df.time_col == "time": logger.info('Re-casting from "time" column to categorical "subannual" format')