From ec314ee0399778196bddbda6be4e4120e131b992 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 14:12:28 +0200 Subject: [PATCH 01/12] Add scenario names for ECEMF task 5.3 --- definitions/scenario/scenarios.yaml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/definitions/scenario/scenarios.yaml b/definitions/scenario/scenarios.yaml index 0b25d12f..7abced80 100644 --- a/definitions/scenario/scenarios.yaml +++ b/definitions/scenario/scenarios.yaml @@ -99,3 +99,35 @@ - WP5 EastWest-1150 - WP5 EastWestTech-650 - WP5 EastWestTech-1150 + +# ECEMF - Work package 5 task 5.3 +- WP5 OPT-MIX +- WP5 RAP-MIX +- WP5 OPT-CP +- WP5 RAP-CP +- WP5 OPT-REG +- WP5 RAP-REG +- WP5 OPT-MIX-LimBio +- WP5 RAP-MIX-LimBio +- WP5 OPT-CP-LimBio +- WP5 RAP-CP-LimBio +- WP5 OPT-REG-LimBio +- WP5 RAP-REG-LimBio +- WP5 OPT-MIX-LimCCS +- WP5 RAP-MIX-LimCCS +- WP5 OPT-CP-LimCCS +- WP5 RAP-CP-LimCCS +- WP5 OPT-REG-LimCCS +- WP5 RAP-REG-LimCCS +- WP5 OPT-MIX-LimNuc +- WP5 RAP-MIX-LimNuc +- WP5 OPT-CP-LimNuc +- WP5 RAP-CP-LimNuc +- WP5 OPT-REG-LimNuc +- WP5 RAP-REG-LimNuc +- WP5 OPT-MIX-LimRES +- WP5 RAP-MIX-LimRES +- WP5 OPT-CP-LimRES +- WP5 RAP-CP-LimRES +- WP5 OPT-REG-LimRES +- WP5 RAP-REG-LimRES From 71834a6ad65418b9b8dbb2e3b9bab1234a126ebb Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 14:17:57 +0200 Subject: [PATCH 02/12] Bump nomenclature versions in dependency --- .github/workflows/pytest.yml | 4 ++-- .github/workflows/validation.yml | 6 +++--- setup.cfg | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5f8d4a03..5f7ddcdd 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,10 +17,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.11 - name: Install the package and run the tests run: | diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 2c94eff7..2bf6b513 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -17,13 +17,13 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies - run: pip install nomenclature-iamc==0.9.0 + run: pip install nomenclature-iamc==0.11.0 - name: Run the nomenclature project validation run: nomenclature validate-project . --dimensions "['region', 'scenario', 'variable']" diff --git a/setup.cfg b/setup.cfg index 42145738..de7e434f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,8 +13,7 @@ packages = openentrance install_requires = setuptools >= 41 pyyaml - nomenclature-iamc >= 0.9, <0.9.1 - pyam-iamc >= 1.0 # the pyam package is released on pypi under this name + nomenclature-iamc >= 0.11 iam-units >= 2021.11.12 setup_requires = setuptools >= 41 From 4ee95d322217888d0338c5e63747097685b5317b Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 14:24:33 +0200 Subject: [PATCH 03/12] Use latest version of pyam --- .github/workflows/validation.yml | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 2bf6b513..b5677d01 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -23,7 +23,7 @@ jobs: python-version: 3.11 - name: Install dependencies - run: pip install nomenclature-iamc==0.11.0 + run: pip install nomenclature-iamc==0.11.0 pyam-iamc>=2.0.0 - name: Run the nomenclature project validation run: nomenclature validate-project . --dimensions "['region', 'scenario', 'variable']" diff --git a/setup.cfg b/setup.cfg index de7e434f..6c42b267 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,7 @@ packages = openentrance install_requires = setuptools >= 41 pyyaml + pyam-iamc >= 2.0.0 nomenclature-iamc >= 0.11 iam-units >= 2021.11.12 setup_requires = From 8316f50cabf9a76da0b832415a61f265ecd12ce8 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 14:30:20 +0200 Subject: [PATCH 04/12] Pin pyam version to <2.0 --- .github/workflows/validation.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index b5677d01..87869f38 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -23,7 +23,7 @@ jobs: python-version: 3.11 - name: Install dependencies - run: pip install nomenclature-iamc==0.11.0 pyam-iamc>=2.0.0 + run: pip install nomenclature-iamc==0.11.0 pyam-iamc<2.0 - name: Run the nomenclature project validation run: nomenclature validate-project . --dimensions "['region', 'scenario', 'variable']" diff --git a/setup.cfg b/setup.cfg index 6c42b267..e0efdf03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ packages = openentrance install_requires = setuptools >= 41 pyyaml - pyam-iamc >= 2.0.0 + pyam-iamc < 2.0 nomenclature-iamc >= 0.11 iam-units >= 2021.11.12 setup_requires = From 473ddf242340754bdd22f9eedff426c90e37b03e Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 15:11:10 +0200 Subject: [PATCH 05/12] Fix dependencies to older version of pandas --- .github/workflows/validation.yml | 2 +- setup.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 87869f38..2bf6b513 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -23,7 +23,7 @@ jobs: python-version: 3.11 - name: Install dependencies - run: pip install nomenclature-iamc==0.11.0 pyam-iamc<2.0 + run: pip install nomenclature-iamc==0.11.0 - name: Run the nomenclature project validation run: nomenclature validate-project . --dimensions "['region', 'scenario', 'variable']" diff --git a/setup.cfg b/setup.cfg index e0efdf03..57c7c8fe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,8 @@ install_requires = setuptools >= 41 pyyaml pyam-iamc < 2.0 - nomenclature-iamc >= 0.11 + pandas < 2.1 + nomenclature-iamc == 0.9.0 iam-units >= 2021.11.12 setup_requires = setuptools >= 41 From 723ca6600d9c9971d1a0fe892753841a3365d475 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 15:11:25 +0200 Subject: [PATCH 06/12] Clean up of tests --- tests/test_validate.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/test_validate.py b/tests/test_validate.py index eb154b21..802a4656 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -10,13 +10,13 @@ from workflow import main as workflow -TEST_DF = pd.DataFrame( +TEST_DATA = pd.DataFrame( [ ["model_a", "scen_a", "Europe", "Primary Energy", "EJ/yr", 1, 6.0], ], columns=["model", "scenario", "region", "variable", "unit", 2005, 2010], ) -df = IamDataFrame(TEST_DF) +TEST_DF = IamDataFrame(TEST_DATA) def validate(df): @@ -30,29 +30,31 @@ def validate(df): def test_validate(): # test simple validation - assert validate(df) + assert validate(TEST_DF) def test_validate_fail(): # test that simple validation fails on variable and region dimension - assert not (validate(df.rename(variable={"Primary Energy": "foo"}))) - assert not (validate(df.rename(region={"Europe": "foo"}))) + print(validate(TEST_DF.rename(variable={"Primary Energy": "foo"}))) + + assert not validate(TEST_DF.rename(variable={"Primary Energy": "foo"})) + assert not validate(TEST_DF.rename(region={"Europe": "foo"})) def test_validate_directional(): # test that validation works as expected with directional data - assert validate(df.rename(region={"Europe": "Austria>Germany"})) - assert not validate(df.rename(region={"Europe": "Austria>foo"})) + assert validate(TEST_DF.rename(region={"Europe": "Austria>Germany"})) + assert not validate(TEST_DF.rename(region={"Europe": "Austria>foo"})) # test that directional data with more than one `>` fails - assert not validate(df.rename(region={"Europe": "Austria>Italy>France"})) + assert not validate(TEST_DF.rename(region={"Europe": "Austria>Italy>France"})) def test_validate_subannual_months(): # test that validation works as expected with months # (and representative timeslices generally) - assert validate(IamDataFrame(TEST_DF, subannual="January")) - assert not validate(IamDataFrame(TEST_DF, subannual="foo")) + assert validate(IamDataFrame(TEST_DATA, subannual="January")) + assert not validate(IamDataFrame(TEST_DATA, subannual="foo")) @pytest.mark.parametrize( @@ -66,7 +68,7 @@ def test_validate_subannual_months(): ) def test_validate_subannual_datetime(subannual, status): # test that validation works as expected with continuous time as subannual - assert validate(IamDataFrame(TEST_DF, subannual=subannual)) == status + assert validate(IamDataFrame(TEST_DATA, subannual=subannual)) == status @pytest.mark.parametrize( @@ -80,7 +82,7 @@ def test_validate_subannual_datetime(subannual, status): def test_validate_time_entry(rename_mapping, status): # test that validation works as expected with datetime-domain _df = IamDataFrame( - IamDataFrame(TEST_DF) + IamDataFrame(TEST_DATA) .data.rename(columns={"year": "time"}) .replace(rename_mapping) ) From cb957c20edc7e555aaa223375c8db9ac9930ecca Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 15:11:43 +0200 Subject: [PATCH 07/12] More clean-up of tests --- tests/test_validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate.py b/tests/test_validate.py index 802a4656..a9908860 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -90,4 +90,4 @@ def test_validate_time_entry(rename_mapping, status): def test_validate_unit_entry(): - assert not (validate(df.rename(unit={"EJ/yr": "MWh"}))) + assert not (validate(TEST_DF.rename(unit={"EJ/yr": "MWh"}))) From 2f4d3e4379757dedf553ae47d282496fe1685bb4 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 15:15:38 +0200 Subject: [PATCH 08/12] Try other combination of dependencies --- .github/workflows/validation.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 2bf6b513..d222d860 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -23,7 +23,7 @@ jobs: python-version: 3.11 - name: Install dependencies - run: pip install nomenclature-iamc==0.11.0 + run: pip install nomenclature-iamc==0.11.0 "pyam-iamc<2.0" - name: Run the nomenclature project validation run: nomenclature validate-project . --dimensions "['region', 'scenario', 'variable']" diff --git a/setup.cfg b/setup.cfg index 57c7c8fe..fafcadf0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ install_requires = pyyaml pyam-iamc < 2.0 pandas < 2.1 - nomenclature-iamc == 0.9.0 + nomenclature-iamc == 0.11.0 iam-units >= 2021.11.12 setup_requires = setuptools >= 41 From 42511182581cd5b9586a78970cd0139da618b7ab Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 16:00:32 +0200 Subject: [PATCH 09/12] Try again using latest pyam release --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index fafcadf0..80e0a2f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,8 +13,7 @@ packages = openentrance install_requires = setuptools >= 41 pyyaml - pyam-iamc < 2.0 - pandas < 2.1 + pyam-iamc >= 2.0 nomenclature-iamc == 0.11.0 iam-units >= 2021.11.12 setup_requires = From a1d7e690981892cbb1680811b4cf8ec71c4eb7bb Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 17:41:00 +0200 Subject: [PATCH 10/12] Use latest release of `nomenclature-iamc` --- .github/workflows/validation.yml | 2 +- setup.cfg | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index d222d860..047b79ec 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -23,7 +23,7 @@ jobs: python-version: 3.11 - name: Install dependencies - run: pip install nomenclature-iamc==0.11.0 "pyam-iamc<2.0" + run: pip install nomenclature-iamc>=0.12.0 - name: Run the nomenclature project validation run: nomenclature validate-project . --dimensions "['region', 'scenario', 'variable']" diff --git a/setup.cfg b/setup.cfg index 80e0a2f6..7ae33600 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,8 +13,7 @@ packages = openentrance install_requires = setuptools >= 41 pyyaml - pyam-iamc >= 2.0 - nomenclature-iamc == 0.11.0 + nomenclature-iamc >= 0.12.0 iam-units >= 2021.11.12 setup_requires = setuptools >= 41 From ff91946a3d7f5c6a94f0e06ae70757aad7fecddb Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 22:19:04 +0200 Subject: [PATCH 11/12] Simplify the tests --- tests/test_validate.py | 46 +++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/tests/test_validate.py b/tests/test_validate.py index a9908860..3ef1b1dc 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -19,42 +19,37 @@ TEST_DF = IamDataFrame(TEST_DATA) -def validate(df): - try: - workflow(df) - return True - except ValueError as e: - print(e) - return False - - def test_validate(): # test simple validation - assert validate(TEST_DF) + workflow(TEST_DF) def test_validate_fail(): # test that simple validation fails on variable and region dimension - print(validate(TEST_DF.rename(variable={"Primary Energy": "foo"}))) - assert not validate(TEST_DF.rename(variable={"Primary Energy": "foo"})) - assert not validate(TEST_DF.rename(region={"Europe": "foo"})) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(variable={"Primary Energy": "foo"})) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(region={"Europe": "foo"})) def test_validate_directional(): # test that validation works as expected with directional data - assert validate(TEST_DF.rename(region={"Europe": "Austria>Germany"})) - assert not validate(TEST_DF.rename(region={"Europe": "Austria>foo"})) + workflow(TEST_DF.rename(region={"Europe": "Austria>Germany"})) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(region={"Europe": "Austria>foo"})) # test that directional data with more than one `>` fails - assert not validate(TEST_DF.rename(region={"Europe": "Austria>Italy>France"})) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(region={"Europe": "Austria>Italy>France"})) def test_validate_subannual_months(): # test that validation works as expected with months # (and representative timeslices generally) - assert validate(IamDataFrame(TEST_DATA, subannual="January")) - assert not validate(IamDataFrame(TEST_DATA, subannual="foo")) + workflow(IamDataFrame(TEST_DATA, subannual="January")) + with pytest.raises(ValueError): + workflow(IamDataFrame(TEST_DATA, subannual="foo")) @pytest.mark.parametrize( @@ -68,7 +63,11 @@ def test_validate_subannual_months(): ) def test_validate_subannual_datetime(subannual, status): # test that validation works as expected with continuous time as subannual - assert validate(IamDataFrame(TEST_DATA, subannual=subannual)) == status + if status: + workflow(IamDataFrame(TEST_DATA, subannual=subannual)) + else: + with pytest.raises(ValueError): + workflow(IamDataFrame(TEST_DATA, subannual=subannual)) @pytest.mark.parametrize( @@ -86,8 +85,13 @@ def test_validate_time_entry(rename_mapping, status): .data.rename(columns={"year": "time"}) .replace(rename_mapping) ) - assert validate(_df) == status + if status: + workflow(_df) + else: + with pytest.raises(ValueError): + workflow(_df) def test_validate_unit_entry(): - assert not (validate(TEST_DF.rename(unit={"EJ/yr": "MWh"}))) + with pytest.raises(ValueError): + workflow(TEST_DF.rename(unit={"EJ/yr": "MWh"})) From 4712ccaa54257ba15bb6171e7e731f2660df9535 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 18 Sep 2023 22:19:25 +0200 Subject: [PATCH 12/12] Add a quickfix for a nomenclature-issue --- workflow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workflow.py b/workflow.py index bbc4c60e..af91f593 100755 --- a/workflow.py +++ b/workflow.py @@ -51,6 +51,10 @@ def main(df: pyam.IamDataFrame, dimensions=["region", "variable"]) -> pyam.IamDa # validate the region and variable dimensions, apply region processing df = process(df, definition, dimensions=dimensions, processor=processor) + # this is a quick-fix for https://github.com/IAMconsortium/nomenclature/issues/283 + if invalid_regions := definition.region.validate_items(df.region): + raise ValueError(f"Invalid regions: {invalid_regions}") + # convert to subannual format if data provided in datetime format if df.time_col == "time": logger.info('Re-casting from "time" column to categorical "subannual" format')