diff --git a/datacube/api/core.py b/datacube/api/core.py index 2e23b6681..9c88cde2c 100644 --- a/datacube/api/core.py +++ b/datacube/api/core.py @@ -476,6 +476,7 @@ def filter_jan(dataset): return dataset.time.begin.month == 1 :param limit: Optional. If provided, limit the maximum number of datasets returned. Useful for testing and debugging. + Can also be provided via the ``dc_load_limit`` config option. :param driver: Optional. If provided, use the specified driver to load the data. @@ -495,6 +496,9 @@ def filter_jan(dataset): return dataset.time.begin.month == 1 if datasets is None: assert product is not None # For type checker + if limit is None: + # check if a value was provided via the envvar + limit = self.index.environment["dc_load_limit"] datasets = self.find_datasets(ensure_location=True, dataset_predicate=dataset_predicate, like=like, limit=limit, diff --git a/datacube/cfg/api.py b/datacube/cfg/api.py index eecc8a9e5..43cdd70e3 100644 --- a/datacube/cfg/api.py +++ b/datacube/cfg/api.py @@ -15,7 +15,7 @@ from .cfg import find_config, parse_text from .exceptions import ConfigException -from .opt import ODCOptionHandler, AliasOptionHandler, IndexDriverOptionHandler, BoolOptionHandler +from .opt import ODCOptionHandler, AliasOptionHandler, IndexDriverOptionHandler, BoolOptionHandler, IntOptionHandler from .utils import ConfigDict, check_valid_env_name @@ -274,6 +274,7 @@ def __init__(self, AliasOptionHandler("alias", self), IndexDriverOptionHandler("index_driver", self, default="default"), BoolOptionHandler("skip_broken_datasets", self, default=False), + IntOptionHandler("dc_load_limit", self, minval=0), ] def get_all_aliases(self): diff --git a/datacube/cfg/opt.py b/datacube/cfg/opt.py index 48587b069..09eac8fa0 100644 --- a/datacube/cfg/opt.py +++ b/datacube/cfg/opt.py @@ -159,6 +159,8 @@ def __init__(self, *args, minval: int | None = None, maxval: int | None = None, def validate_and_normalise(self, value: Any) -> Any: # Call super() to get handle default value value = super().validate_and_normalise(value) + if value is None: + return value try: ival = int(value) except ValueError: diff --git a/datacube/scripts/system.py b/datacube/scripts/system.py index b8ef1ee39..cff4a1fff 100644 --- a/datacube/scripts/system.py +++ b/datacube/scripts/system.py @@ -80,6 +80,7 @@ def echo_field(name, value): db_url = psql_url_from_config(cfg_env) echo_field('Database URL:', db_url) echo_field('Skip broken datasets:', cfg_env.skip_broken_datasets) + echo_field('Datacube load limit:', cfg_env.dc_load_limit) echo() echo('Valid connection:\t', nl=False) diff --git a/integration_tests/test_config_tool.py b/integration_tests/test_config_tool.py index 5f3830a67..915fabb66 100644 --- a/integration_tests/test_config_tool.py +++ b/integration_tests/test_config_tool.py @@ -123,6 +123,7 @@ def test_config_check(clirunner, index, cfg_env): assert cfg_env['db_hostname'] in result.output assert cfg_env['db_username'] in result.output assert str(cfg_env['skip_broken_datasets']) in result.output + assert str(cfg_env['dc_load_limit']) in result.output def test_list_users_does_not_fail(clirunner, cfg_env, index):