add load limit config option as well

datacube/api/core.py

@@ -476,6 +476,7 @@ def filter_jan(dataset): return dataset.time.begin.month == 1
 
         :param limit:
             Optional. If provided, limit the maximum number of datasets returned. Useful for testing and debugging.
+            Can also be provided via the ``dc_load_limit`` config option.
 
         :param driver:
             Optional. If provided, use the specified driver to load the data.
@@ -495,6 +496,9 @@ def filter_jan(dataset): return dataset.time.begin.month == 1
 
         if datasets is None:
             assert product is not None   # For type checker
+            if limit is None:
+                # check if a value was provided via the envvar
+                limit = self.index.environment["dc_load_limit"]
             datasets = self.find_datasets(ensure_location=True,
                                           dataset_predicate=dataset_predicate, like=like,
                                           limit=limit,

datacube/cfg/api.py

@@ -15,7 +15,7 @@
 
 from .cfg import find_config, parse_text
 from .exceptions import ConfigException
-from .opt import ODCOptionHandler, AliasOptionHandler, IndexDriverOptionHandler, BoolOptionHandler
+from .opt import ODCOptionHandler, AliasOptionHandler, IndexDriverOptionHandler, BoolOptionHandler, IntOptionHandler
 from .utils import ConfigDict, check_valid_env_name
 
 
@@ -274,6 +274,7 @@ def __init__(self,
             AliasOptionHandler("alias", self),
             IndexDriverOptionHandler("index_driver", self, default="default"),
             BoolOptionHandler("skip_broken_datasets", self, default=False),
+            IntOptionHandler("dc_load_limit", self, minval=0),
         ]
 
     def get_all_aliases(self):

datacube/cfg/opt.py

@@ -159,6 +159,8 @@ def __init__(self, *args, minval: int | None = None, maxval: int | None = None,
     def validate_and_normalise(self, value: Any) -> Any:
         # Call super() to get handle default value
         value = super().validate_and_normalise(value)
+        if value is None:
+            return value
         try:
             ival = int(value)
         except ValueError:

datacube/scripts/system.py

@@ -80,6 +80,7 @@ def echo_field(name, value):
     db_url = psql_url_from_config(cfg_env)
     echo_field('Database URL:', db_url)
     echo_field('Skip broken datasets:', cfg_env.skip_broken_datasets)
+    echo_field('Datacube load limit:', cfg_env.dc_load_limit)
 
     echo()
     echo('Valid connection:\t', nl=False)

integration_tests/test_config_tool.py

@@ -123,6 +123,7 @@ def test_config_check(clirunner, index, cfg_env):
     assert cfg_env['db_hostname'] in result.output
     assert cfg_env['db_username'] in result.output
     assert str(cfg_env['skip_broken_datasets']) in result.output
+    assert str(cfg_env['dc_load_limit']) in result.output
 
 
 def test_list_users_does_not_fail(clirunner, cfg_env, index):