Merge pull request #11 from automl/feat-metrics

feat: Metric definitions

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ files: |
   )/.*\.py$
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files
         files: ".*"
@@ -26,15 +26,15 @@ repos:
       - id: debug-statements
         files: '^src/.*\.py$'
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.23.3
+    rev: 0.27.1
     hooks:
       - id: check-github-workflows
         files: '^github/workflows/.*\.ya?ml$'
         types: ["yaml"]
       - id: check-dependabot
         files: '^\.github/dependabot\.ya?ml$'
   - repo: https://github.com/ambv/black
-    rev: 23.7.0
+    rev: 23.11.0
     hooks:
       - id: black
         name: black formatter mfpbench
@@ -43,7 +43,7 @@ repos:
         name: black formatter tests
         args: ["--config=pyproject.toml"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.7.1
     hooks:
       - id: mypy
         name: mypy
@@ -55,7 +55,7 @@ repos:
           - "--show-traceback"
           - "--allow-untyped-decorators" # Test decorators are not properly typed
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.278
+    rev: v0.1.6
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --no-cache]

docs/quickstart.md

@@ -139,7 +139,7 @@ print("contains", "X_1" in config)
 
 print("len", len(config))
 
-print("dict", dict(config))
+print("dict", config.as_dict())
 ```
 
 ??? tip "How is that done?"
@@ -150,16 +150,16 @@ print("dict", dict(config))
     and other pythonic things!
 
 
-=== "`dict()`/`from_dict()`"
+=== "`as_dict()`/`from_dict()`"
 
-    [`Config.dict()`][mfpbench.Config.dict] returns a dictionary of the config. This is useful for
+    [`Config.as_dict()`][mfpbench.Config.as_dict] returns a dictionary of the config. This is useful for
     working with the config in other libraries.
 
     ```python exec="true" source="material-block" result="python" session="quickstart"
     config = benchmark.sample()
     print(config)
 
-    config_dict = config.dict()
+    config_dict = config.as_dict()
     print(config_dict)
 
     new_config = benchmark.Config.from_dict(config_dict)
@@ -246,7 +246,7 @@ print("cost", result.cost)
 print(result)
 ```
 
-These share the [`dict()`][mfpbench.Result.dict] and [`from_dict()`][mfpbench.Result.from_dict]
+These share the [`as_dict()`][mfpbench.Result.as_dict] and [`from_dict()`][mfpbench.Result.from_dict]
 methods as [`Config`][mfpbench.Config] objects but do not behave like dictionaries.
 
 The most notable property of [`Result`][mfpbench.Result] objects is that also have the
@@ -278,7 +278,7 @@ identify the config in the table. **This is what's used to retrieve results from
 If this is missing when doing a [`query()`][mfpbench.Benchmark.query], we'll do our best to match
 the config to the table and get the correct id, but this is not guaranteed.
 
-When using [`dict()`][mfpbench.TabularConfig.dict], this `id` is **not** included in the dictionary.
+When using [`as_dict()`][mfpbench.TabularConfig.as_dict], this `id` is **not** included in the dictionary.
 In general you should either store the `config` object itself or at least `config.id`, that you can
 include back in before calling [`query()`][mfpbench.Benchmark.query].
 

pyproject.toml

@@ -10,12 +10,12 @@ dependencies = [
   "more_itertools",
   "pyarrow"
 ]
-version = "1.7.3"
+version = "1.7.4"
 description = "A wrapper for multi-fidelity benchmarks with priors"
 authors = [{name = "Eddie Bergman", email="[email protected]"}]
 readme = "README.md"
 license = { file = "LICENSE.txt" }
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 classifiers = [
   'Intended Audience :: Science/Research',
   'Intended Audience :: Developers',
@@ -61,7 +61,7 @@ dev = [
 
 [tool.pytest.ini_options]
 testpaths = ["tests"] # path to the test directory
-minversion = "3.7"
+minversion = "3.8"
 # addopts = "--cov=mfpbench" # Should be package name
 
 [tool.coverage.run]

src/mfpbench/__init__.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
 
 from mfpbench.benchmark import Benchmark
-from mfpbench.config import Config, GenericTabularConfig, TabularConfig
+from mfpbench.config import Config, TabularConfig
 from mfpbench.get import _mapping, get
 from mfpbench.jahs import JAHSBenchmark
 from mfpbench.lcbench_tabular import (
     LCBenchTabularBenchmark,
     LCBenchTabularConfig,
     LCBenchTabularResult,
 )
+from mfpbench.metric import Metric
 from mfpbench.pd1 import (
     PD1Benchmark,
     PD1cifar100_wideresnet_2048,
@@ -17,7 +18,7 @@
     PD1translatewmt_xformer_64,
     PD1uniref50_transformer_128,
 )
-from mfpbench.result import GenericTabularResult, Result
+from mfpbench.result import Result
 from mfpbench.synthetic.hartmann import (
     MFHartmann3Benchmark,
     MFHartmann3BenchmarkBad,
@@ -31,7 +32,7 @@
     MFHartmann6BenchmarkTerrible,
     MFHartmannBenchmark,
 )
-from mfpbench.tabular import GenericTabularBenchmark, TabularBenchmark
+from mfpbench.tabular import TabularBenchmark
 from mfpbench.yahpo import (
     IAMLglmnetBenchmark,
     IAMLrangerBenchmark,
@@ -58,11 +59,8 @@
     "YAHPOBenchmark",
     "PD1Benchmark",
     "TabularBenchmark",
-    "GenericTabularBenchmark",
     "Config",
     "TabularConfig",
-    "GenericTabularConfig",
-    "GenericTabularResult",
     "MFHartmannBenchmark",
     "MFHartmann3Benchmark",
     "MFHartmann6Benchmark",
@@ -97,5 +95,6 @@
     "PD1lm1b_transformer_2048",
     "PD1translatewmt_xformer_64",
     "PD1uniref50_transformer_128",
+    "Metric",
     "_mapping",
 ]