refactor entities

README.md

@@ -17,7 +17,7 @@ Inside this repo you can find all important pieces for running MedPerf. In its c
 
 If you use MedPerf, please cite our main paper: Karargyris, A., Umeton, R., Sheller, M.J. et al. Federated benchmarking of medical artificial intelligence with MedPerf. *Nature Machine Intelligence* **5**, 799–810 (2023). [https://www.nature.com/articles/s42256-023-00652-2](https://www.nature.com/articles/s42256-023-00652-2)
 
-Additonally, here you can see how others used MedPerf already: [https://scholar.google.com/scholar?q="medperf"](https://scholar.google.com/scholar?q="medperf").
+Additionally, here you can see how others used MedPerf already: [https://scholar.google.com/scholar?q="medperf"](https://scholar.google.com/scholar?q="medperf").
 
 ## Experiments
 

cli/cli_tests.sh

@@ -5,7 +5,6 @@
 ################### Start Testing ########################
 ##########################################################
 
-
 ##########################################################
 echo "=========================================="
 echo "Printing MedPerf version"
@@ -195,7 +194,7 @@ echo "Running data submission step"
 echo "====================================="
 print_eval "medperf dataset submit -p $PREP_UID -d $DIRECTORY/dataset_a -l $DIRECTORY/dataset_a --name='dataset_a' --description='mock dataset a' --location='mock location a' -y"
 checkFailed "Data submission step failed"
-DSET_A_UID=$(medperf dataset ls | grep dataset_a | tr -s ' ' | cut -d ' ' -f 1)
+DSET_A_UID=$(medperf dataset ls | grep dataset_a | tr -s ' ' | awk '{$1=$1;print}' | cut -d ' ' -f 1)
 echo "DSET_A_UID=$DSET_A_UID"
 ##########################################################
 

cli/medperf/cli.py

@@ -71,7 +71,7 @@ def execute(
             please run the command again with the --no-cache option.\n"""
         )
     else:
-        ResultSubmission.run(result.generated_uid, approved=approval)
+        ResultSubmission.run(result.local_id, approved=approval)
     config.ui.print("✅ Done!")
 
 

cli/medperf/commands/benchmark/benchmark.py

@@ -16,14 +16,16 @@
 @app.command("ls")
 @clean_except
 def list(
-    local: bool = typer.Option(False, "--local", help="Get local benchmarks"),
+    unregistered: bool = typer.Option(
+        False, "--unregistered", help="Get unregistered benchmarks"
+    ),
     mine: bool = typer.Option(False, "--mine", help="Get current-user benchmarks"),
 ):
-    """List benchmarks stored locally and remotely from the user"""
+    """List benchmarks"""
     EntityList.run(
         Benchmark,
         fields=["UID", "Name", "Description", "State", "Approval Status", "Registered"],
-        local_only=local,
+        unregistered=unregistered,
         mine_only=mine,
     )
 
@@ -162,10 +164,10 @@ def view(
         "--format",
         help="Format to display contents. Available formats: [yaml, json]",
     ),
-    local: bool = typer.Option(
+    unregistered: bool = typer.Option(
         False,
-        "--local",
-        help="Display local benchmarks if benchmark ID is not provided",
+        "--unregistered",
+        help="Display unregistered benchmarks if benchmark ID is not provided",
     ),
     mine: bool = typer.Option(
         False,
@@ -180,4 +182,4 @@ def view(
     ),
 ):
     """Displays the information of one or more benchmarks"""
-    EntityView.run(entity_id, Benchmark, format, local, mine, output)
+    EntityView.run(entity_id, Benchmark, format, unregistered, mine, output)

cli/medperf/commands/benchmark/submit.py

@@ -79,7 +79,7 @@ def run_compatibility_test(self):
         self.ui.print("Running compatibility test")
         self.bmk.write()
         data_uid, results = CompatibilityTestExecution.run(
-            benchmark=self.bmk.generated_uid,
+            benchmark=self.bmk.local_id,
             no_cache=self.no_cache,
             skip_data_preparation_step=self.skip_data_preparation_step,
         )

cli/medperf/commands/compatibility_test/compatibility_test.py

@@ -95,7 +95,11 @@ def run(
 @clean_except
 def list():
     """List previously executed tests reports."""
-    EntityList.run(TestReport, fields=["UID", "Data Source", "Model", "Evaluator"])
+    EntityList.run(
+        TestReport,
+        fields=["UID", "Data Source", "Model", "Evaluator"],
+        unregistered=True,
+    )
 
 
 @app.command("view")
@@ -116,4 +120,4 @@ def view(
     ),
 ):
     """Displays the information of one or more test reports"""
-    EntityView.run(entity_id, TestReport, format, output=output)
+    EntityView.run(entity_id, TestReport, format, unregistered=True, output=output)

cli/medperf/commands/compatibility_test/run.py

@@ -239,7 +239,7 @@ def cached_results(self):
         """
         if self.no_cache:
             return
-        uid = self.report.generated_uid
+        uid = self.report.local_id
         try:
             report = TestReport.get(uid)
         except InvalidArgumentError:

cli/medperf/commands/compatibility_test/utils.py

@@ -138,23 +138,23 @@ def create_test_dataset(
     # TODO: existing dataset could make problems
     # make some changes since this is a test dataset
     config.tmp_paths.remove(data_creation.dataset.path)
-    data_creation.dataset.write()
     if skip_data_preparation_step:
         data_creation.make_dataset_prepared()
     dataset = data_creation.dataset
+    old_generated_uid = dataset.generated_uid
+    old_path = dataset.path
 
     # prepare/check dataset
     DataPreparation.run(dataset.generated_uid)
 
     # update dataset generated_uid
-    old_path = dataset.path
-    generated_uid = get_folders_hash([dataset.data_path, dataset.labels_path])
-    dataset.generated_uid = generated_uid
-    dataset.write()
-    if dataset.input_data_hash != dataset.generated_uid:
+    new_generated_uid = get_folders_hash([dataset.data_path, dataset.labels_path])
+    if new_generated_uid != old_generated_uid:
         # move to a correct location if it underwent preparation
-        new_path = old_path.replace(dataset.input_data_hash, generated_uid)
+        new_path = old_path.replace(old_generated_uid, new_generated_uid)
         remove_path(new_path)
         os.rename(old_path, new_path)
+        dataset.generated_uid = new_generated_uid
+        dataset.write()
 
-    return generated_uid
+    return new_generated_uid

cli/medperf/commands/dataset/dataset.py

@@ -17,17 +17,19 @@
 @app.command("ls")
 @clean_except
 def list(
-    local: bool = typer.Option(False, "--local", help="Get local datasets"),
+    unregistered: bool = typer.Option(
+        False, "--unregistered", help="Get unregistered datasets"
+    ),
     mine: bool = typer.Option(False, "--mine", help="Get current-user datasets"),
     mlcube: int = typer.Option(
         None, "--mlcube", "-m", help="Get datasets for a given data prep mlcube"
     ),
 ):
-    """List datasets stored locally and remotely from the user"""
+    """List datasets"""
     EntityList.run(
         Dataset,
         fields=["UID", "Name", "Data Preparation Cube UID", "State", "Status", "Owner"],
-        local_only=local,
+        unregistered=unregistered,
         mine_only=mine,
         mlcube=mlcube,
     )
@@ -149,8 +151,10 @@ def view(
         "--format",
         help="Format to display contents. Available formats: [yaml, json]",
     ),
-    local: bool = typer.Option(
-        False, "--local", help="Display local datasets if dataset ID is not provided"
+    unregistered: bool = typer.Option(
+        False,
+        "--unregistered",
+        help="Display unregistered datasets if dataset ID is not provided",
     ),
     mine: bool = typer.Option(
         False,
@@ -165,4 +169,4 @@ def view(
     ),
 ):
     """Displays the information of one or more datasets"""
-    EntityView.run(entity_id, Dataset, format, local, mine, output)
+    EntityView.run(entity_id, Dataset, format, unregistered, mine, output)

cli/medperf/commands/execution.py

@@ -47,23 +47,23 @@ def prepare(self):
         logging.debug(f"tmp results output: {self.results_path}")
 
     def __setup_logs_path(self):
-        model_uid = self.model.generated_uid
-        eval_uid = self.evaluator.generated_uid
-        data_hash = self.dataset.generated_uid
+        model_uid = self.model.local_id
+        eval_uid = self.evaluator.local_id
+        data_uid = self.dataset.local_id
 
         logs_path = os.path.join(
-            config.experiments_logs_folder, str(model_uid), str(data_hash)
+            config.experiments_logs_folder, str(model_uid), str(data_uid)
         )
         os.makedirs(logs_path, exist_ok=True)
         model_logs_path = os.path.join(logs_path, "model.log")
         metrics_logs_path = os.path.join(logs_path, f"metrics_{eval_uid}.log")
         return model_logs_path, metrics_logs_path
 
     def __setup_predictions_path(self):
-        model_uid = self.model.generated_uid
-        data_hash = self.dataset.generated_uid
+        model_uid = self.model.local_id
+        data_uid = self.dataset.local_id
         preds_path = os.path.join(
-            config.predictions_folder, str(model_uid), str(data_hash)
+            config.predictions_folder, str(model_uid), str(data_uid)
         )
         if os.path.exists(preds_path):
             msg = f"Found existing predictions for model {self.model.id} on dataset "

cli/medperf/commands/list.py

@@ -1,3 +1,5 @@
+from typing import List, Type
+from medperf.entities.interface import Entity
 from medperf.exceptions import InvalidArgumentError
 from tabulate import tabulate
 
@@ -8,29 +10,38 @@
 class EntityList:
     @staticmethod
     def run(
-        entity_class,
-        fields,
-        local_only: bool = False,
+        entity_class: Type[Entity],
+        fields: List[str],
+        unregistered: bool = False,
         mine_only: bool = False,
         **kwargs,
     ):
         """Lists all local datasets
 
         Args:
-            local_only (bool, optional): Display all local results. Defaults to False.
-            mine_only (bool, optional): Display all current-user results. Defaults to False.
+            unregistered (bool, optional): Display only local unregistered results. Defaults to False.
+            mine_only (bool, optional): Display all registered current-user results. Defaults to False.
             kwargs (dict): Additional parameters for filtering entity lists.
         """
-        entity_list = EntityList(entity_class, fields, local_only, mine_only, **kwargs)
+        entity_list = EntityList(
+            entity_class, fields, unregistered, mine_only, **kwargs
+        )
         entity_list.prepare()
         entity_list.validate()
         entity_list.filter()
         entity_list.display()
 
-    def __init__(self, entity_class, fields, local_only, mine_only, **kwargs):
+    def __init__(
+        self,
+        entity_class: Type[Entity],
+        fields: List[str],
+        unregistered: bool,
+        mine_only: bool,
+        **kwargs,
+    ):
         self.entity_class = entity_class
         self.fields = fields
-        self.local_only = local_only
+        self.unregistered = unregistered
         self.mine_only = mine_only
         self.filters = kwargs
         self.data = []
@@ -40,7 +51,7 @@ def prepare(self):
             self.filters["owner"] = get_medperf_user_data()["id"]
 
         entities = self.entity_class.all(
-            local_only=self.local_only, filters=self.filters
+            unregistered=self.unregistered, filters=self.filters
         )
         self.data = [entity.display_dict() for entity in entities]
 

cli/medperf/commands/mlcube/mlcube.py

@@ -16,14 +16,16 @@
 @app.command("ls")
 @clean_except
 def list(
-    local: bool = typer.Option(False, "--local", help="Get local mlcubes"),
+    unregistered: bool = typer.Option(
+        False, "--unregistered", help="Get unregistered mlcubes"
+    ),
     mine: bool = typer.Option(False, "--mine", help="Get current-user mlcubes"),
 ):
-    """List mlcubes stored locally and remotely from the user"""
+    """List mlcubes"""
     EntityList.run(
         Cube,
         fields=["UID", "Name", "State", "Registered"],
-        local_only=local,
+        unregistered=unregistered,
         mine_only=mine,
     )
 
@@ -148,8 +150,10 @@ def view(
         "--format",
         help="Format to display contents. Available formats: [yaml, json]",
     ),
-    local: bool = typer.Option(
-        False, "--local", help="Display local mlcubes if mlcube ID is not provided"
+    unregistered: bool = typer.Option(
+        False,
+        "--unregistered",
+        help="Display unregistered mlcubes if mlcube ID is not provided",
     ),
     mine: bool = typer.Option(
         False,
@@ -164,4 +168,4 @@ def view(
     ),
 ):
     """Displays the information of one or more mlcubes"""
-    EntityView.run(entity_id, Cube, format, local, mine, output)
+    EntityView.run(entity_id, Cube, format, unregistered, mine, output)

cli/medperf/commands/result/create.py

@@ -1,5 +1,6 @@
 import os
 from typing import List, Optional
+from medperf.account_management.account_management import get_medperf_user_data
 from medperf.commands.execution import Execution
 from medperf.entities.result import Result
 from tabulate import tabulate
@@ -143,7 +144,9 @@ def __validate_models(self, benchmark_models):
             raise InvalidArgumentError(msg)
 
     def load_cached_results(self):
-        results = Result.all()
+        user_id = get_medperf_user_data()["id"]
+        results = Result.all(filters={"owner": user_id})
+        results += Result.all(unregistered=True)
         benchmark_dset_results = [
             result
             for result in results
@@ -254,7 +257,7 @@ def print_summary(self):
                 data_lists_for_display.append(
                     [
                         experiment["model_uid"],
-                        experiment["result"].generated_uid,
+                        experiment["result"].local_id,
                         experiment["result"].metadata["partial"],
                         experiment["cached"],
                         experiment["error"],

cli/medperf/commands/result/result.py

@@ -62,17 +62,19 @@ def submit(
 @app.command("ls")
 @clean_except
 def list(
-    local: bool = typer.Option(False, "--local", help="Get local results"),
+    unregistered: bool = typer.Option(
+        False, "--unregistered", help="Get unregistered results"
+    ),
     mine: bool = typer.Option(False, "--mine", help="Get current-user results"),
     benchmark: int = typer.Option(
         None, "--benchmark", "-b", help="Get results for a given benchmark"
     ),
 ):
-    """List results stored locally and remotely from the user"""
+    """List results"""
     EntityList.run(
         Result,
         fields=["UID", "Benchmark", "Model", "Dataset", "Registered"],
-        local_only=local,
+        unregistered=unregistered,
         mine_only=mine,
         benchmark=benchmark,
     )
@@ -88,8 +90,10 @@ def view(
         "--format",
         help="Format to display contents. Available formats: [yaml, json]",
     ),
-    local: bool = typer.Option(
-        False, "--local", help="Display local results if result ID is not provided"
+    unregistered: bool = typer.Option(
+        False,
+        "--unregistered",
+        help="Display unregistered results if result ID is not provided",
     ),
     mine: bool = typer.Option(
         False,
@@ -107,4 +111,6 @@ def view(
     ),
 ):
     """Displays the information of one or more results"""
-    EntityView.run(entity_id, Result, format, local, mine, output, benchmark=benchmark)
+    EntityView.run(
+        entity_id, Result, format, unregistered, mine, output, benchmark=benchmark
+    )