Skip to content

Commit

Permalink
Improved integration tests (#575)
Browse files Browse the repository at this point in the history 
* Improved tests

now config.yaml location can be defined with custom value (env var)
integration tests use a new /tmp/ folder storage with every run
After every command db is backed up to /tmp/%dir%/snapshots/ for easy debug
Every command is printed before execution

* debug

* Debug

* debug

* debug

* debug

* Fix: reodered folder creation and clean on -f

* import os breaks pytest

* Renamed `ev` to `print_eval`

* Instead of real storage moving just set up config properly

* bugfix

* bugfix

* bugfix

* bugfix

* Covered multisource tests also

---------

Co-authored-by: hasan7n <[email protected]>
  • Loading branch information
@VukW @hasan7n
VukW and hasan7n committed May 20, 2024
1 parent 363a22f commit 9bb807b
Show file tree
Hide file tree
Showing 7 changed files with 246 additions and 191 deletions.
59 changes: 31 additions & 28 deletions cli/cli_chestxray_tutorial_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,42 +13,45 @@ wget -P $DIRECTORY https://storage.googleapis.com/medperf-storage/chestxray_tuto
tar -xzvf $DIRECTORY/sample_raw_data.tar.gz -C $DIRECTORY
chmod a+w $DIRECTORY/sample_raw_data

##########################################################


echo "=========================================="
echo "Creating test profiles for each user"
echo "=========================================="
medperf profile activate local
print_eval medperf profile activate local
checkFailed "local profile creation failed"

medperf profile create -n testbenchmark
print_eval medperf profile create -n testbenchmark
checkFailed "testbenchmark profile creation failed"
medperf profile create -n testdata
print_eval medperf profile create -n testdata
checkFailed "testdata profile creation failed"

echo "=========================================="
echo "Login each user"
echo "=========================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth login -e $BENCHMARKOWNER
print_eval medperf auth login -e $BENCHMARKOWNER
checkFailed "testbenchmark login failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth login -e $DATAOWNER
print_eval medperf auth login -e $DATAOWNER
checkFailed "testdata login failed"

echo "====================================="
echo "Activate dataowner profile"
echo "====================================="
medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"
echo "\n"
echo "====================================="
echo "Registering dataset with medperf"
echo "====================================="
medperf dataset submit -b 1 -d $DIRECTORY/sample_raw_data/images -l $DIRECTORY/sample_raw_data/labels --name="nih_chestxray" --description="sample dataset" --location="mock location" -y
print_eval "medperf dataset submit -b 1 -d $DIRECTORY/sample_raw_data/images -l $DIRECTORY/sample_raw_data/labels --name='nih_chestxray' --description='sample dataset' --location='mock location' -y"
checkFailed "Data registration step failed"

echo "\n"
Expand All @@ -57,86 +60,86 @@ echo "Dataset UID: $DSET_UID"
echo "====================================="
echo "Running data preparation step"
echo "====================================="
medperf dataset prepare -d $DSET_UID
print_eval medperf dataset prepare -d $DSET_UID
checkFailed "Data preparation step failed"

echo "\n"

echo "====================================="
echo "Running data set operational step"
echo "====================================="
medperf dataset set_operational -d $DSET_UID -y
print_eval medperf dataset set_operational -d $DSET_UID -y
checkFailed "Data set operational step failed"

echo "====================================="
echo "Creating dataset benchmark association"
echo "====================================="
medperf dataset associate -d $DSET_UID -b 1 -y
print_eval medperf dataset associate -d $DSET_UID -b 1 -y
checkFailed "Data association step failed"

echo "====================================="
echo ""Activate benchmarkowner profile""
echo "====================================="
# Log in as the benchmark owner
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"
# Get association information
ASSOC_INFO=$(medperf association ls | head -n 4 | tail -n 1 | tr -s ' ')
ASSOC_DSET_UID=$(echo $ASSOC_INFO | cut -d ' ' -f 1)
ASSOC_BMK_UID=$(echo $ASSOC_INFO | cut -d ' ' -f 2)
# Mark dataset-benchmark association as approved
medperf association approve -b $ASSOC_BMK_UID -d $ASSOC_DSET_UID
print_eval medperf association approve -b $ASSOC_BMK_UID -d $ASSOC_DSET_UID
checkFailed "Association approval failed"

echo "====================================="
echo "Running benchmark execution step"
echo "====================================="
# log back as user
medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"
# Create results
medperf run -b 1 -d $DSET_UID -m 4 -y
print_eval medperf run -b 1 -d $DSET_UID -m 4 -y
checkFailed "Benchmark execution step failed"

# Test offline compatibility test
wget -P $MODEL_LOCAL/workspace/additional_files "https://storage.googleapis.com/medperf-storage/chestxray_tutorial/cnn_weights.tar.gz"
tar -xzvf $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz -C $MODEL_LOCAL/workspace/additional_files
medperf test run --offline --no-cache \
print_eval wget -P $MODEL_LOCAL/workspace/additional_files "https://storage.googleapis.com/medperf-storage/chestxray_tutorial/cnn_weights.tar.gz"
print_eval tar -xzvf $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz -C $MODEL_LOCAL/workspace/additional_files
print_eval medperf test run --offline --no-cache \
--demo_dataset_url https://storage.googleapis.com/medperf-storage/chestxray_tutorial/demo_data.tar.gz \
--demo_dataset_hash "71faabd59139bee698010a0ae3a69e16d97bc4f2dde799d9e187b94ff9157c00" \
-p $PREP_LOCAL \
-m $MODEL_LOCAL \
-e $METRIC_LOCAL

checkFailed "offline compatibility test execution step failed"
rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz
rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.pth
print_eval rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz
print_eval rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.pth

echo "====================================="
echo "Logout users"
echo "====================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

echo "====================================="
echo "Delete test profiles"
echo "====================================="
medperf profile activate default
print_eval medperf profile activate default
checkFailed "default profile activation failed"

medperf profile delete testbenchmark
print_eval medperf profile delete testbenchmark
checkFailed "Profile deletion failed"

medperf profile delete testdata
print_eval medperf profile delete testdata
checkFailed "Profile deletion failed"

if ${CLEANUP}; then
Expand Down
72 changes: 38 additions & 34 deletions cli/cli_test_manual_prep.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
echo "=========================================="
echo "Printing MedPerf version"
echo "=========================================="
medperf --version
print_eval medperf --version
checkFailed "MedPerf version failed"
##########################################################

Expand All @@ -20,14 +20,14 @@ echo "\n"
echo "=========================================="
echo "Creating test profiles for each user"
echo "=========================================="
medperf profile activate local
print_eval medperf profile activate local
checkFailed "local profile creation failed"

medperf profile create -n testbenchmark
print_eval medperf profile create -n testbenchmark
checkFailed "testbenchmark profile creation failed"
medperf profile create -n testmodel
print_eval medperf profile create -n testmodel
checkFailed "testmodel profile creation failed"
medperf profile create -n testdata
print_eval medperf profile create -n testdata
checkFailed "testdata profile creation failed"
##########################################################

Expand All @@ -51,22 +51,22 @@ echo "\n"
echo "=========================================="
echo "Login each user"
echo "=========================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth login -e $BENCHMARKOWNER
print_eval medperf auth login -e $BENCHMARKOWNER
checkFailed "testbenchmark login failed"

medperf profile activate testmodel
print_eval medperf profile activate testmodel
checkFailed "testmodel profile activation failed"

medperf auth login -e $MODELOWNER
print_eval medperf auth login -e $MODELOWNER
checkFailed "testmodel login failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth login -e $DATAOWNER
print_eval medperf auth login -e $DATAOWNER
checkFailed "testdata login failed"
##########################################################

Expand All @@ -76,7 +76,7 @@ echo "\n"
echo "====================================="
echo "Activate modelowner profile"
echo "====================================="
medperf profile activate testmodel
print_eval medperf profile activate testmodel
checkFailed "testmodel profile activation failed"
##########################################################

Expand All @@ -89,9 +89,10 @@ echo "====================================="

PREP_MLCUBE="https://raw.githubusercontent.com/aristizabal95/medperf-2/4aea7de62fd71b377fd3a0b58352d104fd8f9c08/examples/DataPrepManualSteps/data_prep/mlcube/mlcube.yaml"
PREP_PARAMS="https://raw.githubusercontent.com/aristizabal95/medperf-2/4aea7de62fd71b377fd3a0b58352d104fd8f9c08/examples/DataPrepManualSteps/data_prep/mlcube/workspace/parameters.yaml"
medperf mlcube submit --name manprep -m $PREP_MLCUBE -p $PREP_PARAMS
print_eval medperf mlcube submit --name manprep -m $PREP_MLCUBE -p $PREP_PARAMS
checkFailed "Prep submission failed"
PREP_UID=$(medperf mlcube ls | tail -n 1 | tr -s ' ' | cut -d ' ' -f 2)
echo "PREP_UID=$PREP_UID"

##########################################################

Expand All @@ -101,7 +102,7 @@ echo "\n"
echo "====================================="
echo "Activate dataowner profile"
echo "====================================="
medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"
##########################################################

Expand All @@ -111,9 +112,11 @@ echo "\n"
echo "====================================="
echo "Running data creation step"
echo "====================================="
medperf dataset submit -p $PREP_UID -d $DATA_PATH -l $LABELS_PATH --name="manual_a" --description="mock manual a" --location="mock location a" -y
print_eval "medperf dataset submit -p $PREP_UID -d $DATA_PATH -l $LABELS_PATH --name='manual_a' --description='mock manual a' --location='mock location a' -y"
checkFailed "Data submission step failed"
DSET_A_UID=$(medperf dataset ls | grep manual_a | tr -s ' ' | cut -d ' ' -f 2)
echo "DSET_A_UID=$DSET_A_UID"

##########################################################

echo "\n"
Expand All @@ -122,7 +125,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step (it will fail, needs manual steps)"
echo "====================================="
medperf dataset prepare -d $DSET_A_UID -y
print_eval medperf dataset prepare -d $DSET_A_UID -y
checkSucceeded "Data preparation step should fail"
##########################################################

Expand All @@ -132,7 +135,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step (another way)"
echo "====================================="
echo "y" | medperf dataset prepare -d $DSET_A_UID
print_eval echo "y" | medperf dataset prepare -d $DSET_A_UID
checkSucceeded "Data preparation step should fail"
##########################################################

Expand All @@ -142,7 +145,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step (another way)"
echo "====================================="
echo "n" | medperf dataset prepare -d $DSET_A_UID
print_eval echo "n" | medperf dataset prepare -d $DSET_A_UID
checkSucceeded "Data preparation step should fail"
##########################################################

Expand All @@ -152,7 +155,7 @@ echo "\n"
echo "====================================="
echo "Do manual step of the preparation"
echo "====================================="
sed -i 's/0$/1/' $MEDPERF_STORAGE/data/$SERVER_STORAGE_ID/$DSET_A_UID/data/data.csv
print_eval "sed -i 's/0$/1/' $MEDPERF_STORAGE/data/$SERVER_STORAGE_ID/$DSET_A_UID/data/data.csv"
checkFailed "manual step failed"
##########################################################

Expand All @@ -162,7 +165,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step again (this will succeed)"
echo "====================================="
medperf dataset prepare -d $DSET_A_UID -y
print_eval medperf dataset prepare -d $DSET_A_UID -y
checkFailed "Data preparation step failed"
##########################################################

Expand All @@ -172,7 +175,7 @@ echo "\n"
echo "====================================="
echo "Running data set operational step"
echo "====================================="
medperf dataset set_operational -d $DSET_A_UID -y
print_eval medperf dataset set_operational -d $DSET_A_UID -y
checkFailed "Data activattion step failed"
##########################################################

Expand All @@ -182,9 +185,10 @@ echo "\n"
echo "====================================="
echo "Running data creation step"
echo "====================================="
medperf dataset submit -p $PREP_UID -d $PREPARED_DATA_PATH -l $PREPARED_LABELS_PATH --name="already_a" --description="mock already a" --location="mock location a" -y --submit-as-prepared
print_eval "medperf dataset submit -p $PREP_UID -d $PREPARED_DATA_PATH -l $PREPARED_LABELS_PATH --name='already_a' --description='mock already a' --location='mock location a' -y --submit-as-prepared"
checkFailed "Data submission step failed"
DSET_A_UID=$(medperf dataset ls | grep already_a | tr -s ' ' | cut -d ' ' -f 2)
echo "DSET_A_UID=$DSET_A_UID"
##########################################################

echo "\n"
Expand All @@ -193,7 +197,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step"
echo "====================================="
medperf dataset prepare -d $DSET_A_UID
print_eval medperf dataset prepare -d $DSET_A_UID
checkFailed "Data preparation step failed"
##########################################################

Expand All @@ -203,7 +207,7 @@ echo "\n"
echo "====================================="
echo "Running data set operational step"
echo "====================================="
medperf dataset set_operational -d $DSET_A_UID -y
print_eval medperf dataset set_operational -d $DSET_A_UID -y
checkFailed "Data activattion step failed"
##########################################################

Expand All @@ -213,22 +217,22 @@ echo "\n"
echo "====================================="
echo "Logout users"
echo "====================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

medperf profile activate testmodel
print_eval medperf profile activate testmodel
checkFailed "testmodel profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"
##########################################################

Expand All @@ -238,16 +242,16 @@ echo "\n"
echo "====================================="
echo "Delete test profiles"
echo "====================================="
medperf profile activate default
print_eval medperf profile activate default
checkFailed "default profile activation failed"

medperf profile delete testbenchmark
print_eval medperf profile delete testbenchmark
checkFailed "Profile deletion failed"

medperf profile delete testmodel
print_eval medperf profile delete testmodel
checkFailed "Profile deletion failed"

medperf profile delete testdata
print_eval medperf profile delete testdata
checkFailed "Profile deletion failed"
##########################################################

Expand Down
Loading

0 comments on commit 9bb807b

Please sign in to comment.