CI: Skip AzureML and Integration tests (#1212)

## Describe your changes Storage accounts are currently not accessible. Commenting out or skipping affected tests until a solution is found. ## Checklist before requesting a review - [ ] Add unit tests for this change. - [ ] Make sure all tests can pass. - [ ] Update documents if necessary. - [ ] Lint and apply fixes to your code by running `lintrunner -a` - [ ] Is this a user-facing change? If yes, give a description of this change to be included in the release notes. - [ ] Is this PR including examples changes? If yes, please remember to update [example documentation](https://github.com/microsoft/Olive/blob/main/docs/source/examples.md) in a follow-up PR. ## (Optional) Issue link
microsoft · Jun 27, 2024 · b59ef7d · b59ef7d
1 parent 7fa2c41
commit b59ef7d
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 39 deletions.
diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml
@@ -17,11 +17,11 @@ jobs:
     test_type: 'unit_test'
     onnxruntime: ${{parameters.onnxruntime}}
 
-- template: olive-test-template.yaml
-  parameters:
-    name: ${{parameters.name}}_Integration_Test_Olive
-    pool: ${{parameters.pool}}
-    device: ${{parameters.device}}
-    windows: ${{parameters.windows}}
-    test_type: 'integ_test'
-    onnxruntime: ${{parameters.onnxruntime}}
+# - template: olive-test-template.yaml
+#   parameters:
+#     name: ${{parameters.name}}_Integration_Test_Olive
+#     pool: ${{parameters.pool}}
+#     device: ${{parameters.device}}
+#     windows: ${{parameters.windows}}
+#     test_type: 'integ_test'
+#     onnxruntime: ${{parameters.onnxruntime}}
diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml
@@ -88,9 +88,9 @@ jobs:
       bert_ptq_cpu:
         exampleFolder: bert
         exampleName: bert_ptq_cpu
-      bert_ptq_cpu_aml:
-        exampleFolder: bert
-        exampleName: bert_ptq_cpu_aml
+      # bert_ptq_cpu_aml:
+      #   exampleFolder: bert
+      #   exampleName: bert_ptq_cpu_aml
       bert_ptq_cpu_docker:
         exampleFolder: bert
         exampleName: bert_ptq_cpu_docker
@@ -106,12 +106,12 @@ jobs:
       whisper:
         exampleFolder: whisper
         exampleName: whisper
-      mobilenet_qnn_toolkit:
-        exampleFolder: mobilenet
-        exampleName: qnn_tooklit
-      inception_snpe_toolkit:
-        exampleFolder: inception
-        exampleName: snpe_toolkit
+      # mobilenet_qnn_toolkit:
+      #   exampleFolder: mobilenet
+      #   exampleName: qnn_tooklit
+      # inception_snpe_toolkit:
+      #   exampleFolder: inception
+      #   exampleName: snpe_toolkit
       mobilenet_qnn_ep:
         exampleFolder: mobilenet
         exampleName: mobilenet_qnn_ep
@@ -126,9 +126,9 @@ jobs:
       bert_ptq_cpu:
         exampleFolder: bert
         exampleName: bert_ptq_cpu
-      bert_ptq_cpu_aml:
-        exampleFolder: bert
-        exampleName: bert_ptq_cpu_aml
+      # bert_ptq_cpu_aml:
+      #   exampleFolder: bert
+      #   exampleName: bert_ptq_cpu_aml
       resnet_ptq_cpu:
         exampleFolder: resnet
         exampleName: resnet_ptq_cpu
@@ -138,12 +138,12 @@ jobs:
       whisper:
         exampleFolder: whisper
         exampleName: whisper
-      mobilenet_qnn_toolkit:
-        exampleFolder: mobilenet
-        exampleName: qnn_tooklit
-      inception_snpe_toolkit:
-        exampleFolder: inception
-        exampleName: snpe_toolkit
+      # mobilenet_qnn_toolkit:
+      #   exampleFolder: mobilenet
+      #   exampleName: qnn_tooklit
+      # inception_snpe_toolkit:
+      #   exampleFolder: inception
+      #   exampleName: snpe_toolkit
       mobilenet_qnn_ep:
         exampleFolder: mobilenet
         exampleName: mobilenet_qnn_ep
@@ -174,16 +174,16 @@ jobs:
     onnxruntime: onnxruntime
     python_version: '3.8'
 
-# Multiple EP Windows testing
-- template: job_templates/olive-test-template.yaml
-  parameters:
-    name: Windows_CI_Multiple_EP_Test_Olive
-    pool: $(OLIVE_POOL_WIN2019)
-    device: 'cpu'
-    windows: 'True'
-    test_type: 'multiple_ep'
-    onnxruntime: onnxruntime
-    python_version: '3.10'
+# # Multiple EP Windows testing
+# - template: job_templates/olive-test-template.yaml
+#   parameters:
+#     name: Windows_CI_Multiple_EP_Test_Olive
+#     pool: $(OLIVE_POOL_WIN2019)
+#     device: 'cpu'
+#     windows: 'True'
+#     test_type: 'multiple_ep'
+#     onnxruntime: onnxruntime
+#     python_version: '3.10'
 
 # build documents
 - template: job_templates/olive-build-doc-template.yaml

diff --git a/.azure_pipelines/olive-ort-last.yaml b/.azure_pipelines/olive-ort-last.yaml
@@ -69,7 +69,9 @@ jobs:
       whisper:
         exampleFolder: whisper
         exampleName: whisper
-      # TODO(anyone): add mobilenet_qnn_ep when last version is 1.17.x
+      mobilenet_qnn_ep:
+        exampleFolder: mobilenet
+        exampleName: mobilenet_qnn_ep
 
 # # Windows examples test
 - template: job_templates/olive-example-template.yaml
@@ -90,7 +92,9 @@ jobs:
       whisper:
         exampleFolder: whisper
         exampleName: whisper
-      # TODO(anyone): add mobilenet_qnn_ep when last version is 1.17.x
+      mobilenet_qnn_ep:
+        exampleFolder: mobilenet
+        exampleName: mobilenet_qnn_ep
 
 # Linux GPU examples testing.
 - template: job_templates/olive-example-template.yaml

diff --git a/examples/test/test_bert_cuda_gpu.py b/examples/test/test_bert_cuda_gpu.py
@@ -27,6 +27,9 @@ def setup():
 def test_bert(search_algorithm, execution_order, system, olive_json, enable_cuda_graph):
     from olive.workflows import run as olive_run
 
+    if system == "aml_system":
+        pytest.skip("Skipping AML system test")
+
     olive_config = patch_config(olive_json, search_algorithm, execution_order, system, is_gpu=True)
     olive_config["passes"]["perf_tuning"]["config"]["enable_cuda_graph"] = enable_cuda_graph
 

diff --git a/examples/test/test_resnet_ptq_cpu.py b/examples/test/test_resnet_ptq_cpu.py
@@ -50,6 +50,9 @@ def test_resnet(search_algorithm, execution_order, system, olive_json):
     # TODO(jambayk): add gpu e2e test
     from olive.workflows import run as olive_run
 
+    if system == "aml_system":
+        pytest.skip("Skipping AML system test")
+
     olive_config = patch_config(olive_json, search_algorithm, execution_order, system)
 
     footprint = olive_run(olive_config)

diff --git a/examples/test/test_resnet_vitis_ai_ptq_cpu.py b/examples/test/test_resnet_vitis_ai_ptq_cpu.py
@@ -32,6 +32,9 @@ def test_resnet(system, olive_json):
     # TODO(jambayk): add gpu e2e test
     from olive.workflows import run as olive_run
 
+    if system == "aml_system":
+        pytest.skip("Skipping AML system test")
+
     olive_config = patch_config(olive_json, None, None, system)
 
     footprint = olive_run(olive_config)

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -5,5 +5,5 @@ flake8
 isort
 lintrunner
 lintrunner-adapters
-pylint
+pylint<3.2.4 # https://github.com/pylint-dev/pylint/issues/9751
 ruff>=0.2.0
diff --git a/test/multiple_ep/test_aml_system.py b/test/multiple_ep/test_aml_system.py
@@ -13,6 +13,7 @@
 # pylint: disable=attribute-defined-outside-init
 
 
+@pytest.mark.skip(reason="Skip test on AzureML.")
 @pytest.mark.skipif(platform.system() == OS.WINDOWS, reason="Skip test on Windows.")
 class TestOliveAzureMLSystem:
     @pytest.fixture(autouse=True)

diff --git a/test/multiple_ep/test_docker_system.py b/test/multiple_ep/test_docker_system.py
@@ -15,6 +15,7 @@
 # pylint: disable=attribute-defined-outside-init
 
 
+@pytest.mark.skip(reason="Skip test as it downloads resources from Azure storage.")
 @pytest.mark.skipif(platform.system() == OS.WINDOWS, reason="Docker target does not support windows")
 class TestOliveManagedDockerSystem:
     @pytest.fixture(autouse=True)