Remove conda install for aarch64 validation (pytorch#1757)

* Remove conda install for aarch64

* test

* Revert "test"

This reverts commit 6add257.

* fix_aarch64

* Increase timeout for aarch64 jobs

* test

.github/scripts/validate_binaries.sh

@@ -5,7 +5,7 @@ else
 
     if [[ ${TARGET_OS} == 'macos-arm64' ]]; then
         conda update -y -n base -c defaults conda
-    else
+    elif [[ ${TARGET_OS} != 'linux-aarch64' ]]; then
         # Conda pinned see issue: https://github.com/ContinuumIO/anaconda-issues/issues/13350
         conda install -y conda=23.11.0
     fi
@@ -69,6 +69,10 @@ else
         source ./.github/scripts/validate_test_ops.sh
     fi
 
-    conda deactivate
-    conda env remove -n ${ENV_NAME}
+    # TODO: remove if statement currently this step is timing out on linx-aarch64
+    if [[ ${TARGET_OS} != 'linux-aarch64' ]]; then
+        conda deactivate
+        conda env remove -n ${ENV_NAME}
+    fi
+
 fi