Merge pull request #40 from karlaspuldaro/client-update-results

Process new results and fix CI run
qiskit-community · Dec 5, 2023 · 97c91c8 · 97c91c8
2 parents e72c645 + f8dbb23
commit 97c91c8
Show file tree

Hide file tree

Showing 16 changed files with 560 additions and 318 deletions.
diff --git a/.github/workflows/run_experiments.yml b/.github/workflows/run_experiments.yml
@@ -1,4 +1,4 @@
-name: Check for new qiskit versions, run benchmark experiments and submit results to metriq.info
+name: Run benchmark experiments on new Qiskit versions and submit results to Metriq.info
 
 on:
   schedule:
@@ -13,47 +13,46 @@ jobs:
 
     env:
       METRIQ_TOKEN: ${{ secrets.METRIQ_TOKEN}}
+      BOT_ACCESS_TOKEN: ${{ secrets.BOT_ACCESS_TOKEN }}
 
     steps:
-      - name: Setup repository on this workflow
+      - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
-
-      - name: Install and configure Github CLI
-        uses: cli/cli/setup@v2
-        with:
-          gh-version: 2
-
-      - name: Install tox
-        run: pip install tox 
+          python-version: 3.11
 
-      - name: Run experiment steps in tox
-        run: tox -ve py38
+      - name: Install tox
+        run: pip install tox
 
-      - name: Configure Git with Bot Git Config
-        run: |
-          git config user.name "${{ secrets.BOT_USERNAME }}"
-          git config user.email "${{ secrets.BOT_EMAIL }}"
+      - name: Run experiments in tox
+        run: tox -ve py311
 
-      - name: Check for updates on result files and open new PR with Bot user
+      - name: Check for new artifacts
         run: |
-          untracked_csv_files=$(find ./benchmarking/results/ -name '*.csv' -type f)
-          deleted_csv_files=$(git diff --name-only HEAD@{1} HEAD | grep -E '^../results/.*\.csv')
-          if [ -n "$untracked_csv_files" ] || [ -n "deleted_csv_files"]; then
-            git checkout -b bot-branch
-            git add $untracked_csv_files
-            git rm $deleted_csv_files
-            git commit -m "Add new result files"
-            git push --set-upstream origin bot-branch
-            gh pr create --base main --head bot-branch --title "Add new result files"
-            git checkout main
+          cd benchmarking/results
+          if git status --porcelain | grep -E "\.csv$" -q; then
+            echo "New result files found"
+            echo "changed_csv_files=1" >> $GITHUB_ENV
+            NEW_BRANCH="automated-branch"
+            echo "NEW_BRANCH=$NEW_BRANCH" >> $GITHUB_ENV
+          else
+            echo "No result files to process"
+            echo "changed_csv_files=0" >> $GITHUB_ENV
           fi
-        env:
-          BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
-          BOT_EMAIL: ${{ secrets.BOT_EMAIL }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN}}
+          cd ../../
+      
+      - name: Create Pull Request for new artifacts
+        if: env.changed_csv_files == '1'
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+          commit-message: Update results
+          title: Update results
+          base: main
+          branch: ${{ env.NEW_BRANCH }}
+          delete-branch: true
+
 
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 __pycache__/
 *.egg-info/
-.tox/
+.tox/
+.DS_Store
diff --git a/README.md b/README.md
@@ -33,7 +33,7 @@ Compiled for the IBMQ Rochester architecture
 
 ## Requirements
 * [tox](https://pypi.org/project/tox/)
-* Python 3.8+
+* Python 3.11+
 
 ## Run locally
 ### To run a benchmark experiment using the current stable version of `Qiskit`:

diff --git a/benchmarking/processed_data_summary.json b/benchmarking/processed_data_summary.json
@@ -408,7 +408,7 @@
         ]
     },
     {
-        "0.44.1-ibm_rochester": [
+        "0.25.3-ibm_rochester": [
             {
                 "Circuit depth": {
                     "ave": 10.4,
@@ -424,7 +424,7 @@
                 }
             }
         ],
-        "0.44.1-rigetti_16q_aspen": [
+        "0.25.3-rigetti_16q_aspen": [
             {
                 "Circuit depth": {
                     "ave": 9.14,
@@ -440,5 +440,39 @@
                 }
             }
         ]
+    },
+    {
+        "0.45.0-ibm_rochester": [
+            {
+                "Circuit depth": {
+                    "ave": 10.28,
+                    "stdev": 1.538,
+                    "stderr": 0.154
+                }
+            },
+            {
+                "Gate count": {
+                    "ave": 16.13,
+                    "stdev": 2.465,
+                    "stderr": 0.246
+                }
+            }
+        ],
+        "0.45.0-rigetti_16q_aspen": [
+            {
+                "Circuit depth": {
+                    "ave": 9.09,
+                    "stdev": 1.232,
+                    "stderr": 0.123
+                }
+            },
+            {
+                "Gate count": {
+                    "ave": 15.39,
+                    "stdev": 2.77,
+                    "stderr": 0.277
+                }
+            }
+        ]
     }
 ]
diff --git a/benchmarking/results/ex1_226-qiskit0.25.3-ibm_rochester.csv b/benchmarking/results/ex1_226-qiskit0.25.3-ibm_rochester.csv
@@ -0,0 +1,101 @@
+|Qasm file|Method|Date|Opt level|Platform|Seed|Circuit depth|Gate count
+0|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|0|11|15
+1|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|1|11|16
+2|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|2|9|17
+3|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|3|8|12
+4|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|4|11|16
+5|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|5|10|15
+6|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|6|11|15
+7|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|7|13|20
+8|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|8|9|13
+9|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|9|10|17
+10|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|10|11|15
+11|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|11|10|17
+12|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|12|9|13
+13|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|13|9|12
+14|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|14|11|17
+15|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|15|12|22
+16|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|16|11|17
+17|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|17|11|16
+18|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|18|12|20
+19|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|19|10|16
+20|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|20|11|18
+21|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|21|8|12
+22|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|22|9|17
+23|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|23|10|17
+24|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|24|11|18
+25|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|25|11|16
+26|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|26|10|17
+27|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|27|10|16
+28|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|28|13|19
+29|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|29|13|19
+30|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|30|11|17
+31|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|31|11|15
+32|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|32|9|13
+33|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|33|11|16
+34|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|34|10|17
+35|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|35|13|21
+36|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|36|11|16
+37|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|37|10|16
+38|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|38|13|22
+39|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|39|11|17
+40|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|40|11|16
+41|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|41|10|17
+42|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|42|10|17
+43|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|43|9|13
+44|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|44|13|18
+45|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|45|8|13
+46|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|46|13|18
+47|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|47|11|17
+48|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|48|9|17
+49|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|49|11|18
+50|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|50|8|12
+51|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|51|11|15
+52|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|52|8|12
+53|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|53|10|16
+54|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|54|7|13
+55|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|55|10|17
+56|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|56|11|16
+57|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|57|10|16
+58|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|58|9|17
+59|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|59|13|19
+60|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|60|9|13
+61|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|61|11|15
+62|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|62|10|17
+63|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|63|10|17
+64|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|64|10|17
+65|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|65|9|12
+66|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|66|11|15
+67|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|67|10|15
+68|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|68|9|17
+69|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|69|10|17
+70|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|70|8|12
+71|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|71|9|17
+72|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|72|11|16
+73|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|73|10|17
+74|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|74|9|12
+75|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|75|11|17
+76|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|76|11|17
+77|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|77|8|12
+78|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|78|11|16
+79|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|79|11|15
+80|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|80|9|17
+81|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|81|12|21
+82|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|82|10|15
+83|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|83|9|17
+84|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|84|11|17
+85|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|85|13|20
+86|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|86|12|21
+87|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|87|11|16
+88|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|88|13|19
+89|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|89|11|16
+90|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|90|11|16
+91|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|91|9|17
+92|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|92|13|19
+93|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|93|13|22
+94|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|94|13|19
+95|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|95|9|12
+96|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|96|9|17
+97|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|97|9|17
+98|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|98|7|12
+99|ex1_226.qasm|Qiskit 0.25.3 compilation|2023-10-25|3|ibm_rochester|99|11|15