V2 Performance Signal Detected by TorchBench CI on '2.5.0.dev20240821+cu124'

[xuzhao9]

TorchBench CI has detected a performance signal.

Base PyTorch version: 2.4.0.dev20240606+cu121

Base PyTorch commit: ffaea656b5d8ff6518669494cc8f664b94f8e8b1

Affected PyTorch version: 2.5.0.dev20240821+cu124

Affected PyTorch commit: c42ac54d9e817bf0a0366eb78e6c8beba4d5eff5

Affected Tests:

• test_train[Super_SloMo-cuda-eager]: +14.63890%
• test_eval[Super_SloMo-cuda-eager]: +11.05591%
• test_train[Super_SloMo-cuda-jit]: +14.75525%
• test_eval[Super_SloMo-cuda-jit]: +11.21743%
• test_eval[alexnet-cuda-eager]: +10.29962%
• test_eval[alexnet-cuda-jit]: +13.76529%
• test_train[dcgan-cuda-eager]: +63.51934%
• test_train[dcgan-cuda-jit]: +63.61128%
• test_eval[pyhpc_equation_of_state-cpu-jit]: +12.79470%
• test_eval[pytorch_stargan-cpu-eager]: -9.34371%
• test_eval[pytorch_stargan-cpu-jit]: -10.78782%
• test_eval[shufflenet_v2_x1_0-cuda-jit]: -27.37628%
• test_train[timm_nfnet-cpu-jit]: -8.82225%
• test_eval[timm_regnet-cuda-eager]: +169.70704%
• test_eval[timm_regnet-cuda-jit]: +354.10665%
• test_train[yolov3-cuda-eager]: -9.81313%

cc @xuzhao9

Result json:

{
  "start": "ffaea656b5d8ff6518669494cc8f664b94f8e8b1",
  "end": "c42ac54d9e817bf0a0366eb78e6c8beba4d5eff5",
  "threshold": 7,
  "timeout": 120,
  "torchbench_branch": "v2.0",
  "result": [
    {
      "commit1": "548c460bf1f",
      "commit1_time": "2024-07-31 10:54:03 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09728763541206717
      },
      "commit2": "f4f7aba75d6",
      "commit2_time": "2024-07-31 11:33:09 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06987479254603386
      }
    },
    {
      "commit1": "69cbf055293",
      "commit1_time": "2024-08-01 17:09:42 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07111594118177891
      },
      "commit2": "7b816d7d6d5",
      "commit2_time": "2024-08-01 17:11:18 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0875588702969253
      }
    },
    {
      "commit1": "e7eeee473c6",
      "commit1_time": "2024-07-31 10:42:50 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08563343547284603
      },
      "commit2": "89053e382a6",
      "commit2_time": "2024-07-31 10:51:46 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.11059268023818732
      }
    },
    {
      "commit1": "89053e382a6",
      "commit1_time": "2024-07-31 10:51:46 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.11059268023818732
      },
      "commit2": "46994e753bf",
      "commit2_time": "2024-07-31 10:51:46 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08336173947900534
      }
    },
    {
      "commit1": "46994e753bf",
      "commit1_time": "2024-07-31 10:51:46 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08336173947900534
      },
      "commit2": "548c460bf1f",
      "commit2_time": "2024-07-31 10:54:03 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09728763541206717
      }
    },
    {
      "commit1": "f4f7aba75d6",
      "commit1_time": "2024-07-31 11:33:09 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06987479254603386
      },
      "commit2": "45e6a364ee1",
      "commit2_time": "2024-07-31 13:13:39 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08149707010015846
      }
    },
    {
      "commit1": "bdd7a0322d8",
      "commit1_time": "2024-07-31 16:39:59 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08201208543032408
      },
      "commit2": "9826c542f0b",
      "commit2_time": "2024-07-31 16:41:04 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07365901088342071
      }
    },
    {
      "commit1": "9826c542f0b",
      "commit1_time": "2024-07-31 16:41:04 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07365901088342071
      },
      "commit2": "784a6ec5a30",
      "commit2_time": "2024-07-31 16:49:21 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09859710223972798
      }
    },
    {
      "commit1": "c3a31d90e7d",
      "commit1_time": "2024-07-31 17:37:43 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0970154045149684
      },
      "commit2": "7eb2a99585b",
      "commit2_time": "2024-07-31 17:51:03 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0801084939390421
      }
    },
    {
      "commit1": "043e41f4f47",
      "commit1_time": "2024-08-01 07:02:35 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08118315152823925
      },
      "commit2": "bdd83c4c7f1",
      "commit2_time": "2024-08-01 07:28:52 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06543507492169738
      }
    },
    {
      "commit1": "bdd83c4c7f1",
      "commit1_time": "2024-08-01 07:28:52 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06543507492169738
      },
      "commit2": "d95756f6a5a",
      "commit2_time": "2024-08-01 09:57:43 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07576586417853833
      }
    },
    {
      "commit1": "d7d61904936",
      "commit1_time": "2024-08-01 14:46:33 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07744492432102561
      },
      "commit2": "0d88dd0f779",
      "commit2_time": "2024-08-01 15:08:02 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06758335959166288
      }
    },
    {
      "commit1": "0d88dd0f779",
      "commit1_time": "2024-08-01 15:08:02 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06758335959166288
      },
      "commit2": "c99adce9a15",
      "commit2_time": "2024-08-01 15:12:12 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08533393973484635
      }
    },
    {
      "commit1": "a6985c09cbc",
      "commit1_time": "2024-08-01 15:26:45 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08478634031489492
      },
      "commit2": "221350e3a49",
      "commit2_time": "2024-08-01 15:44:51 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07842492377385497
      }
    },
    {
      "commit1": "920f0426ae5",
      "commit1_time": "2024-08-01 15:44:51 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07829177314415574
      },
      "commit2": "35fcd59fd81",
      "commit2_time": "2024-08-01 16:03:10 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09558446686714887
      }
    },
    {
      "commit1": "c59f3fff52d",
      "commit1_time": "2024-08-01 16:35:56 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09270042590796948
      },
      "commit2": "30293319a8f",
      "commit2_time": "2024-08-01 17:07:14 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08437996199354529
      }
    },
    {
      "commit1": "30293319a8f",
      "commit1_time": "2024-08-01 17:07:14 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08437996199354529
      },
      "commit2": "69cbf055293",
      "commit2_time": "2024-08-01 17:09:42 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07111594118177891
      }
    },
    {
      "commit1": "ff4ca0d02a1",
      "commit1_time": "2024-08-01 17:13:37 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09223348516970872
      },
      "commit2": "010fc7858ad",
      "commit2_time": "2024-08-01 17:22:04 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08074744027107954
      }
    },
    {
      "commit1": "1362d51e7d4",
      "commit1_time": "2024-08-01 22:43:28 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0826474622823298
      },
      "commit2": "87ddf70fc63",
      "commit2_time": "2024-08-01 23:25:07 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10644191401079298
      }
    },
    {
      "commit1": "87ddf70fc63",
      "commit1_time": "2024-08-01 23:25:07 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10644191401079298
      },
      "commit2": "0c3ac428a29",
      "commit2_time": "2024-08-01 23:34:33 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06442798301577568,
        "test_train[timm_nfnet-cpu-jit]": 11.29581815013662
      }
    }
  ]
}

Bisection workflow link: https://github.com/pytorch/benchmark/actions/runs/10505721095