-
Notifications
You must be signed in to change notification settings - Fork 32
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add performance check for mscclpp-test (#110)
- Add ndmv4 perf baseline - change mscclpp-test to output perf number into a json file - add python script to check the perf result with the baseline
- Loading branch information
1 parent
cd7797f
commit 2640578
Showing
16 changed files
with
295 additions
and
130 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| [tool.black] | ||
| line-length = 120 | ||
| target-version = ['py38'] | ||
| include = '\.pyi?$' | ||
| extend-exclude = 'python/' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| {"name":"allgather", "kernel":1, "ranks":8, "ranksPerNode":8, "algBw":271.83, "busBw":237.85, "size":1073741824, "time":3949.94, "target":"throughput"} | ||
| {"name":"allgather", "kernel":2, "ranks":16,"ranksPerNode":8, "algBw":243.86, "busBw":228.62, "size":3221225472, "time":13209.19,"target":"throughput"} | ||
| {"name":"allgather", "kernel":3, "ranks":8, "ranksPerNode":8, "algBw":0.1133, "busBw":0.1016, "size":8192, "time":72.88, "target":"latency"} | ||
| {"name":"allreduce", "kernel":1, "ranks":8, "ranksPerNode":8, "algBw":139.04, "busBw":243.32, "size":1073741824, "time":7722.32, "target":"throughput"} | ||
| {"name":"allreduce", "kernel":2, "ranks":8, "ranksPerNode":8, "algBw":1.40, "busBw":2.45, "size":8192, "time":5.86, "target":"latency"} | ||
| {"name":"alltoall", "kernel":0, "ranks":16,"ranksPerNode":8, "algBw":46.49, "busBw":43.5928,"size":1073741824, "time":23091.7, "target":"throughput"} | ||
| {"name":"alltoall", "kernel":1, "ranks":8, "ranksPerNode":8, "algBw":275.54, "busBw":241.10, "size":1073741824, "time":3896.75, "target":"throughput"} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| # Copyright (c) Microsoft Corporation. | ||
| # Licensed under the MIT License. | ||
|
|
||
| import json | ||
| import logging | ||
|
|
||
|
|
||
| def load_perf_file(perf_fine: str) -> dict: | ||
| res = {} | ||
| with open(perf_fine, "r") as f: | ||
| for line in f: | ||
| data = json.loads(line) | ||
| res[(data["name"], data["kernel"], data["ranks"], data["ranksPerNode"], data["size"])] = { | ||
| "algBw": data["algBw"], | ||
| "busBw": data["busBw"], | ||
| "time": data["time"], | ||
| } | ||
| if "target" in data: | ||
| res[(data["name"], data["kernel"], data["ranks"], data["ranksPerNode"], data["size"])]["target"] = data[ | ||
| "target" | ||
| ] | ||
| return res | ||
|
|
||
|
|
||
| def check_perf_result(perf_result: dict, baseline: dict, time_threshold: float, bandwidth_threshold: float) -> bool: | ||
| res = True | ||
| for key, value in perf_result.items(): | ||
| if key not in baseline: | ||
| continue | ||
| if baseline[key]["target"] == "latency": | ||
| if abs(value["time"] - baseline[key]["time"]) / baseline[key]["time"] > time_threshold: | ||
| logging.error( | ||
| "%s: time %f not match baseline %f with threshold %f", | ||
| str(key), | ||
| value["time"], | ||
| baseline[key]["time"], | ||
| time_threshold, | ||
| ) | ||
| res = False | ||
| elif baseline[key]["target"] == "throughput": | ||
| if abs(value["algBw"] - baseline[key]["algBw"]) / baseline[key]["algBw"] > bandwidth_threshold: | ||
| logging.error( | ||
| "%s: algBw %f not match baseline %f with threshold %f", | ||
| str(key), | ||
| value["algBw"], | ||
| baseline[key]["algBw"], | ||
| bandwidth_threshold, | ||
| ) | ||
| res = False | ||
| if abs(value["busBw"] - baseline[key]["busBw"]) / baseline[key]["busBw"] > bandwidth_threshold: | ||
| logging.error( | ||
| "%s: busBw %f not match baseline %f with threshold %f", | ||
| str(key), | ||
| value["busBw"], | ||
| baseline[key]["busBw"], | ||
| bandwidth_threshold, | ||
| ) | ||
| res = False | ||
| return res | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| import argparse | ||
|
|
||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument("--perf-file", type=str, required=True) | ||
| parser.add_argument("--baseline-file", type=str, required=True) | ||
| # We use different threshold for latency and bandwidth. For latency, | ||
| # small data size is used which introduces more variance. For bandwidth, the performance is more stable. | ||
| parser.add_argument("--time-threshold", type=float, default=0.15) | ||
| parser.add_argument("--bandwidth-threshold", type=float, default=0.05) | ||
| args = parser.parse_args() | ||
|
|
||
| perf_result = load_perf_file(args.perf_file) | ||
| baseline = load_perf_file(args.baseline_file) | ||
| if check_perf_result(perf_result, baseline, args.time_threshold, args.bandwidth_threshold): | ||
| print("PASS") | ||
| else: | ||
| print("FAIL") | ||
| exit(1) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.