From 09959a51274d92ebe70add5cb9922126c0fee41c Mon Sep 17 00:00:00 2001 From: Srikanth Myakam Date: Sat, 30 Mar 2024 15:03:40 +0530 Subject: [PATCH 1/3] 'io_uring' support to LISA Added 'io_uring' support to LISA io perf tests and new testcase 'perf_nvme_io_uring' --- lisa/tools/__init__.py | 3 ++- lisa/tools/fio.py | 25 +++++++++++++++++--- microsoft/testsuites/performance/common.py | 10 ++++++-- microsoft/testsuites/performance/nvmeperf.py | 24 +++++++++++++++++-- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/lisa/tools/__init__.py b/lisa/tools/__init__.py index 3d77b098b6..0a8aeaf1a6 100644 --- a/lisa/tools/__init__.py +++ b/lisa/tools/__init__.py @@ -36,7 +36,7 @@ from .fallocate import Fallocate from .fdisk import Fdisk from .find import Find -from .fio import FIOMODES, Fio, FIOResult +from .fio import FIOMODES, Fio, FIOResult, IoEngine from .firewall import Firewall, Iptables from .free import Free from .gcc import Gcc @@ -159,6 +159,7 @@ "Ip", "IpInfo", "Iperf3", + "IoEngine", "HibernationSetup", "Hostname", "Hugepages", diff --git a/lisa/tools/fio.py b/lisa/tools/fio.py index 87fec2f2c4..6ee07d3b74 100644 --- a/lisa/tools/fio.py +++ b/lisa/tools/fio.py @@ -37,6 +37,16 @@ class FIOResult: ) +class IoEngine(Enum): + IO_URING = "io_uring" + LIBAIO = "libaio" + POSIXAIO = "posixaio" + NONE = "" + + def __str__(self) -> str: + return self.value + + class Fio(Tool): fio_repo = "https://github.com/axboe/fio/" branch = "fio-3.29" @@ -93,6 +103,7 @@ def launch( verify_dump: bool = False, verify_fatal: bool = False, verify: str = "", + ioengine: IoEngine = IoEngine.NONE, cwd: Optional[pathlib.PurePath] = None, ) -> FIOResult: cmd = self._get_command( @@ -114,6 +125,7 @@ def launch( verify_dump, verify_fatal, verify, + ioengine, ) result = self.run( cmd, @@ -149,6 +161,7 @@ def launch_async( verify_dump: bool = False, verify_fatal: bool = False, verify: str = "", + ioengine: IoEngine = IoEngine.NONE, cwd: Optional[pathlib.PurePath] = None, ) -> Process: cmd = self._get_command( @@ -170,6 +183,7 @@ def launch_async( verify_dump, verify_fatal, verify, + ioengine, ) process = self.run_async( cmd, @@ -244,7 +258,7 @@ def create_performance_messages( fio_message.append(fio_result_message) return fio_message - def _get_command( + def _get_command( # noqa: C901 self, name: str, filename: str, @@ -264,10 +278,15 @@ def _get_command( verify_dump: bool = False, verify_fatal: bool = False, verify: str = "", + ioengine: IoEngine = IoEngine.LIBAIO, ) -> str: - ioengine = "posixaio" if isinstance(self.node.os, BSD) else "libaio" + if isinstance(self.node.os, BSD): + ioengine = IoEngine.POSIXAIO + elif ioengine == IoEngine.NONE: + ioengine = IoEngine.LIBAIO + cmd = ( - f"--ioengine={ioengine} --filename={filename} " + f"--ioengine={ioengine.value} --filename={filename} " f"--readwrite={mode} --iodepth={iodepth} " f"--name={name}" ) diff --git a/microsoft/testsuites/performance/common.py b/microsoft/testsuites/performance/common.py index 0434f4278e..95c1c0a87c 100644 --- a/microsoft/testsuites/performance/common.py +++ b/microsoft/testsuites/performance/common.py @@ -44,6 +44,7 @@ Ssh, Sysctl, ) +from lisa.tools.fio import IoEngine from lisa.tools.ntttcp import ( NTTTCP_TCP_CONCURRENCY, NTTTCP_TCP_CONCURRENCY_BSD, @@ -70,12 +71,14 @@ def perf_disk( size_mb: int = 0, numjob: int = 0, overwrite: bool = False, + ioengine: IoEngine = IoEngine.NONE, cwd: Optional[pathlib.PurePath] = None, ) -> None: fio_result_list: List[FIOResult] = [] fio = node.tools[Fio] numjobiterator = 0 - # In fio test, numjob*max_iodepth (aio-nr) should always be less than aio-max-nr. + # In fio test with ioengine == 'libaio', + # numjob*max_iodepth (aio-nr) should always be less than aio-max-nr. # The default value of aio-max-nr is 65536. # As max_iodepth is 256, numjob which is equal to 'nproc' should not exceed 256. # /proc/sys/fs/aio-nr is the number of events currently active. @@ -84,7 +87,9 @@ def perf_disk( # fail with EAGAIN. # read: https://www.kernel.org/doc/Documentation/sysctl/fs.txt # So we set numjob to 256 if numjob is larger than 256. - numjob = min(numjob, 256) + # This limitation is only needed for 'libaio' ioengine but not for 'io_uring'. + if ioengine == IoEngine.LIBAIO: + numjob = min(numjob, 256) for mode in FIOMODES: iodepth = start_iodepth numjobindex = 0 @@ -102,6 +107,7 @@ def perf_disk( overwrite=overwrite, numjob=numjob, cwd=cwd, + ioengine=ioengine, ) fio_result_list.append(fio_result) iodepth = iodepth * 2 diff --git a/microsoft/testsuites/performance/nvmeperf.py b/microsoft/testsuites/performance/nvmeperf.py index bec8de4f2d..20def332e4 100644 --- a/microsoft/testsuites/performance/nvmeperf.py +++ b/microsoft/testsuites/performance/nvmeperf.py @@ -13,6 +13,7 @@ from lisa.messages import DiskSetupType, DiskType from lisa.testsuite import TestResult from lisa.tools import Echo, Lscpu +from lisa.tools.fio import IoEngine from microsoft.testsuites.performance.common import perf_disk @@ -24,7 +25,7 @@ """, ) class NvmePerformace(TestSuite): - TIME_OUT = 5000 + TIME_OUT = 7200 @TestCaseMetadata( description=""" @@ -37,6 +38,25 @@ class NvmePerformace(TestSuite): ), ) def perf_nvme(self, node: Node, result: TestResult) -> None: + self._perf_nvme(node, IoEngine.LIBAIO, result) + + @TestCaseMetadata( + description=""" + This test case uses fio to test NVMe disk performance + with 'io_uring' as ioengine. + """, + priority=3, + timeout=TIME_OUT, + requirement=simple_requirement( + supported_features=[Nvme], + ), + ) + def perf_nvme_io_uring(self, node: Node, result: TestResult) -> None: + self._perf_nvme(node, IoEngine.IO_URING, result, max_iodepth=1024) + + def _perf_nvme( + self, node: Node, ioengine: IoEngine, result: TestResult, max_iodepth: int = 256 + ) -> None: nvme = node.features[Nvme] nvme_namespaces = nvme.get_raw_nvme_disks() disk_count = len(nvme_namespaces) @@ -59,7 +79,6 @@ def perf_nvme(self, node: Node, result: TestResult) -> None: cpu = node.tools[Lscpu] core_count = cpu.get_core_count() start_iodepth = 1 - max_iodepth = 256 perf_disk( node, start_iodepth, @@ -71,4 +90,5 @@ def perf_nvme(self, node: Node, result: TestResult) -> None: disk_setup_type=DiskSetupType.raw, disk_type=DiskType.nvme, test_result=result, + ioengine=ioengine, ) From b29f0b2fa66a51e945ad17bf582cef18b5ad8164 Mon Sep 17 00:00:00 2001 From: Srikanth Myakam Date: Sat, 30 Mar 2024 15:03:40 +0530 Subject: [PATCH 2/3] Remove None IO engine --- lisa/tools/fio.py | 7 ++----- microsoft/testsuites/performance/common.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lisa/tools/fio.py b/lisa/tools/fio.py index 6ee07d3b74..59f56fdc56 100644 --- a/lisa/tools/fio.py +++ b/lisa/tools/fio.py @@ -41,7 +41,6 @@ class IoEngine(Enum): IO_URING = "io_uring" LIBAIO = "libaio" POSIXAIO = "posixaio" - NONE = "" def __str__(self) -> str: return self.value @@ -103,7 +102,7 @@ def launch( verify_dump: bool = False, verify_fatal: bool = False, verify: str = "", - ioengine: IoEngine = IoEngine.NONE, + ioengine: IoEngine = IoEngine.LIBAIO, cwd: Optional[pathlib.PurePath] = None, ) -> FIOResult: cmd = self._get_command( @@ -161,7 +160,7 @@ def launch_async( verify_dump: bool = False, verify_fatal: bool = False, verify: str = "", - ioengine: IoEngine = IoEngine.NONE, + ioengine: IoEngine = IoEngine.LIBAIO, cwd: Optional[pathlib.PurePath] = None, ) -> Process: cmd = self._get_command( @@ -282,8 +281,6 @@ def _get_command( # noqa: C901 ) -> str: if isinstance(self.node.os, BSD): ioengine = IoEngine.POSIXAIO - elif ioengine == IoEngine.NONE: - ioengine = IoEngine.LIBAIO cmd = ( f"--ioengine={ioengine.value} --filename={filename} " diff --git a/microsoft/testsuites/performance/common.py b/microsoft/testsuites/performance/common.py index 95c1c0a87c..975ccadac5 100644 --- a/microsoft/testsuites/performance/common.py +++ b/microsoft/testsuites/performance/common.py @@ -71,7 +71,7 @@ def perf_disk( size_mb: int = 0, numjob: int = 0, overwrite: bool = False, - ioengine: IoEngine = IoEngine.NONE, + ioengine: IoEngine = IoEngine.LIBAIO, cwd: Optional[pathlib.PurePath] = None, ) -> None: fio_result_list: List[FIOResult] = [] From f357094cbe86937f4fb27609af75f19260aac9c4 Mon Sep 17 00:00:00 2001 From: SrikanthMyakam Date: Thu, 19 Sep 2024 13:59:39 +0530 Subject: [PATCH 3/3] Update nvmeperf.py --- microsoft/testsuites/performance/nvmeperf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/microsoft/testsuites/performance/nvmeperf.py b/microsoft/testsuites/performance/nvmeperf.py index 20def332e4..ebe3276d7a 100644 --- a/microsoft/testsuites/performance/nvmeperf.py +++ b/microsoft/testsuites/performance/nvmeperf.py @@ -29,7 +29,8 @@ class NvmePerformace(TestSuite): @TestCaseMetadata( description=""" - This test case uses fio to test NVMe disk performance. + This test case uses fio to test NVMe disk performance + using 'libaio' as ioengine """, priority=3, timeout=TIME_OUT, @@ -43,7 +44,7 @@ def perf_nvme(self, node: Node, result: TestResult) -> None: @TestCaseMetadata( description=""" This test case uses fio to test NVMe disk performance - with 'io_uring' as ioengine. + using 'io_uring' as ioengine. """, priority=3, timeout=TIME_OUT,