Added 'io_uring' support to LISA io perf tests and new testcase 'perf…

…_nvme_io_uring'

Added 'io_uring' support to LISA io perf tests and new testcase 'perf_nvme_io_uring'

lisa/tools/fio.py

@@ -93,6 +93,7 @@ def launch(
         verify_dump: bool = False,
         verify_fatal: bool = False,
         verify: str = "",
+        ioengine: str = "",
         cwd: Optional[pathlib.PurePath] = None,
     ) -> FIOResult:
         cmd = self._get_command(
@@ -114,6 +115,7 @@ def launch(
             verify_dump,
             verify_fatal,
             verify,
+            ioengine,
         )
         result = self.run(
             cmd,
@@ -149,6 +151,7 @@ def launch_async(
         verify_dump: bool = False,
         verify_fatal: bool = False,
         verify: str = "",
+        ioengine: str = "",
         cwd: Optional[pathlib.PurePath] = None,
     ) -> Process:
         cmd = self._get_command(
@@ -170,6 +173,7 @@ def launch_async(
             verify_dump,
             verify_fatal,
             verify,
+            ioengine,
         )
         process = self.run_async(
             cmd,
@@ -244,7 +248,7 @@ def create_performance_messages(
             fio_message.append(fio_result_message)
         return fio_message
 
-    def _get_command(
+    def _get_command(  # noqa: C901
         self,
         name: str,
         filename: str,
@@ -264,8 +268,11 @@ def _get_command(
         verify_dump: bool = False,
         verify_fatal: bool = False,
         verify: str = "",
+        ioengine: str = "",
     ) -> str:
-        ioengine = "posixaio" if isinstance(self.node.os, BSD) else "libaio"
+        if ioengine == "":
+            ioengine = "posixaio" if isinstance(self.node.os, BSD) else "libaio"
+
         cmd = (
             f"--ioengine={ioengine} --filename={filename} "
             f"--readwrite={mode} --iodepth={iodepth} "

microsoft/testsuites/performance/common.py

@@ -70,12 +70,14 @@ def perf_disk(
     size_mb: int = 0,
     numjob: int = 0,
     overwrite: bool = False,
+    ioengine: str = "",
     cwd: Optional[pathlib.PurePath] = None,
 ) -> None:
     fio_result_list: List[FIOResult] = []
     fio = node.tools[Fio]
     numjobiterator = 0
-    # In fio test, numjob*max_iodepth (aio-nr) should always be less than aio-max-nr.
+    # In fio test with ioengine == 'libaio',
+    # numjob*max_iodepth (aio-nr) should always be less than aio-max-nr.
     # The default value of aio-max-nr is 65536.
     # As max_iodepth is 256, numjob which is equal to 'nproc' should not exceed 256.
     # /proc/sys/fs/aio-nr is the number of events currently active.
@@ -84,7 +86,9 @@ def perf_disk(
     # fail with EAGAIN.
     # read: https://www.kernel.org/doc/Documentation/sysctl/fs.txt
     # So we set numjob to 256 if numjob is larger than 256.
-    numjob = min(numjob, 256)
+    # This limitation is only needed for 'libaio' ioengine but not for 'io_uring'.
+    if ioengine == "libaio":
+        numjob = min(numjob, 256)
     for mode in FIOMODES:
         iodepth = start_iodepth
         numjobindex = 0
@@ -102,6 +106,7 @@ def perf_disk(
                 overwrite=overwrite,
                 numjob=numjob,
                 cwd=cwd,
+                ioengine=ioengine,
             )
             fio_result_list.append(fio_result)
             iodepth = iodepth * 2

microsoft/testsuites/performance/nvmeperf.py

@@ -24,7 +24,7 @@
     """,
 )
 class NvmePerformace(TestSuite):
-    TIME_OUT = 5000
+    TIME_OUT = 7200
 
     @TestCaseMetadata(
         description="""
@@ -72,3 +72,52 @@ def perf_nvme(self, node: Node, result: TestResult) -> None:
             disk_type=DiskType.nvme,
             test_result=result,
         )
+
+    @TestCaseMetadata(
+        description="""
+        This test case uses fio to test NVMe disk performance
+        with 'io_uring' as ioengine.
+        """,
+        priority=3,
+        timeout=TIME_OUT,
+        requirement=simple_requirement(
+            supported_features=[NvmeSettings(disk_count=8)],
+        ),
+    )
+    def perf_nvme_io_uring(self, node: Node, result: TestResult) -> None:
+        nvme = node.features[Nvme]
+        nvme_namespaces = nvme.get_raw_nvme_disks()
+        disk_count = len(nvme_namespaces)
+        assert_that(disk_count).described_as(
+            "At least 1 NVMe disk for fio testing."
+        ).is_greater_than(0)
+        filename = ":".join(nvme_namespaces)
+        echo = node.tools[Echo]
+        # This will have kernel avoid sending IPI to finish I/O on the issuing CPUs
+        # if they are not on the same NUMA node of completion CPU.
+        # This setting will give a better and more stable IOPS.
+        for nvme_namespace in nvme_namespaces:
+            # /dev/nvme0n1 => nvme0n1
+            disk_name = nvme_namespace.split("/")[-1]
+            echo.write_to_file(
+                "0",
+                node.get_pure_path(f"/sys/block/{disk_name}/queue/rq_affinity"),
+                sudo=True,
+            )
+        cpu = node.tools[Lscpu]
+        core_count = cpu.get_core_count()
+        start_iodepth = 1
+        max_iodepth = 1024
+        perf_disk(
+            node,
+            start_iodepth,
+            max_iodepth,
+            filename,
+            core_count=core_count,
+            disk_count=disk_count,
+            numjob=core_count,
+            disk_setup_type=DiskSetupType.raw,
+            disk_type=DiskType.nvme,
+            test_result=result,
+            ioengine="io_uring",
+        )