diff --git a/lisa/tools/git.py b/lisa/tools/git.py index 35fa778a18..38d13350c0 100644 --- a/lisa/tools/git.py +++ b/lisa/tools/git.py @@ -343,14 +343,16 @@ def get_latest_commit_details(self, cwd: pathlib.PurePath) -> Dict[str, str]: expected_exit_code_failure_message="Failed to fetch author email.", ).stdout - describe = self.run( + describe_result = self.run( "describe", shell=True, cwd=cwd, force_run=True, - expected_exit_code=0, - expected_exit_code_failure_message="Failed to run git describe", - ).stdout + ) + if describe_result.exit_code == 0: + describe = describe_result.stdout + else: + describe = "" result = { "full_commit_id": filter_ansi_escape(latest_commit_id), diff --git a/lisa/util/__init__.py b/lisa/util/__init__.py index 3434d4b4e6..7217f82051 100644 --- a/lisa/util/__init__.py +++ b/lisa/util/__init__.py @@ -24,6 +24,7 @@ Union, cast, ) +from urllib.parse import urlparse import paramiko import pluggy @@ -47,7 +48,7 @@ # source - # https://github.com/django/django/blob/stable/1.3.x/django/core/validators.py#L45 __url_pattern = re.compile( - r"^(?:http|ftp)s?://" # http:// or https:// + r"^(?:http|https|sftp|ftp)://" # http:// or https:// r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)" r"+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" # ...domain r"localhost|" # localhost... @@ -598,6 +599,112 @@ def is_valid_url(url: str, raise_error: bool = True) -> bool: return is_url +def _raise_or_log_failure(log: "Logger", raise_error: bool, failure_msg: str) -> bool: + if raise_error: + raise LisaException(failure_msg) + else: + log.debug(failure_msg) + return False + + +# big function to check the parts of a url +# allow raising exceptions or log and return a bool +# allows checks for: +# expected domains +# protocols (require https, sftp, etc) +# filenames (pattern matching) +def check_url( + log: "Logger", + source_url: str, + allowed_protocols: Optional[List[str]] = None, + expected_domains_pattern: Optional[Pattern[str]] = None, + expected_filename_pattern: Optional[Pattern[str]] = None, + raise_error: bool = False, +) -> bool: + # avoid using a mutable default parameter + if not allowed_protocols: + allowed_protocols = [ + "https", + ] + # pylinter doesn't like returning too many times in a function + # instead we'll assign to a boolean and check it repeatedly. + # thanks, linter. + result = True + # first, check if it's a url. + failure_msg = f"{source_url} is not a valid URL, check your arguments." + if not ( + is_valid_url(url=source_url, raise_error=False) + or _raise_or_log_failure(log, raise_error, failure_msg) + ): + # fast return false, other checks depend on this one + return False + + # NOTE: urllib might not work as you'd expect. + # It doesn't throw on lots of things you wouldn't expect to be urls. + # You must verify the parts on your own, some of them may be empty, some null. + # check: https://docs.python.org/3/library/urllib.parse.html#url-parsing + failure_msg = f"urlparse failed to parse url {source_url}, check your arguments." + try: + parts = urlparse(source_url) + except ValueError: + if not _raise_or_log_failure(log, raise_error, failure_msg): + # another fast return, other checks depend on this one + return False + + # ex: from https://www.com/path/to/file.tar + # scheme : https + # netloc : www.com + # path : path/to/file.tar + + # get the filename from the path portion of the url + file_path = parts.path.split("/")[-1] + full_match = None + # check we can match against the filename + if expected_filename_pattern: + full_match = expected_filename_pattern.match(file_path) + failure_msg = ( + f"File at {source_url} did not match pattern " + f"{expected_filename_pattern.pattern}." + ) + if not full_match: + result &= _raise_or_log_failure(log, raise_error, failure_msg) + + # check the expected domain is correct if present + if ( + result + and expected_domains_pattern + and not expected_domains_pattern.match(parts.netloc) + ): + # logging domains requires check that expected_domains != None + failure_msg = ( + f"net location of url {source_url} did not match " + f"expected domains { expected_domains_pattern.pattern } " + ) + result &= _raise_or_log_failure(log, raise_error, failure_msg) + + # Check the protocol (aka scheme) in the url + # default is check access is via https + failure_msg = ( + f"URL {source_url} uses an invalid protocol " + "or net location! Check url argument." + ) + valid_scheme = any([parts.scheme == x for x in allowed_protocols]) + if result and not valid_scheme: + result &= _raise_or_log_failure(log, raise_error, failure_msg) + # finally verify the full match we found matches the actual filename + # avoids an accidental partial match + if result and expected_filename_pattern and full_match: + path_matches = full_match.group(0) == file_path + failure_msg = ( + f"File at url {source_url} failed to match" + f" pattern {expected_filename_pattern.pattern}." + ) + if not path_matches: + result &= _raise_or_log_failure(log, raise_error, failure_msg) + + return result + + def filter_ansi_escape(content: str) -> str: return __ansi_escape.sub("", content) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 32a0e2d952..ed1420f155 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -19,7 +19,6 @@ Kill, Lscpu, Lspci, - Make, Modprobe, Pidof, Pkgconfig, @@ -40,6 +39,7 @@ is_ubuntu_latest_or_prerelease, is_ubuntu_lts_version, ) +from microsoft.testsuites.dpdk.rdma_core import RdmaCoreManager PACKAGE_MANAGER_SOURCE = "package_manager" @@ -136,22 +136,6 @@ def command(self) -> str: _rx_pps_key: r"Rx-pps:\s+([0-9]+)", } - def get_rdma_core_package_name(self) -> str: - distro = self.node.os - package = "" - # check if rdma-core is installed already... - if self.node.tools[Pkgconfig].package_info_exists("libibuverbs"): - return package - if isinstance(distro, Debian): - package = "rdma-core ibverbs-providers libibverbs-dev" - elif isinstance(distro, Suse): - package = "rdma-core-devel librdmacm1" - elif isinstance(distro, Fedora): - package = "librdmacm-devel" - else: - fail("Invalid OS for rdma-core source installation.") - return package - @property def can_install(self) -> bool: for _os in [Debian, Fedora, Suse]: @@ -465,8 +449,17 @@ def add_sample_apps_to_build_list(self, apps: Union[List[str], None]) -> None: def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) + # set source args for builds if needed, first for dpdk self._dpdk_source = kwargs.pop("dpdk_source", PACKAGE_MANAGER_SOURCE) self._dpdk_branch = kwargs.pop("dpdk_branch", "main") + # then for rdma-core + rdma_core_source = kwargs.pop("rdma_core_source", "") + rdma_core_ref = kwargs.pop("rdma_core_ref", "") + self.rdma_core = RdmaCoreManager( + node=self.node, + rdma_core_source=rdma_core_source, + rdma_core_ref=rdma_core_ref, + ) self._sample_apps_to_build = kwargs.pop("sample_apps", []) self._dpdk_version_info = VersionInfo(0, 0) self._testpmd_install_path: str = "" @@ -522,54 +515,6 @@ def _check_pps_data(self, rx_or_tx: str) -> None: f"empty or all zeroes for dpdktestpmd.{rx_or_tx.lower()}_pps_data." ).is_true() - def _install_upstream_rdma_core_for_mana(self) -> None: - node = self.node - wget = node.tools[Wget] - make = node.tools[Make] - tar = node.tools[Tar] - distro = node.os - - if isinstance(distro, Debian): - distro.install_packages( - "cmake libudev-dev " - "libnl-3-dev libnl-route-3-dev ninja-build pkg-config " - "valgrind python3-dev cython3 python3-docutils pandoc " - "libssl-dev libelf-dev python3-pip libnuma-dev" - ) - elif isinstance(distro, Fedora): - distro.group_install_packages("Development Tools") - distro.install_packages( - "cmake gcc libudev-devel " - "libnl3-devel pkg-config " - "valgrind python3-devel python3-docutils " - "openssl-devel unzip " - "elfutils-devel python3-pip libpcap-devel " - "tar wget dos2unix psmisc kernel-devel-$(uname -r) " - "librdmacm-devel libmnl-devel kernel-modules-extra numactl-devel " - "kernel-headers elfutils-libelf-devel meson ninja-build libbpf-devel " - ) - else: - # check occcurs before this function - return - - tar_path = wget.get( - url=( - "https://github.com/linux-rdma/rdma-core/" - "releases/download/v46.0/rdma-core-46.0.tar.gz" - ), - file_path=str(node.working_path), - ) - - tar.extract(tar_path, dest_dir=str(node.working_path), gzip=True, sudo=True) - source_path = node.working_path.joinpath("rdma-core-46.0") - node.execute( - "cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr", - shell=True, - cwd=source_path, - sudo=True, - ) - make.make_install(source_path) - def _set_backport_repo_args(self) -> None: distro = self.node.os # skip attempting to use backports for latest/prerlease @@ -597,6 +542,18 @@ def _install(self) -> bool: self._testpmd_output_during_rescind = "" self._last_run_output = "" node = self.node + distro = node.os + if not ( + isinstance(distro, Fedora) + or isinstance(distro, Debian) + or isinstance(distro, Suse) + ): + raise SkippedException( + UnsupportedDistroException( + distro, "DPDK tests not implemented for this OS." + ) + ) + # before doing anything: determine if backport repo needs to be enabled self._set_backport_repo_args() @@ -608,42 +565,50 @@ def _install(self) -> bool: self._load_drivers_for_dpdk() return True - # otherwise, install from package manager, git, or tar - - self._install_dependencies() + # if this is mana VM, we don't support other distros yet + is_mana_test_supported = isinstance(distro, Ubuntu) or isinstance( + distro, Fedora + ) + if self.is_mana and not is_mana_test_supported: + raise SkippedException("MANA DPDK test is not supported on this OS") - # if this is mana VM, we need an upstream rdma-core package (for now) - if self.is_mana: - if not (isinstance(node.os, Ubuntu) or isinstance(node.os, Fedora)): - raise SkippedException("MANA DPDK test is not supported on this OS") + # if we need an rdma-core source install, do it now. + if self.rdma_core.can_install_from_source() or ( + is_mana_test_supported and self.is_mana + ): + # ensure no older version is installed + distro.uninstall_packages("rdma-core") + self.rdma_core.do_source_install() - # ensure no older dependency is installed - node.os.uninstall_packages("rdma-core") - self._install_upstream_rdma_core_for_mana() + # otherwise, install kernel and dpdk deps from package manager, git, or tar + self._install_dependencies() + # install any missing rdma-core packages + rdma_packages = self.rdma_core.get_missing_distro_packages() + distro.install_packages(rdma_packages) # installing from distro package manager if self.use_package_manager_install(): self.node.log.info( "Installing dpdk and dev package from package manager..." ) - if isinstance(node.os, Debian): - node.os.install_packages( + if isinstance(distro, Debian): + distro.install_packages( ["dpdk", "dpdk-dev"], extra_args=self._backport_repo_args, ) - elif isinstance(node.os, (Fedora, Suse)): - node.os.install_packages(["dpdk", "dpdk-devel"]) + elif isinstance(distro, (Fedora, Suse)): + distro.install_packages(["dpdk", "dpdk-devel"]) else: raise NotImplementedError( "Dpdk package names are missing in dpdktestpmd.install" - f" for os {node.os.name}" + f" for os {distro.name}" ) self.node.log.info( f"Installed DPDK version {str(self._dpdk_version_info)} " "from package manager" ) - self._dpdk_version_info = node.os.get_package_information("dpdk") + self._dpdk_version_info = distro.get_package_information("dpdk") self.find_testpmd_binary() self._load_drivers_for_dpdk() return True @@ -867,9 +832,6 @@ def _install_suse_dependencies(self) -> None: suse.install_packages(self._suse_packages) if not self.use_package_manager_install(): self._install_ninja_and_meson() - rdma_core_packages = self.get_rdma_core_package_name() - if rdma_core_packages: - suse.install_packages(rdma_core_packages.split()) def _install_ubuntu_dependencies(self) -> None: node = self.node @@ -904,9 +866,6 @@ def _install_ubuntu_dependencies(self) -> None: # MANA tests use linux-modules-extra-azure, install if it's available. if self.is_mana and ubuntu.is_package_in_repo("linux-modules-extra-azure"): ubuntu.install_packages("linux-modules-extra-azure") - rdma_core_packages = self.get_rdma_core_package_name() - if rdma_core_packages: - ubuntu.install_packages(rdma_core_packages.split()) def _install_fedora_dependencies(self) -> None: node = self.node @@ -919,7 +878,7 @@ def _install_fedora_dependencies(self) -> None: return # appease the type checker # DPDK is very sensitive to rdma-core/kernel mismatches - # update to latest kernel before instaling dependencies + # update to latest kernel before installing dependencies rhel.install_packages("kernel") node.reboot() @@ -928,16 +887,13 @@ def _install_fedora_dependencies(self) -> None: rhel.install_packages(["libmnl-devel", "libbpf-devel"]) try: - rhel.install_packages("kernel-devel-$(uname -r)") - except MissingPackagesException: - node.log.debug("kernel-devel-$(uname -r) not found. Trying kernel-devel") rhel.install_packages("kernel-devel") + except MissingPackagesException: + node.log.debug("Fedora: kernel-devel not found, attempting to continue") # RHEL 8 doesn't require special cases for installed packages. # TODO: RHEL9 may require updates upon release - rdma_core_packages = self.get_rdma_core_package_name() - if rdma_core_packages: - self._fedora_packages += rdma_core_packages.split() + if not self.rdma_core.is_installed_from_source: rhel.group_install_packages("Infiniband Support") rhel.group_install_packages("Development Tools") diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 8b21409554..9e536b7b74 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -312,6 +312,8 @@ def initialize_node_resources( _set_forced_source_by_distro(node, variables) dpdk_source = variables.get("dpdk_source", PACKAGE_MANAGER_SOURCE) dpdk_branch = variables.get("dpdk_branch", "") + rdma_core_source = variables.get("rdma_core_source", "") + rdma_core_ref = variables.get("rdma_core_git_ref", "") force_net_failsafe_pmd = variables.get("dpdk_force_net_failsafe_pmd", False) log.info( "Dpdk initialize_node_resources running" @@ -348,6 +350,8 @@ def initialize_node_resources( dpdk_branch=dpdk_branch, sample_apps=sample_apps, force_net_failsafe_pmd=force_net_failsafe_pmd, + rdma_core_source=rdma_core_source, + rdma_core_ref=rdma_core_ref, ) # init and enable hugepages (required by dpdk) diff --git a/microsoft/testsuites/dpdk/rdma_core.py b/microsoft/testsuites/dpdk/rdma_core.py new file mode 100644 index 0000000000..c312e1e767 --- /dev/null +++ b/microsoft/testsuites/dpdk/rdma_core.py @@ -0,0 +1,185 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import re +from urllib.parse import urlparse + +from assertpy import fail + +from lisa import Node +from lisa.operating_system import Debian, Fedora, Suse +from lisa.tools import Git, Make, Pkgconfig, Tar, Wget +from lisa.util import LisaException, SkippedException, check_url + + +class RdmaCoreManager: + def __init__(self, node: Node, rdma_core_source: str, rdma_core_ref: str) -> None: + self.is_installed_from_source = False + self.node = node + self._rdma_core_source = rdma_core_source + self._rdma_core_ref = rdma_core_ref + + def get_missing_distro_packages(self) -> str: + distro = self.node.os + package = "" + # check if rdma-core is installed already... + if self.node.tools[Pkgconfig].package_info_exists("libibuverbs"): + return package + if isinstance(distro, Debian): + package = "rdma-core ibverbs-providers libibverbs-dev" + elif isinstance(distro, Suse): + package = "rdma-core-devel librdmacm1" + elif isinstance(distro, Fedora): + package = "librdmacm-devel" + else: + fail("Invalid OS for rdma-core source installation.") + return package + + def _check_source_name(self) -> bool: + source = self._rdma_core_source + try: + parts = urlparse(self._rdma_core_source) + except ValueError: + raise LisaException(f"Invalid rdma-core source build url: {source}") + file_path = parts.path.split("/")[-1] + return ( + any([parts.scheme == x for x in ["https", "ssh"]]) + and parts.netloc != "" + and ( + file_path == "rdma-core.git" + or (file_path.startswith("rdma-core") and file_path.endswith(".tar.gz")) + ) + ) + + _rdma_core_domain_pattern = re.compile( + ( + r"^((?:www\.)?(?:(?:(?:microsoft|msazure)\.)" + r"?(?:visualstudio|gitlab|github)\.com)|git\.launchpad\.net)" + ) + ) + _source_pattern = re.compile(r"rdma-core(.v?[0-9]+)*.(git|tar(\.gz)?)") + + def _check_source_install(self) -> None: + if self._rdma_core_source: + # accept either a tar.gz or a git tree + if self.is_from_tarball(): + self._rdma_core_ref = "" + elif self.is_from_git(): + # will check ref later + pass + else: + raise SkippedException( + "rdma-core source must be rdma-core.*tar.gz " + f"or https://.../rdma-core.git. found {self._rdma_core_source}" + ) + elif self._rdma_core_ref: + # if there's a ref but no tree, use a default tree + self._rdma_core_source = "https://github.com/linux-rdma/rdma-core.git" + else: + # no ref, no tree, use a default tar.gz + self._rdma_core_source = ( + "https://github.com/linux-rdma/rdma-core/" + "releases/download/v46.0/rdma-core-46.0.tar.gz" + ) + + # finally, validate what we have looks reasonable and cool + is_valid_package = check_url( + self.node.log, + source_url=self._rdma_core_source, + allowed_protocols=["https"], + expected_domains_pattern=self._rdma_core_domain_pattern, + expected_filename_pattern=self._source_pattern, + ) + if not is_valid_package: + raise SkippedException(self._get_source_pkg_error_message()) + + self.is_installed_from_source = True + + def _get_source_pkg_error_message(self) -> str: + return ( + "rdma-source package provided did not validate. " + "Use https for a git named rdma-core.git or " + "https/sftp to fetch a tar.gz package named rdma-core(.xx).tar.gz. " + "Source site must be at visualstudio, gitlab, github, or git.launchpad.net." + f"Found: {self._rdma_core_source}" + ) + + def is_from_git(self) -> bool: + return bool( + self._rdma_core_source and self._rdma_core_source.endswith("rdma-core.git") + ) + + def is_from_tarball(self) -> bool: + return bool( + self._rdma_core_source and self._rdma_core_source.endswith(".tar.gz") + ) + + def can_install_from_source(self) -> bool: + return bool(self._rdma_core_source or self._rdma_core_ref) + + def do_source_install(self) -> None: + node = self.node + wget = node.tools[Wget] + make = node.tools[Make] + tar = node.tools[Tar] + distro = node.os + + # setup looks at options and selects some reasonable defaults + # allow a tar.gz or git + # if ref and no tree, use the default tree at github + # if tree and no ref, checkout latest tag + # if tree and ref... you get the idea + self._check_source_install() + + # for dependencies, see https://github.com/linux-rdma/rdma-core#building + if isinstance(distro, Debian): + distro.install_packages( + "cmake libudev-dev " + "libnl-3-dev libnl-route-3-dev ninja-build pkg-config " + "valgrind python3-dev cython3 python3-docutils pandoc " + "libssl-dev libelf-dev python3-pip libnuma-dev" + ) + elif isinstance(distro, Fedora): + distro.group_install_packages("Development Tools") + distro.install_packages( + "cmake gcc libudev-devel " + "libnl3-devel pkg-config " + "valgrind python3-devel python3-docutils " + "openssl-devel unzip " + "elfutils-devel python3-pip libpcap-devel " + "tar wget dos2unix psmisc kernel-devel-$(uname -r) " + "librdmacm-devel libmnl-devel kernel-modules-extra numactl-devel " + "kernel-headers elfutils-libelf-devel meson ninja-build libbpf-devel " + ) + else: + # no-op, throw for invalid distro is before this function + return + + if self.is_from_git(): + git = node.tools[Git] + source_path = git.clone( + self._rdma_core_source, cwd=node.working_path, ref=self._rdma_core_ref + ) + # if there wasn't a ref provided, check out the latest tag + if not self._rdma_core_ref: + git_ref = git.get_tag(cwd=source_path) + git.checkout(git_ref, cwd=source_path) + elif self.is_from_tarball(): + tar_path = wget.get( + url=(self._rdma_core_source), + file_path=str(node.working_path), + ) + + tar.extract(tar_path, dest_dir=str(node.working_path), gzip=True, sudo=True) + source_folder = tar_path.replace(".tar.gz", "") + source_path = node.get_pure_path(source_folder) + else: + raise SkippedException(self._get_source_pkg_error_message()) + + node.execute( + "cmake -DIN_PLACE=0 -DNO_MAN_PAGES=1 -DCMAKE_INSTALL_PREFIX=/usr", + shell=True, + cwd=source_path, + sudo=True, + ) + make.make_install(source_path)