fix(tools/python_dep_check): replace deprecated pkg_resources with im…

…portlib

Closes #11712

tools/check_python_dependencies.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2018-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
@@ -9,13 +9,22 @@
 import sys
 
 try:
-    import pkg_resources
+    from packaging.requirements import Requirement
+    from packaging.version import Version
 except ImportError:
-    print('pkg_resources cannot be imported. The most common cause is a missing pip or setuptools package. '
-          'If you\'ve installed a custom Python then these packages are provided separately and have to be installed as well. '
+    print('packaging cannot be imported. '
+          'If you\'ve installed a custom Python then this package is provided separately and have to be installed as well. '
           'Please refer to the Get Started section of the ESP-IDF Programming Guide for setting up the required packages.')
     sys.exit(1)
 
+try:
+    from importlib.metadata import PackageNotFoundError, requires
+    from importlib.metadata import version as get_version
+except ImportError:
+    # compatibility for python <=3.7
+    from importlib_metadata import PackageNotFoundError, requires  # type: ignore
+    from importlib_metadata import version as get_version  # type: ignore
+
 try:
     from typing import Set
 except ImportError:
@@ -59,39 +68,55 @@
                 if not name_m:
                     print('Malformed input. Cannot find name in {}'.format(con))
                     sys.exit(1)
-                constr_dict[name_m[0]] = con
+                constr_dict[name_m[0]] = con.partition(' #')[0]  # remove comments
 
     not_satisfied = []  # in string form which will be printed
 
     # already_checked set is used in order to avoid circular checks which would cause looping.
-    already_checked = set()  # type: Set[pkg_resources.Requirement]
+    already_checked = set()  # type: Set[Requirement]
 
     # required_set contains package names in string form without version constraints. If the package has a constraint
     # specification (package name + version requirement) then use that instead. new_req_list is used to store
     # requirements to be checked on each level of breath-first-search of the package dependency tree. The initial
     # version is the direct dependencies deduced from the requirements arguments of the script.
-    new_req_list = [pkg_resources.Requirement.parse(constr_dict.get(i, i)) for i in required_set]
+    new_req_list = [Requirement(constr_dict.get(i, i)) for i in required_set]
+
+    def version_check(requirement: Requirement) -> None:
+        # compare installed version with required
+        version = Version(get_version(requirement.name))
+        if version.base_version not in requirement.specifier:
+            not_satisfied.append(f"Requirement '{requirement}' was not met. Installed version: {version}")
+
+    # evaluate markers and check versions of direct requirements
+    for req in new_req_list[:]:
+        if not req.marker or req.marker.evaluate():
+            version_check(req)
+        else:
+            new_req_list.remove(req)
 
     while new_req_list:
         req_list = new_req_list
         new_req_list = []
         already_checked.update(req_list)
         for requirement in req_list:  # check one level of the dependency tree
             try:
-                dependency_requirements = set(pkg_resources.get_distribution(requirement).requires())
+                dependency_requirements = set()
+                extras = list(requirement.extras) or ['']
+                for name in requires(requirement.name) or []:
+                    sub_req = Requirement(name)
+                    # check extras e.g. esptool[hsm]
+                    for extra in extras:
+                        # evaluate markers if present
+                        if not sub_req.marker or sub_req.marker.evaluate(environment={'extra': extra}):
+                            dependency_requirements.add(sub_req)
+                            version_check(sub_req)
                 # dependency_requirements are the direct dependencies of "requirement". They belong to the next level
                 # of the dependency tree. They will be checked only if they haven't been already. Note that the
                 # version is taken into account as well because packages can have different requirements for a given
                 # Python package. The dependencies need to be checked for all of them because they can be different.
                 new_req_list.extend(dependency_requirements - already_checked)
-            except pkg_resources.ResolutionError as e:
-                not_satisfied.append(' - '.join([str(requirement), str(e)]))
-            except IndexError:
-                # If the requirement is not installed because of a marker (requirement.marker), for example different
-                # operating system or python version, then pkg_resources.get_distribution() will fail with IndexError.
-                # We could avoid this by checking packaging.markers.Marker(requirement.marker).evaluate() but it would
-                # add dependency on packaging.
-                pass
+            except PackageNotFoundError as e:
+                not_satisfied.append(f"'{e}' - was not found and is required by the application")
 
     if len(not_satisfied) > 0:
         print('The following Python requirements are not satisfied:')

tools/ci/build_template_app.sh

@@ -62,8 +62,7 @@ build_stage2() {
         --build-log ${BUILD_LOG_CMAKE} \
         --size-file size.json \
         --collect-size-info size_info.txt \
-        --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3 \
-        --ignore-warning-str "DeprecationWarning: pkg_resources is deprecated as an API"
+        --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3
         # add esp32h2 back after IDF-5541
 }
 
@@ -78,8 +77,7 @@ build_stage1() {
         --build-log ${BUILD_LOG_CMAKE} \
         --size-file size.json \
         --collect-size-info size_info.txt \
-        --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32h2 \
-        --ignore-warning-str "DeprecationWarning: pkg_resources is deprecated as an API"
+        --default-build-targets esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32h2
 }
 
 # Default arguments

tools/ci/ignore_build_warnings.txt

@@ -15,4 +15,3 @@ CryptographyDeprecationWarning
 Warning: \d+/\d+ app partitions are too small for binary
 CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\)
 The smallest .+ partition is nearly full \(\d+% free space left\)!
-DeprecationWarning: pkg_resources is deprecated as an API

tools/requirements/requirements.core.txt

@@ -1,6 +1,9 @@
 # Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems.
 
 setuptools
+packaging
+# importlib_metadata: is part of python3.8 and newer as importlib.metadata
+importlib_metadata; python_version < "3.8"
 click
 pyserial
 future