Add support for Python's "Limited API"; Test cibuildwheel and py >=3.5 using TestPythonSML.py

.github/workflows/build.yml

@@ -311,16 +311,99 @@ jobs:
         if: matrix.os == 'macos-latest'
         run: brew install swig
 
+      - name: Setup SWIG (windows-latest)
+        if: matrix.os == 'windows-latest'
+        uses: MinoruSekine/setup-scoop@v4
+        with:
+          apps: swig
+
       - name: Build wheels
         uses: pypa/[email protected]
         with:
           package-dir: Core/ClientSMLSWIG/Python/
 
+      - name: ABI3Audit (*nix)
+        if: matrix.os != 'windows-latest'
+        run: pipx run abi3audit -S -v wheelhouse/*
+
+      - name: ABI3Audit (Windows)
+        if: matrix.os == 'windows-latest'
+        # For windows we can't use glob expansion, so we need to enumerate the files manually.
+        # https://stackoverflow.com/a/16804630/8700553
+        run: Get-ChildItem -File wheelhouse | Foreach {pipx run abi3audit -S -v $_.fullname}
+
       - uses: actions/upload-artifact@v4
         with:
           name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
           path: ./wheelhouse/*.whl
 
+  test_python:
+    name: Test wheel on all python versions
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          # linux target which supports all python versions we want to install for
+          - ubuntu-20.04
+            # latest available X86_64 target
+          - macos-12
+            # latest is ARM
+          - macos-latest
+        python-minor:
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          - 11
+          - 12
+
+        # for macos-latest, we can't use python 3.5 - 3.7
+        exclude:
+          - os: macos-latest
+            python-minor: 5
+          - os: macos-latest
+            python-minor: 6
+          - os: macos-latest
+            python-minor: 7
+
+    needs:
+      # We depend on the python wheel builds because we need their artifacts
+      - python_wheels
+
+      # We depend on the builds themselves to gatekeep the upload if build + testing fails,
+      # usually then also the python wheel build should fail, but the normal builds do more thorough checking.
+      - Posix
+      - Windows
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-wheels-*
+          path: wheelhouse/
+          merge-multiple: true
+
+      - name: Setup Base Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: |
+            3.${{ matrix.python-minor }}
+            3.12
+          # Latest version is installed under `python`
+        env:
+          # Older python versions fail install if we dont do this: https://github.com/actions/setup-python/issues/866
+          PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org"
+
+      - name: Run python test command
+        # TODO: replace with actual test command: https://github.com/SoarGroup/Soar/issues/460
+        run: |
+          pip3.${{ matrix.python-minor }} install soar-sml -f wheelhouse --no-index
+          python3.${{ matrix.python-minor }} -c "import soar_sml;
+          k=soar_sml.Kernel.CreateKernelInNewThread();
+          a=k.CreateAgent('soar');
+          assert(a.ExecuteCommandLine('echo hello world').strip()=='hello world')"
+
   python_push_dev:
     name: Publish to test.pypi.org
     runs-on: ubuntu-latest
@@ -329,14 +412,8 @@ jobs:
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     # Alternative, to upload on every commit to development;
     # if: github.event_name != 'schedule' && github.event_name != 'release'
-    needs:
-      # We depend on the python wheel builds because we need their artifacts
-      - python_wheels
 
-      # We depend on the builds themselves to gatekeep the upload if build + testing fails,
-      # usually then also the python wheel build should fail, but the normal builds do more thorough checking.
-      - Posix
-      - Windows
+    needs: test_python
 
     # We use Trusted Publishing to manage our access to pypi:
     # https://github.com/marketplace/actions/pypi-publish#trusted-publishing
@@ -370,14 +447,7 @@ jobs:
     name: Publish to pypi.org
     runs-on: ubuntu-latest
     if: github.event_name == 'release'
-    needs:
-      # We depend on the python wheel builds because we need their artifacts
-      - python_wheels
-
-      # We depend on the builds themselves to gatekeep the upload if build + testing fails,
-      # usually then also the python wheel build should fail, but the normal builds do more thorough checking.
-      - Posix
-      - Windows
+    needs: test_python
 
     # We use Trusted Publishing to manage our access to pypi:
     # https://github.com/marketplace/actions/pypi-publish#trusted-publishing

Core/ClientSMLSWIG/Python/Python_sml_ClientInterface.i

@@ -11,6 +11,12 @@
 // handle windows calling convention, __declspec(dllimport), correctly
 %include <windows.i>
 
+%begin %{
+#ifdef Soar_Python_ABI3
+#define Py_LIMITED_API 0x03050000
+#endif
+%}
+
 %{
 	// helps quell warnings
 	#ifndef unused
@@ -252,7 +258,10 @@
 			return "";
 		}
 
-		std::string res = PyUnicode_AsUTF8 (result);
+		PyObject* unicode = PyUnicode_AsUTF8String (result);
+		std::string res = PyBytes_AsString(unicode);
+
+		Py_DECREF(unicode);
 		Py_DECREF(result);
 
 		PyGILState_Release(gstate); /* Release the thread. No Python API allowed beyond this point. */
@@ -303,7 +312,10 @@
 			return "";
 		}
 
-		std::string res = PyUnicode_AsUTF8 (result);
+		PyObject* unicode = PyUnicode_AsUTF8String (result);
+		std::string res = PyBytes_AsString(unicode);
+
+		Py_DECREF(unicode);
 		Py_DECREF(result);
 
 		PyGILState_Release(gstate); /* Release the thread. No Python API allowed beyond this point. */

Core/ClientSMLSWIG/Python/SConscript

@@ -59,6 +59,18 @@ install_source = env.Install(lib_install_dir, source)
 install_lib = env.Install(lib_install_dir, shlib)
 install_test = env.Install(lib_install_dir, env.File('TestPythonSML.py'))
 
+if "SOAR_PYTHON_ABI3" in env["ENV"]:
+    if env["SWIG_VERSION"] >= (4, 2, 0):
+        print("SOAR_PYTHON_ABI3 defined, building Python Limited API...")
+        # Signal to SConstruct that we're building the limited API
+        env["PYTHON_LIMITED_API"] = True
+        # Signal to the .i file that we're building the limited API
+        clone.Append(CPPDEFINES = { 'Soar_Python_ABI3': None })
+    else:
+        print("'Requested building Python SML bindings with Python's Limited API, "
+              "while SWIG does not support it, aborting.'", file=sys.stderr)
+        Exit(1)
+
 # We add soarlib to the python_sml explicitly, as some operating systems don't pick up on this dependency,
 # and crash the build.
 Import('soarlib')

Core/ClientSMLSWIG/Python/pyproject.toml

@@ -50,7 +50,7 @@ authors = [
     { name = "Y. Wang" },
 ]
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.5"
 dependencies = []
 keywords = ["soar", "sml", "cog-arch", "cognitive architecture", "cognitive", "soar-sml", "ai"]
 classifiers = [
@@ -70,7 +70,7 @@ requires = [
     # and other python build/install frontends (such as pip itself).
     #
     # We use a forked version of enscons to add python 3.12 support.
-    "enscons @ git+https://github.com/ShadowJonathan/enscons-soar@544f39f",
+    "enscons @ git+https://github.com/ShadowJonathan/enscons-soar@190091866ac35fb5d390c425bafaf13443baee5e",
 
     # Required sub-dependencies of enscons.
     "toml>=0.1",
@@ -123,6 +123,11 @@ skip = [
     # which would be a duplicate with the x86 runner.
     "cp38-macosx_arm64"
 ]
+# We only need one target to build on, since we test all versions later.
+#
+# We pick python 3.9, since macOS ARM64 doesn't have 3.8, and so this is the earliest version on all platforms.
+build = "cp39*"
+environment = { SOAR_PYTHON_ABI3="1" }
 # This test command:
 #  - imports soar_sml
 #  - creates the kernel

Core/SConscript

@@ -16,21 +16,28 @@ def recursive_glob(treeroot, pattern):
 	return results
 # 	('SoarKernel/SoarKernel.cxx',       recursive_glob(kernel_env.Dir('SoarKernel/src').abspath(), '*.cpp')),    # Kernel
 
-def CheckSWIG(env):
+def SWIGVersion(env):
 	if not env.WhereIs('swig'):
-		return False
-
+		return None
 	p = sub.Popen('swig -version'.split(), stdout=sub.PIPE)
 	out = p.communicate()[0].decode().split()
 	p.wait()
 
-	version = tuple(int(x) for x in out[2].split('.'))
+	return tuple(int(x) for x in out[2].split('.'))
+
+
+def CheckSWIG(env):
+	version = SWIGVersion(env)
+	if version is None:
+		return False
+
 	if version >= (1, 3, 31):
 		return True
 
 	print('swig version 1.3.31 or higher is required')
 	return False
 
+
 kernel_env = env.Clone()
 
 # Windows DLLs need to get linked to dependencies, whereas Linux and Mac shared objects do not
@@ -110,6 +117,7 @@ env.Alias('headers', headers)
 if not CheckSWIG(env):
 	print('SWIG not found. Will not define sml_* build targets. Install swig to build wrappers for other languages.')
 else:
+	env["SWIG_VERSION"] = SWIGVersion(env)
 	for x in 'Python Java Tcl CSharp'.split():
 		SConscript(os.path.join('ClientSMLSWIG', x, 'SConscript'))
 

SConstruct

@@ -417,11 +417,14 @@ py_sources += [
 env.Alias(SML_PYTHON_ALIAS + "_dev", py_sources)
 
 if enscons_active:
-    env['PACKAGE_METADATA'] = enscons.get_pyproject(env)['project']
     # Instead of giving an explicit tag, we tell enscons that we're not building a "pure" (python-only) library,
     # and so we let it determine the wheel tag by itself.
     env['ROOT_IS_PURELIB'] = False
 
+    if env.get('PYTHON_LIMITED_API', False):
+        # This enables tagging the wheel with the limited api, and also sets the target python version.
+        env["LIMITED_API_TARGET"] = (3, 5)
+
     # Whl and WhlFile add multiple targets (sdist, dist_info, bdist_wheel, editable) to env
     # for enscons (python build backend for scons; required for building with cibuildwheel).
     whl = env.Whl("platlib", py_sources, root="")