Implement a safe_url based on all standards

.bandit.yml

@@ -1,2 +1,3 @@
 skips:
+- B101  # Use of assert; we use them for mypy
 - B107

.coveragerc

@@ -1,3 +1,2 @@
 [run]
 branch = true
-include = w3lib/*

.gitignore

@@ -14,3 +14,5 @@ coverage.xml
 /index.txt
 .dmypy.json
 .hypothesis/
+*.so
+*.html

MANIFEST.in

@@ -1,3 +1,6 @@
+# Include C and Cython files
+include **/*.c **/*.pyx
+
 # Include tests into distribution
 recursive-include tests *.py *.txt
 
@@ -10,3 +13,4 @@ include NEWS
 include README.rst
 include pytest.ini
 include tox.ini
+include w3lib/idna.txt

setup.py

@@ -1,6 +1,47 @@
-from setuptools import setup, find_packages
+import os
+from setuptools import setup, find_packages, Extension
 
 
+# https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html#distributing-cython-modules
+def no_cythonize(extensions, **_ignore):
+    for extension in extensions:
+        sources = []
+        for sfile in extension.sources:
+            path, ext = os.path.splitext(sfile)
+            if ext in (".pyx", ".py"):
+                if extension.language == "c++":
+                    ext = ".cpp"
+                else:
+                    ext = ".c"
+                sfile = path + ext
+            sources.append(sfile)
+        extension.sources[:] = sources
+    return extensions
+
+extensions = [
+    Extension(f"w3lib._{name}", [f"w3lib/_{name}.pyx"])
+    for name in (
+        "infra",
+        "rfc2396",
+        "rfc3986",
+        "rfc5892",
+        "types",
+        "url",
+        "util",
+        "utr46",
+    )
+]
+
+if bool(int(os.getenv("CYTHONIZE", 0))):
+    from Cython.Build import cythonize
+    compiler_directives = {
+        "language_level": 3,
+        "profile": bool(int(os.getenv("CYTHON_PROFILE", 0))),
+    }
+    extensions = cythonize(extensions, compiler_directives=compiler_directives, force=bool(int(os.getenv("CYTHON_FORCE", 0))))
+else:
+    extensions = no_cythonize(extensions)
+
 setup(
     name="w3lib",
     version="2.1.2",
@@ -22,6 +63,10 @@
     zip_safe=False,
     platforms=["Any"],
     python_requires=">=3.8",
+    install_requires=[
+        "idna",
+    ],
+    ext_modules=extensions,
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: BSD License",

tests/percent-encoding-test-data.json

@@ -0,0 +1,50 @@
+[
+    "Source: https://github.com/web-platform-tests/wpt/blob/77d95d52351ebf51bc80ee21eea455de31a5e356/url/resources/percent-encoding.json",
+
+    "Tests for percent-encoding.",
+    {
+      "input": "\u2020",
+      "output": {
+        "big5": "%26%238224%3B",
+        "euc-kr": "%A2%D3",
+        "utf-8": "%E2%80%A0",
+        "windows-1252": "%86"
+      }
+    },
+    "This uses a trailing A to prevent the URL parser from trimming the C0 control.",
+    {
+      "input": "\u000EA",
+      "output": {
+        "big5": "%0EA",
+        "iso-2022-jp": "%26%2365533%3BA",
+        "utf-8": "%0EA"
+      }
+    },
+    {
+      "input": "\u203E\u005C",
+      "output": {
+        "iso-2022-jp": "%1B(J~%1B(B\\",
+        "utf-8": "%E2%80%BE\\"
+      }
+    },
+    {
+      "input": "\uE5E5",
+      "output": {
+        "gb18030": "%26%2358853%3B",
+        "utf-8": "%EE%97%A5"
+      }
+    },
+    {
+      "input": "\u2212",
+      "output": {
+        "shift_jis": "%81|",
+        "utf-8": "%E2%88%92"
+      }
+    },
+    {
+      "input": "á|",
+      "output": {
+        "utf-8": "%C3%A1|"
+      }
+    }
+  ]

tests/test_encoding.py

@@ -2,6 +2,9 @@
 import unittest
 from typing import Optional, Union, List, Any
 
+import pytest
+
+from w3lib._encoding import _get_encoding, _LABEL_ENCODINGS
 from w3lib.encoding import (
     html_body_declared_encoding,
     http_content_type_encoding,
@@ -11,6 +14,26 @@
     to_unicode,
 )
 
+# Encodings from the spec that Python does not support.
+_UNSUPPORTED_ENCODINGS = {
+    "iso-8859-8-i",  # https://bugs.python.org/msg213772
+    "replacement",  # Not an actual encoding
+    # Not supported.
+    # We could bring support to it with the webencodings package.
+    "x-user-defined",
+}
+
+
+@pytest.mark.parametrize(
+    "label,name", tuple((label, name) for label, name in _LABEL_ENCODINGS.items())
+)
+def test_get_encoding_python(label, name):
+    """The encodings that _get_encoding can return must work as encoding
+    aliases in Python."""
+    assert _get_encoding(label) == name
+    if name not in _UNSUPPORTED_ENCODINGS:
+        codecs.lookup(name)  # Raises LookupError if not found.
+
 
 class RequestEncodingTests(unittest.TestCase):
     utf8_fragments = [

tests/test_html.py

@@ -1,5 +1,6 @@
 import unittest
 
+from w3lib._infra import _C0_CONTROL_OR_SPACE
 from w3lib.html import (
     get_base_url,
     get_meta_refresh,