Merge remote-tracking branch 'scrapy/master' into safer-url

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.1
+current_version = 2.1.2
 commit = True
 tag = True
 tag_name = v{new_version}

.flake8

@@ -10,4 +10,5 @@ ignore =
     W504,
 
     # black disagrees with flake8, and inserts whitespace
-    E203,  # whitespace before ':'
+    # E203: whitespace before ':'
+    E203,

.github/workflows/build.yml

@@ -15,27 +15,27 @@ jobs:
     strategy:
       matrix:
         include:
-        - python-version: "3.7"
+        - python-version: "3.8"
           env:
             TOXENV: docs
-        - python-version: "3.10"
+        - python-version: "3.12"
           env:
             TOXENV: flake8
-        - python-version: "3.10"
+        - python-version: "3.12"
           env:
             TOXENV: pylint
-        - python-version: "3.10"
+        - python-version: "3.12"
           env:
             TOXENV: security
-        - python-version: "3.10"
+        - python-version: "3.12"
           env:
             TOXENV: black
-        - python-version: "3.10"
+        - python-version: "3.12"
           env:
             TOXENV: typing
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4

.github/workflows/publish.yml

@@ -10,12 +10,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.12
       uses: actions/setup-python@v4
       with:
-        python-version: 3.9
+        python-version: 3.12
 
     - name: Check Tag
       id: check-release-tag

.github/workflows/tests.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.10"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4

.readthedocs.yml

@@ -0,0 +1,15 @@
+version: 2
+formats: all
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+build:
+  os: ubuntu-22.04
+  tools:
+    # For available versions, see:
+    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
+    python: "3.12"  # Keep in sync with .github/workflows/build.yml
+python:
+  install:
+    - requirements: docs/requirements.txt
+    - path: .

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[[email protected]](mailto:[email protected])
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

NEWS

@@ -1,6 +1,78 @@
 w3lib release notes
 ===================
 
+2.1.2 (2023-08-03)
+------------------
+
+- Fix test failures on Python 3.11.4+ (#212, #213).
+- Fix an incorrect type hint (#211).
+- Add project URLs to setup.py (#215).
+
+2.1.1 (2022-12-09)
+------------------
+
+- :func:`~w3lib.url.safe_url_string`, :func:`~w3lib.url.safe_download_url`
+  and :func:`~w3lib.url.canonicalize_url` now strip whitespace and control
+  characters urls according to the URL living standard.
+
+
+2.1.0 (2022-11-28)
+------------------
+
+-   Dropped Python 3.6 support, and made Python 3.11 support official. (#195,
+    #200)
+
+-   :func:`~w3lib.url.safe_url_string` now generates safer URLs.
+
+    To make URLs safer for the `URL living standard`_:
+
+    .. _URL living standard: https://url.spec.whatwg.org/
+
+    -   ``;=`` are percent-encoded in the URL username.
+
+    -   ``;:=`` are percent-encoded in the URL password.
+
+    -   ``'`` is percent-encoded in the URL query if the URL scheme is `special
+        <https://url.spec.whatwg.org/#special-scheme>`__.
+
+    To make URLs safer for `RFC 2396`_ and `RFC 3986`_, ``|[]`` are
+    percent-encoded in URL paths, queries, and fragments.
+
+    .. _RFC 2396: https://www.ietf.org/rfc/rfc2396.txt
+    .. _RFC 3986: https://www.ietf.org/rfc/rfc3986.txt
+
+    (#80, #203)
+
+-   :func:`~w3lib.encoding.html_to_unicode` now checks for the `byte order
+    mark`_ before inspecting the ``Content-Type`` header when determining the
+    content encoding, in line with the `URL living standard`_. (#189, #191)
+
+    .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
+
+-   :func:`~w3lib.url.canonicalize_url` now strips spaces from the input URL,
+    to be more in line with the `URL living standard`_. (#132, #136)
+
+-   :func:`~w3lib.html.get_base_url` now ignores HTML comments. (#70, #77)
+
+-   Fixed :func:`~w3lib.url.safe_url_string` re-encoding percent signs on
+    the URL username and password even when they were being used as part of an
+    escape sequence. (#187, #196)
+
+-   Fixed :func:`~w3lib.http.basic_auth_header` using the wrong flavor of
+    base64 encoding, which could prevent authentication in rare cases. (#181,
+    #192)
+
+-   Fixed :func:`~w3lib.html.replace_entities` raising :exc:`OverflowError` in
+    some cases due to `a bug in CPython
+    <https://github.com/python/cpython/issues/76763>`__. (#199, #202)
+
+-   Improved typing and fixed typing issues. (#190, #206)
+
+-   Made CI and test improvements. (#197, #198)
+
+-   Adopted a Code of Conduct. (#194)
+
+
 2.0.1 (2022-08-11)
 ------------------
 Minor documentation fix (release date is set in the changelog).
@@ -130,8 +202,6 @@ Other improvements and bug fixes:
 - ``url_query_cleaner()``: support new ``keep_fragments`` argument
   (defaulting to ``False``)
 
-.. _RFC 3986: https://tools.ietf.org/html/rfc3986#section-3.2
-
 1.15.0 (2016-07-29)
 -------------------
 

README.rst

@@ -27,7 +27,7 @@ This is a Python library of web-related functions, such as:
 Requirements
 ============
 
-Python 3.7+
+Python 3.8+
 
 Install
 =======

docs/conf.py

@@ -53,7 +53,7 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = '2.0.1'
+release = '2.1.2'
 # The short X.Y version.
 version = '.'.join(release.split('.')[:2])
 
@@ -96,7 +96,7 @@
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/index.rst

@@ -28,7 +28,7 @@ Modules
 Requirements
 ============
 
-Python 3.7+
+Python 3.8+
 
 Install
 =======

setup.py

@@ -41,20 +41,25 @@ def no_cythonize(extensions, **_ignore):
 
 setup(
     name="w3lib",
-    version="2.0.1",
+    version="2.1.2",
     license="BSD",
     description="Library of web-related functions",
     author="Scrapy project",
     author_email="[email protected]",
     url="https://github.com/scrapy/w3lib",
+    project_urls={
+        "Documentation": "https://w3lib.readthedocs.io/en/latest/",
+        "Source Code": "https://github.com/scrapy/w3lib",
+        "Issue Tracker": "https://github.com/scrapy/w3lib/issues",
+    },
     packages=find_packages(exclude=("tests", "tests.*")),
     package_data={
         "w3lib": ["py.typed"],
     },
     include_package_data=True,
     zip_safe=False,
     platforms=["Any"],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=[
         "idna",
     ],
@@ -65,11 +70,11 @@ def no_cythonize(extensions, **_ignore):
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Topic :: Internet :: WWW/HTTP",