spacy model download now with pip

crocs-muni · Feb 20, 2024 · b545fab · b545fab
1 parent 78d2cfd
commit b545fab
Show file tree

Hide file tree

Showing 12 changed files with 74 additions and 110 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -27,7 +27,6 @@ jobs:
       - name: Install sec-certs
         run: |
           pip install -e .
-          python -m spacy download en_core_web_sm
       - name: Run tests
         run: pytest --cov=sec_certs tests
       - name: Code coverage upload

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,12 +1,12 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.5
+    rev: v0.2.2
     hooks:
       - id: ruff
       - id: ruff-format
         args: ["--check"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.6.1"
+    rev: "v1.8.0"
     hooks:
       - id: mypy
         additional_dependencies:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -34,9 +34,12 @@ Note on single-sourcing the package version: More can be read [here](https://pac
 
 ### Currently, the release process is as follows
 
-1. Update dependencies with `pre-commit autoupdate`, pin new versions of linters into `pyproject.toml` and run `cd requirements && ./compile.sh`.
-2. Create a release from GitHub UI. Include release notes, add proper version tag and publish the release (or create it from scratch with new tag).
-3. This will automatically update PyPi and DockerHub packages.
+1. Update dependencies with `pre-commit autoupdate`, pin new versions of linters into `pyproject.toml`.
+2. Run `cd requirements && ./compile.sh` to update dependencies.
+3. Use `python -m spacy download en_core_web_sm` to find out the current version of `en_core_web_sm` dependency. Update pyproject.toml link of `en_core_web_sm` dependency with up-to-date link from [GitHub](https://github.com/explosion/spacy-models/releases).
+4. Run `cd requirements && ./compile.sh` **again** to update dependencies.
+5. Create a release from GitHub UI. Include release notes, add proper version tag and publish the release (or create it from scratch with new tag).
+6. This will automatically update PyPi and DockerHub packages.
 
 
 ## Quality assurance

diff --git a/Dockerfile b/Dockerfile
@@ -58,8 +58,7 @@ RUN \
   pip3 install -U pip wheel pip-tools && \
   pip-sync requirements/requirements.txt && \
   pip3 install --no-cache notebook jupyterlab && \
-  pip3 install -e . && \
-  python3 -m spacy download en_core_web_sm
+  pip3 install -e .
 
 # just to be sure that pdftotext is in $PATH
 ENV PATH /usr/bin/pdftotext:${PATH}

diff --git a/docs/installation.md b/docs/installation.md
@@ -7,7 +7,6 @@ The tool can be installed from PyPi with
 
 ```bash
 pip install -U sec-certs
-python -m spacy download en_core_web_sm
 ```
 
 Note, that `Python>=3.10` is required.
@@ -32,7 +31,6 @@ git clone https://github.com/crocs-muni/sec-certs.git
 python3 -m venv venv
 source venv/bin/activate
 pip install -e .
-python -m spacy download en_core_web_sm
 ```
 
 Alternatively, our Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/Dockerfile) represents a reproducible way of setting up the environment.

diff --git a/docs/quickstart.md b/docs/quickstart.md
@@ -3,7 +3,7 @@
 ::::{tab-set}
 
 :::{tab-item} Common Criteria
-1. Install the latest version with `pip install -U sec-certs && python -m spacy download en_core_web_sm` (see [installation](installation.md)).
+1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)).
 2. In your Python interpreter, type
 ```python
 from sec_certs.dataset.cc import CCDataset
@@ -16,7 +16,7 @@ to obtain to obtain freshly processed dataset from [seccerts.org](https://seccer
 :::
 
 :::{tab-item} FIPS 140
-1. Install the latest version with `pip install -U sec-certs && python -m spacy download en_core_web_sm` (see [installation](installation.md)).
+1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)).
 2. In your Python interpreter, type
 ```python
 from sec_certs.dataset.fips import FIPSDataset

diff --git a/pyproject.toml b/pyproject.toml
@@ -50,6 +50,7 @@
     "ipykernel",
     "ipywidgets",
     "spacy",
+    "en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl",
     "pkgconfig",
     "seaborn",
     "pySankeyBeta",
@@ -63,8 +64,8 @@
 
   [project.optional-dependencies]
     dev = [
-      "ruff==0.1.5",
-      "mypy==1.6.1",
+      "ruff==0.2.2",
+      "mypy==1.8.0",
       "types-PyYAML",
       "types-python-dateutil",
       "types-requests",

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
@@ -18,8 +18,6 @@ appnope==0.1.3
     #   ipython
 asttokens==2.4.1
     # via stack-data
-async-timeout==4.0.3
-    # via aiohttp
 attrs==23.1.0
     # via
     #   aiohttp
@@ -132,12 +130,10 @@ docutils==0.19
     #   myst-parser
     #   pydata-sphinx-theme
     #   sphinx
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
+    # via sec-certs (./../pyproject.toml)
 evaluate==0.4.1
     # via setfit
-exceptiongroup==1.2.0
-    # via
-    #   ipython
-    #   pytest
 executing==2.0.1
     # via stack-data
 fastjsonschema==2.19.0
@@ -309,7 +305,7 @@ murmurhash==1.0.10
     #   preshed
     #   spacy
     #   thinc
-mypy==1.6.1
+mypy==1.8.0
     # via sec-certs (./../pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
@@ -596,7 +592,7 @@ rpds-py==0.13.1
     # via
     #   jsonschema
     #   referencing
-ruff==0.1.5
+ruff==0.2.2
     # via sec-certs (./../pyproject.toml)
 safetensors==0.4.0
     # via transformers
@@ -648,7 +644,9 @@ snowballstemmer==2.2.0
 soupsieve==2.5
     # via beautifulsoup4
 spacy==3.7.2
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   en-core-web-sm
+    #   sec-certs (./../pyproject.toml)
 spacy-legacy==3.0.12
     # via spacy
 spacy-loggers==1.0.5
@@ -714,15 +712,6 @@ tifffile==2023.9.26
     # via scikit-image
 tokenizers==0.15.0
     # via transformers
-tomli==2.0.1
-    # via
-    #   build
-    #   coverage
-    #   mypy
-    #   pip-tools
-    #   pyproject-hooks
-    #   pytest
-    #   setuptools-scm
 toolz==0.12.0
     # via
     #   dask
@@ -778,7 +767,6 @@ types-requests==2.31.0.10
 typing-extensions==4.8.0
     # via
     #   alembic
-    #   cloudpathlib
     #   huggingface-hub
     #   mypy
     #   myst-nb

diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
@@ -16,8 +16,6 @@ appnope==0.1.3
     #   ipython
 asttokens==2.4.1
     # via stack-data
-async-timeout==4.0.3
-    # via aiohttp
 attrs==23.1.0
     # via
     #   aiohttp
@@ -101,10 +99,8 @@ docutils==0.19
     #   myst-parser
     #   pydata-sphinx-theme
     #   sphinx
-exceptiongroup==1.2.0
-    # via
-    #   ipython
-    #   pytest
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
+    # via sec-certs (./../pyproject.toml)
 executing==2.0.1
     # via stack-data
 fastjsonschema==2.19.0
@@ -227,7 +223,7 @@ murmurhash==1.0.10
     #   preshed
     #   spacy
     #   thinc
-mypy==1.6.1
+mypy==1.8.0
     # via sec-certs (./../pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
@@ -424,7 +420,7 @@ rpds-py==0.13.1
     # via
     #   jsonschema
     #   referencing
-ruff==0.1.5
+ruff==0.2.2
     # via sec-certs (./../pyproject.toml)
 scikit-learn==1.3.2
     # via sec-certs (./../pyproject.toml)
@@ -453,7 +449,9 @@ snowballstemmer==2.2.0
 soupsieve==2.5
     # via beautifulsoup4
 spacy==3.7.2
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   en-core-web-sm
+    #   sec-certs (./../pyproject.toml)
 spacy-legacy==3.0.12
     # via spacy
 spacy-loggers==1.0.5
@@ -508,15 +506,6 @@ thinc==8.2.1
     # via spacy
 threadpoolctl==3.2.0
     # via scikit-learn
-tomli==2.0.1
-    # via
-    #   build
-    #   coverage
-    #   mypy
-    #   pip-tools
-    #   pyproject-hooks
-    #   pytest
-    #   setuptools-scm
 tornado==6.3.3
     # via
     #   ipykernel
@@ -550,7 +539,6 @@ types-requests==2.31.0.10
     # via sec-certs (./../pyproject.toml)
 typing-extensions==4.8.0
     # via
-    #   cloudpathlib
     #   huggingface-hub
     #   mypy
     #   myst-nb

diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
@@ -14,8 +14,6 @@ appnope==0.1.3
     #   ipython
 asttokens==2.4.1
     # via stack-data
-async-timeout==4.0.3
-    # via aiohttp
 attrs==23.1.0
     # via
     #   aiohttp
@@ -103,10 +101,10 @@ dill==0.3.7
     #   multiprocess
 distro==1.8.0
     # via tabula-py
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
+    # via sec-certs (./../pyproject.toml)
 evaluate==0.4.1
     # via setfit
-exceptiongroup==1.2.0
-    # via ipython
 executing==2.0.1
     # via stack-data
 filelock==3.13.1
@@ -520,7 +518,9 @@ smart-open==6.4.0
 soupsieve==2.5
     # via beautifulsoup4
 spacy==3.7.2
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   en-core-web-sm
+    #   sec-certs (./../pyproject.toml)
 spacy-legacy==3.0.12
     # via spacy
 spacy-loggers==1.0.5
@@ -551,8 +551,6 @@ tifffile==2023.9.26
     # via scikit-image
 tokenizers==0.15.0
     # via transformers
-tomli==2.0.1
-    # via setuptools-scm
 toolz==0.12.0
     # via
     #   dask
@@ -600,7 +598,6 @@ typer==0.9.0
 typing-extensions==4.8.0
     # via
     #   alembic
-    #   cloudpathlib
     #   huggingface-hub
     #   panel
     #   pydantic