rebase development onto master

.github/workflows/ci-pipeline.yml

@@ -1,9 +1,11 @@
-name: ci pipeline
+name: CI Pipeline
 
 on:
-  - push
-  - pull_request
-  - workflow_dispatch
+  push:
+    branches: [ master, development ]
+  pull_request:
+    branches: [ master, development ]
+  workflow_dispatch:
 
 jobs:
   main:
@@ -26,9 +28,10 @@ jobs:
           sudo apt-get -y install espeak libespeak1 libespeak-dev espeak-data
           sudo apt-get -y install libsndfile-dev
           python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools wheel
           cat requirements.txt | xargs -L 1 pip install
           cat requirements-stretch.txt | xargs -L 1 pip install
-          cat requirements-translation.txt | xargs -L 1 pip install
+          cat requirements-llm.txt | xargs -L 1 pip install
           cat requirements-dev.txt | xargs -L 1 pip install
           cat requirements-site.txt | xargs -L 1 pip install
           pip install -e . --ignore-installed
@@ -43,7 +46,7 @@ jobs:
           coverage run -m unittest discover
           coverage combine
           coverage xml
-          bash <(curl -s https://codecov.io/bash)
+          bash <(curl -s https://codecov.io/bash) -n patch -F 90
       - name: Integration tests
         run: |
           radish -b tests/integration/radish tests/integration/feature

.github/workflows/dockerhub.yml

@@ -60,43 +60,30 @@ jobs:
           tags: baxtree/subaligner:${{ steps.tag.outputs.TAG }}.u20
           push: true
 
-      - name: Build and push the Ubuntu 18 image
-        id: docker_build_u18
+      - name: Build and push the Ubuntu 22 image
+        id: docker_build_u22
         uses: docker/build-push-action@v2
         with:
           context: ./docker
-          file: "./docker/Dockerfile-Ubuntu18"
+          file: "./docker/Dockerfile-Ubuntu22"
           build-args: |
             "RELEASE_VERSION=${{ steps.tag.outputs.TAG }}"
           allow: network.host
           github-token: ${{ github.token }}
-          tags: baxtree/subaligner:${{ steps.tag.outputs.TAG }}.u18
+          tags: baxtree/subaligner:${{ steps.tag.outputs.TAG }}.u22
           push: true
 
-      - name: Build and push the Fedora 31 image
-        id: docker_build_fed31
+      - name: Build and push the Fedora 34 image
+        id: docker_build_fed34
         uses: docker/build-push-action@v2
         with:
           context: ./docker
-          file: "./docker/Dockerfile-Fedora31"
+          file: "./docker/Dockerfile-Fedora34"
           build-args: |
             "RELEASE_VERSION=${{ steps.tag.outputs.TAG }}"
           allow: network.host
           github-token: ${{ github.token }}
-          tags: baxtree/subaligner:${{ steps.tag.outputs.TAG }}.fed31
-          push: true
-
-      - name: Build and push the Debian 10 image
-        id: docker_build_deb10
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          file: "./docker/Dockerfile-Debian10"
-          build-args: |
-            "RELEASE_VERSION=${{ steps.tag.outputs.TAG }}"
-          allow: network.host
-          github-token: ${{ github.token }}
-          tags: baxtree/subaligner:${{ steps.tag.outputs.TAG }}.deb10
+          tags: baxtree/subaligner:${{ steps.tag.outputs.TAG }}.fed34
           push: true
 
       - name: Build and push the ArchLinux image

CITATION.cff

@@ -5,7 +5,6 @@ authors:
     given-names: Xi
     orcid: https://orcid.org/0000-0002-2177-8458
 title: "Subaligner: Towards Automated Subtitle Alignment"
-version: 0.2.1
 doi: 10.5281/zenodo.5603083
 date-released: 2021-10-28
 url: "https://github.com/baxtree/subaligner"

Makefile

@@ -1,13 +1,13 @@
 ifdef PYTHON
 PYTHON := $(PYTHON)
 else
-PYTHON := 3.7.7
+PYTHON := 3.8.2
 endif
 
 ifdef PLATFORM
 PLATFORM := $(PLATFORM)
 else
-PLATFORM := linux-x86_64-cp-37-cp37
+PLATFORM := linux-x86_64-cp-38-cp38
 endif
 
 SUBALIGNER_VERSION := $(SUBALIGNER_VERSION)
@@ -24,14 +24,14 @@ endef
 export BROWSER_PYSCRIPT
 BROWSER := python -c "$$BROWSER_PYSCRIPT"
 
-.PHONY: install uninstall build-gzip build-rpm test test-all pydoc coverage manual dist release clean clean-dist clean-doc clean-manual clean-build clean-pyc clean-test clean-rpm
+.PHONY: install uninstall build-gzip build-rpm test test-all docker-build pydoc coverage manual dist release clean clean-dist clean-doc clean-manual clean-build clean-pyc clean-test clean-rpm
 
 install:
 	if [ ! -e ".$(PYTHON)" ]; then ~/.pyenv/versions/$(PYTHON)/bin/python3 -m venv .$(PYTHON); fi
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-stretch.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	cat requirements-translation.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
+	cat requirements-llm.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	.$(PYTHON)/bin/pip install -e . --ignore-installed
 	cp ./bin/subaligner_1pass .$(PYTHON)/bin/subaligner_1pass
@@ -55,7 +55,7 @@ install-basic:
 	.$(PYTHON)/bin/pip install -e '.' --no-cache-dir
 
 install-translation:
-	.$(PYTHON)/bin/pip install -e '.[translation]' --no-cache-dir
+	.$(PYTHON)/bin/pip install -e '.[llm]' --no-cache-dir
 
 install-stretch:
 	.$(PYTHON)/bin/pip install -e '.[stretch]' --no-cache-dir
@@ -82,7 +82,7 @@ test:
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-stretch.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	cat requirements-translation.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
+	cat requirements-llm.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .$(PYTHON)/bin/pip install
 	PYTHONPATH=. .$(PYTHON)/bin/python -m unittest discover
 	-.$(PYTHON)/bin/pycodestyle subaligner tests examples misc bin/subaligner bin/subaligner_1pass bin/subaligner_2pass bin/subaligner_batch bin/subaligner_convert bin/subaligner_train  bin/subaligner_tune setup.py --ignore=E203,E501,W503 --exclude="subaligner/lib"
@@ -95,7 +95,7 @@ test-int: ## integration test
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-stretch.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	cat requirements-translation.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
+	cat requirements-llm.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .$(PYTHON)/bin/pip install
 	.$(PYTHON)/bin/pip install -e . --ignore-installed
 	( \
@@ -108,7 +108,7 @@ pydoc: clean-doc ## generate pydoc HTML documentation based on docstrings
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-stretch.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	cat requirements-translation.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
+	cat requirements-llm.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	.$(PYTHON)/bin/python -m pydoc -w subaligner; mv subaligner.html docs/index.html
 	.$(PYTHON)/bin/python -m pydoc -w subaligner.embedder; mv subaligner.embedder.html docs
 	.$(PYTHON)/bin/python -m pydoc -w subaligner.exception; mv subaligner.exception.html docs
@@ -131,7 +131,7 @@ coverage: ## check code coverage quickly with the default Python
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-stretch.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	cat requirements-translation.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
+	cat requirements-llm.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .$(PYTHON)/bin/pip install
 	.$(PYTHON)/bin/coverage run --source subaligner -m unittest discover
 	.$(PYTHON)/bin/coverage report
@@ -149,11 +149,11 @@ manual: clean-manual ## generate manual pages
 test-dist:
 	if [ ! -e ".$(PYTHON)" ]; then ~/.pyenv/versions/$(PYTHON)/bin/python3 -m venv .$(PYTHON); fi
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
-	.$(PYTHON)/bin/pip install -e . --use-feature=2020-resolver; \
+	.$(PYTHON)/bin/pip install -e .
 
 dist: clean-dist test-dist
 	cat requirements-dev.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	.$(PYTHON)/bin/python setup.py sdist bdist_wheel bdist_egg
+	.$(PYTHON)/bin/python setup.py sdist bdist_wheel
 
 release:
 	.$(PYTHON)/bin/twine upload dist/*
@@ -167,7 +167,7 @@ profile:
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-stretch.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	cat requirements-translation.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
+	cat requirements-llm.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	cat requirements-dev.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
 	.$(PYTHON)/bin/python -c "import misc.profiler; misc.profiler.generate_profiles()"
 	.$(PYTHON)/bin/kernprof -v -l ./misc/profiler.py
@@ -176,10 +176,18 @@ app: clean-wheels
 	if [ ! -e ".$(PYTHON)" ]; then ~/.pyenv/versions/$(PYTHON)/bin/python3 -m venv .$(PYTHON); fi
 	.$(PYTHON)/bin/pip install --upgrade pip setuptools wheel; \
 	cat requirements-dev.txt | xargs -L 1 .$(PYTHON)/bin/pip install; \
-	.$(PYTHON)/bin/pip wheel --no-cache-dir --wheel-dir=./wheels -r requirements.txt -r requirements-stretch.txt -r requirements-translation.txt; \
+	.$(PYTHON)/bin/pip wheel --no-cache-dir --wheel-dir=./wheels -r requirements.txt -r requirements-stretch.txt -r requirements-llm.txt; \
 	STRETCH_OFF=True .$(PYTHON)/bin/python setup.py bdist_wheel -d ./wheels; \
 	.$(PYTHON)/bin/pex subaligner==$(SUBALIGNER_VERSION) --repo=./wheels --platform $(PLATFORM) --no-pypi --no-build --python-shebang="/usr/bin/env python3" -e subaligner -o subaligner-$(PLATFORM).app; \
 
+docker-build:
+	docker build --build-arg RELEASE_VERSION=$(SUBALIGNER_VERSION) -f docker/Dockerfile-Ubuntu20 .
+	docker build --build-arg RELEASE_VERSION=$(SUBALIGNER_VERSION) -f docker/Dockerfile-Ubuntu22 .
+	docker build --build-arg RELEASE_VERSION=$(SUBALIGNER_VERSION) -f docker/Dockerfile-ArchLinux .
+	docker build --build-arg RELEASE_VERSION=$(SUBALIGNER_VERSION) -f docker/Dockerfile-CentOS7 .
+	docker build --build-arg RELEASE_VERSION=$(SUBALIGNER_VERSION) -f docker/Dockerfile-Debian11 .
+	docker build --build-arg RELEASE_VERSION=$(SUBALIGNER_VERSION) -f docker/Dockerfile-Fedora34 .
+
 docker-images:
 	SUBALIGNER_VERSION=$(SUBALIGNER_VERSION) docker-compose -f ./docker/docker-compose.yml build
 

Pipfile

@@ -22,26 +22,21 @@ sphinx = "==3.3.1"
 sphinx-rtd-theme = "==0.5.0"
 
 [packages]
-absl-py = "~=0.10"
 astor = "==0.7.1"
 astroid = "~=2.5.6"
-audioread = "==2.1.5"
 beautifulsoup4 = "<4.9.0"
 bleach = "==3.3.0"
 cachetools = "==3.1.1"
 captionstransformer = "~=1.2.1"
 certifi = "==2019.11.28"
-chardet = "==3.0.4"
+chardet = "~=3.0.4"
 click = "==5.1"
 cloudpickle = "==0.5.3"
 cycler = "==0.10.0"
-Cython = "~=0.29.22"
-dask = "<2022.1.0"
+dask = ">=2021.10.0,<2022.1.0"
 decorator = "==4.3.0"
 distributed = "==1.13.0"
-filelock = "==3.0.12"
-google-auth = "==1.27.0"
-google-auth-oauthlib = "==0.4.2"
+filelock = "<4.0.0"
 google-pasta = "~=0.2"
 graphviz = "==0.8.3"
 HeapDict = "==1.0.0"
@@ -50,26 +45,25 @@ html5lib = "==1.0b9"
 hyperopt = "==0.2.4"
 idna = "==2.8"
 isort = "==4.3.4"
-joblib = "==0.11"
+joblib = ">=1.2.0"
 Keras-Applications = ">=1.0.8"
 Keras-Preprocessing = ">=1.0.9"
-kiwisolver = "==1.0.1"
-lazy-object-proxy = "==1.4.3"
 le-pycaption = "==2.2.0a1"
-librosa = ">=0.8.0"
+librosa = "<0.10.0"
 locket = "==0.2.0"
 Markdown = "==2.6.11"
 mccabe = "==0.6.1"
-msgpack-python = "==0.5.6"
 numba = ">=0.50.0"
 numpy = "<1.24.0"
 oauthlib = "==3.1.0"
+openai-whisper = "==20230314"
 pbr = "==4.0.2"
+pkgconfig = "~=1.5.5"
 pluggy = "==0.13.1"
-psutil = "==5.6.7"
 py = "==1.10.0"
 pyasn1 = "==0.4.8"
 pyasn1-modules = "==0.2.7"
+pycountry = "~=20.7.3"
 pydot = "==1.2.4"
 pydot-ng = "==1.0.0"
 pydotplus = "==2.0.2"
@@ -82,28 +76,24 @@ pystack-debugger = "==0.8.0"
 python-dateutil = "==2.7.2"
 pytz = "==2018.4"
 PyYAML = ">=4.2b1"
-requests = "~=2.25.1"
-requests-oauthlib = "==1.3.0"
 rsa = "==4.7"
 scipy = "<=1.8.1"
 scikit-learn = ">=0.19.1"
 sentencepiece = "~=0.1.95"
-setuptools = ">=41.0.0"
 six = "~=1.15.0"
 tblib = "==1.3.2"
-tensorflow = ">=1.15.5,<2.9"
+tensorflow = ">=1.15.5,<2.12"
 termcolor = "==1.1.0"
 toml = "==0.10.0"
 toolz = "==0.9.0"
-torch = "<=1.12.0"
+torch = "<1.13.0"
 tornado = "==5.1.0"
-transformers = "~=4.5.1"
+transformers = "<4.27.0"
 typing-extensions = "~=3.7.0"
 urllib3 = "~=1.26.5"
 Werkzeug = ">=0.15.3"
 zict = "==0.1.3"
-zipp = "==0.6.0"
 aeneas = "==1.7.3.0"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"