Update requirements, python version and dependencies (#118)

* update requirements, python version, etc. * small toml file fix * add todos to tests, fix pre-commit * fix test by removing part that is deprecated/changed * fix tests p2 * prep new version - fix tests - split up rest of work for new version later * add weekly cronjob
ing-bank · Aug 17, 2024 · 7db688c · 7db688c
1 parent ef5c1c7
commit 7db688c
Show file tree

Hide file tree

Showing 28 changed files with 188 additions and 170 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # pyproject.toml
+    schedule:
+      interval: "weekly"
diff --git a/.github/workflows/cronjob_unit_tests.yml b/.github/workflows/cronjob_unit_tests.yml
@@ -0,0 +1,57 @@
+name: Cron Test Dependencies
+
+# Controls when the action will run.
+# Every sunday at 4:05
+# See https://crontab.guru/#5 4 * * 0
+on:
+  schedule:
+    - cron: "5 4 * * 0"
+
+jobs:
+  run:
+    name: Run unit tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        build: [macos, ubuntu, windows]
+        include:
+          - build: macos
+            os: macos-latest
+          - build: ubuntu
+            os: ubuntu-latest
+          - build: windows
+            os: windows-latest
+        python-version: [3.9, "3.10", "3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Get latest CMake and Ninja
+      uses: lukka/get-cmake@latest
+      with:
+        cmakeVersion: latest
+        ninjaVersion: latest
+
+    - name: Install LIBOMP on Macos runners
+      if: runner.os == 'macOS'
+      run: |
+        brew install libomp
+
+    - name: Setup Python
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Python dependencies
+      run: |
+        pip3 install --upgrade setuptools pip
+        pip3 install ".[all]"
+
+    - name: Run linters
+      run: |
+        pre-commit install
+        pre-commit run --all-files
+
+    - name: Run (unit) tests
+      run: |
+        pytest
+        pyflakes skorecard
diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
@@ -14,7 +14,7 @@ jobs:
             os: ubuntu-latest
           - build: windows
             os: windows-latest
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.9, "3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@master
 
@@ -24,6 +24,11 @@ jobs:
         cmakeVersion: latest
         ninjaVersion: latest
 
+    - name: Install LIBOMP on Macos runners
+      if: runner.os == 'macOS'
+      run: |
+        brew install libomp
+
     - name: Setup Python
       uses: actions/setup-python@master
       with:
@@ -42,4 +47,4 @@ jobs:
     - name: Run (unit) tests
       run: |
         pytest
-        pyflakes skorecard
+        pyflakes skorecard
diff --git a/.github/workflows/unittests_with_codecov.yml b/.github/workflows/unittests_with_codecov.yml
@@ -24,7 +24,7 @@ jobs:
             os: ubuntu-latest
           - build: windows
             os: windows-latest
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+            python-version: [3.9, "3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@master
 
@@ -34,6 +34,11 @@ jobs:
         cmakeVersion: latest
         ninjaVersion: latest
 
+    - name: Install LIBOMP on Macos runners
+      if: runner.os == 'macOS'
+      run: |
+        brew install libomp
+
     - name: Setup Python
       uses: actions/setup-python@master
       with:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,7 +11,7 @@ repos:
     - id: trailing-whitespace
       name: 'Trailing whitespace: Remove empty spaces'
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.7.0
+    rev: 1.8.5
     hooks:
     - id: nbqa-ruff
       name: 'ruff nb: Check for errors, styling issues and complexity'
@@ -20,49 +20,40 @@ repos:
     - id: nbqa-isort
       name: 'isort nb: Sort file imports'
     - id: nbqa-pyupgrade
-      name: 'pyupgrade nb: Updates code to Python 3.8+ code convention'
-      args: [&py_version --py37-plus]
+      name: 'pyupgrade nb: Updates code to Python 3.9+ code convention'
+      args: [&py_version --py38-plus]
     - id: nbqa-black
       name: 'black nb: PEP8 compliant code formatter'
   - repo: local
     hooks:
-    - id: ruff
-      name: 'ruff: Check for errors, styling issues and complexity'
-      entry: ruff
-      language: system
+      - id: mypy
+        name: "mypy: Static type checking"
+        entry: mypy
+        language: system
+        types: [python]
   - repo: local
     hooks:
-    - id: mypy
-      name: 'mypy: Static type checking'
-      entry: mypy
-      language: system
-      types: [python]
+      - id: ruff-check
+        name: "Ruff: Check for errors, styling issues and complexity, and fixes issues if possible (including import order)"
+        entry: ruff check
+        language: system
+        args: [--fix, --no-cache]
+      - id: ruff-format
+        name: "Ruff: format code in line with PEP8"
+        entry: ruff format
+        language: system
+        args: [--no-cache]
   - repo: local
     hooks:
-    - id: isort
-      name: 'isort: Sort file imports'
-      entry: isort
-      language: system
-      types: [python]
-  - repo: local
-    hooks:
-    - id: codespell
-      name: 'codespell: Check for grammar'
-      entry: codespell
-      language: system
-      types: [python]
-      args: ['-L missings,bu'] # Skip the word "missings" and "bu"
+      - id: codespell
+        name: "codespell: Check for grammar"
+        entry: codespell
+        language: system
+        types: [python]
+        args: ['-L missings,bu'] # Skip the word "missings" and "bu"
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.4.0
-    hooks:
-    - id: pyupgrade
-      name: 'pyupgrade: Updates code to Python 3.8+ code convention'
-      args: [*py_version]
-  - repo: local
+    rev: v3.16.0
     hooks:
-    - id: black
-      name: 'black: PEP8 compliant code formatter'
-      entry: black
-      language: python
-      types: [python]
-      language_version: python3
+      - id: pyupgrade
+        name: "pyupgrade: Updates code to Python 3.9+ code convention"
+        args: [*py_version]
diff --git a/pyproject.toml b/pyproject.toml
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "skorecard"
 dynamic = ["version"]
-requires-python= ">=3.8"
+requires-python= ">=3.9"
 description = "Tools for building scorecard models in python, with a sklearn-compatible API"
 readme = { file = "README.md", content-type = "text/markdown" }
 authors = [
@@ -17,20 +17,20 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3 :: Only",
 ]
 dependencies = [
     "scipy>=1.5.2",
-    "numpy>=1.19.5",
+    "numpy>=1.19.5,<2.0.0",
     "pandas>=1.5.3",
     "scikit-learn>=0.23.2",
     "pyyaml",
     "category_encoders>=2.2.2",
-    "optbinning>=0.8.0",
+    "optbinning>=0.8.0", # TODO Reconsider Optbinning since its big.
 ]
 
 [project.urls]
@@ -50,14 +50,14 @@ reporting = [
 ]
 dev = [
     "black>=19.10b0",
-    "pre-commit>=2.5.0",
     "pytest>=6.0.0",
     "pytest-cov>=2.10.0",
     "pyflakes>=3.0.1",
+    "mypy>=0.770",
+    "pre-commit>=2.7.1",
     "isort>=5.12.0",
     "codespell>=2.2.4",
-    "ruff>=0.0.272",
-    "mypy>=0.770",
+    "ruff>=0.2.2",
     # Mypy type stubs
     "types-PyYAML>=6.0.12.10",
     "types-six>=1.16.21.8",
@@ -86,32 +86,34 @@ mypy = ["--install-types", "--non-interactive", "--ignore-missing-imports"]
 isort = ["--profile=black"]
 black = ["--line-length=120"]
 
+[tool.mypy]
+python_version = "3.9"
+ignore_missing_imports = true
+namespace_packages = true
+pretty = true
+
 [tool.ruff]
 line-length = 120
+extend-exclude = ["docs", "mkdocs.yml", ".github", "*md", "LICENSE", ".pre-commit-config.yaml", ".gitignore", "data", "*.css", "*.zip", "MANIFEST.in"]
+force-exclude = true
+
+[tool.ruff.lint]
 # D100 requires all Python files (modules) to have a "public" docstring even if all functions within have a docstring.
 # D104 requires __init__ files to have a docstring
 # D202 No blank lines allowed after function docstring
-# E203
 # D212
 # D200
-# W293 blank line contains whitespace
 # D411 Missing blank line before section
 # D412 No blank lines allowed between a section header and its content
 # D417 Missing argument descriptions in the docstring # Only ignored because of false positve when using multiline args.
+# E203
 # E731 do not assign a lambda expression, use a def
+# W293 blank line contains whitespace
 ignore = ["D100", "D104", "D202", "D212", "D200", "E203", "E731", "W293", "D412", "D417", "D411", "RUF100"]
-extend-exclude = ["docs", "mkdocs.yml", ".github", "*md", "LICENSE", ".pre-commit-config.yaml", ".gitignore", "data", "*.css", "*.zip", "MANIFEST.in"]
-force-exclude = true
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "google"
 
-[tool.mypy]
-python_version = "3.8"
-ignore_missing_imports = true
-namespace_packages = true
-pretty = true
-
 [tool.isort]
 line_length = "120"
 profile = "black"

diff --git a/skorecard/__init__.py b/skorecard/__init__.py
@@ -1,4 +1,4 @@
 from .skorecard import Skorecard
 
 __all__ = ["Skorecard"]
-__version__ = "1.6.8"
+__version__ = "1.6.9"
diff --git a/skorecard/apps/app_utils.py b/skorecard/apps/app_utils.py
@@ -91,9 +91,7 @@ def perc_data_bars(column):
                     #0074D9 {max_bound_percentage}%,
                     rgb(248, 248, 248) {max_bound_percentage}%,
                     rgb(248, 248, 248) 100%)
-                """.format(
-                        max_bound_percentage=max_bound_percentage
-                    )
+                """.format(max_bound_percentage=max_bound_percentage)
                 ),
                 "paddingBottom": 2,
                 "paddingTop": 2,
@@ -117,9 +115,7 @@ def perc_data_bars(column):
                     #0074D9 {max_bound_percentage}%,
                     white {max_bound_percentage}%,
                     white 100%)
-                """.format(
-                        max_bound_percentage=max_bound_percentage
-                    )
+                """.format(max_bound_percentage=max_bound_percentage)
                 ),
                 "paddingBottom": 2,
                 "paddingTop": 2,

diff --git a/skorecard/bucket_mapping.py b/skorecard/bucket_mapping.py
@@ -1,6 +1,7 @@
 """
 Classes to store features mapping for bucketing.
 """
+
 import dataclasses
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
@@ -67,9 +68,7 @@ def __post_init__(self) -> None:
         assert isinstance(self.specials, dict) or isinstance(self.specials, list)
 
         # Check specials
-        assert all(
-            [isinstance(k, str) for k in self.specials.keys()]
-        ), f"The keys of the special dictionary must be \
+        assert all([isinstance(k, str) for k in self.specials.keys()]), f"The keys of the special dictionary must be \
         strings, got {self.specials.keys()} instead."
         assert all(
             [isinstance(k, list) for k in self.specials.values()]

diff --git a/skorecard/bucketers/__init__.py b/skorecard/bucketers/__init__.py
@@ -1,6 +1,5 @@
 """Import required transformers."""
 
-
 from .bucketers import (
     OptimalBucketer,
     EqualWidthBucketer,

diff --git a/skorecard/bucketers/base_bucketer.py b/skorecard/bucketers/base_bucketer.py
@@ -78,15 +78,15 @@ def _is_allowed_missing_treatment(missing_treatment):
             "passthrough",
         ]
 
-        if type(missing_treatment) == str:
+        if isinstance(missing_treatment, str):
             if missing_treatment not in allowed_str_missing:
                 raise ValueError(f"missing_treatment must be in {allowed_str_missing} or a dict")
 
-        elif type(missing_treatment) == dict:
+        elif isinstance(missing_treatment, dict):
             for _, v in enumerate(missing_treatment):
                 if missing_treatment[v] < 0:
                     raise ValueError("As an integer, missing_treatment must be greater than 0")
-                elif type(missing_treatment[v]) != int:
+                elif not isinstance(missing_treatment[v], int):
                     raise ValueError("Values of the missing_treatment dict must be integers")
 
         else:

diff --git a/skorecard/linear_model/linear_model.py b/skorecard/linear_model/linear_model.py
@@ -139,7 +139,7 @@ def fit(self, X, y, sample_weight=None, calculate_stats=False, **kwargs):
         else:
             X_design = X
 
-        p = np.product(predProbs, axis=1)
+        p = np.prod(predProbs, axis=1)
         self.cov_matrix_ = np.linalg.inv((X_design * p[..., np.newaxis]).T @ X_design)
         std_err = np.sqrt(np.diag(self.cov_matrix_)).reshape(1, -1)
 

diff --git a/skorecard/metrics/__init__.py b/skorecard/metrics/__init__.py
@@ -1,4 +1,5 @@
 """Import required Metric."""
+
 from .metrics import IV_scorer