diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2d21f4..0d4f2a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,13 @@ on: type: 'choice' options: ['no', 'pypi', 'testpypi'] + publish_npm: + description: 'whether to publish the npm packages' + required: false + default: 'no' + type: 'choice' + options: ['no', 'yes'] + pull_request: branches: - main @@ -88,9 +95,6 @@ jobs: if: ${{ github.event.inputs.upload_wheel != 'no' && github.event_name != 'pull_request' }} needs: build_wheels runs-on: ubuntu-latest - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-13, macos-14] steps: - uses: actions/checkout@v4 @@ -98,10 +102,9 @@ jobs: - uses: benjlevesque/short-sha@v3.0 id: short_sha - # Download artifact + # Download all artifacts - uses: actions/download-artifact@v4 with: - name: vectorlite-wheel-${{ matrix.os }}-${{ steps.short_sha.outputs.sha }} path: ./wheelhouse - name: Upload to test.pypi.org @@ -122,4 +125,38 @@ jobs: if: ${{ github.event.inputs.upload_wheel == 'pypi' && !startsWith(github.ref, 'refs/tags/v') }} run: | echo "Error: Uploading to pypi.org requires a tag" - exit 1 \ No newline at end of file + exit 1 + + publish_npm_pkgs: + name: Upload wheels + if: ${{ github.event.inputs.publish_npm == 'yes' && github.event_name != 'pull_request' }} + needs: build_wheels + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: benjlevesque/short-sha@v3.0 + id: short_sha + + # Download all artifacts + - uses: actions/download-artifact@v4 + with: + path: ./wheelhouse + + # extract vectorlite from wheels and copy to nodejs bindings directory + - name: unzip wheels + run: | + sh extract_wheels.sh + + - uses: actions/setup-node@v4 + with: + node-version: lts/* + + - name: Publish to npm + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + run: + cd bindings/nodejs + npm publish --workspaces + diff --git a/.gitignore b/.gitignore index 9727cf4..5281674 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,7 @@ vcpkg_installed/* *.pyc dist/* -*egg-info \ No newline at end of file +*egg-info + +node_modules +bindings/nodejs/vectorlite/package-lock.json \ No newline at end of file diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json new file mode 100644 index 0000000..52821f9 --- /dev/null +++ b/bindings/nodejs/package.json @@ -0,0 +1,4 @@ +{ + "name": "vectorlite-workspaces", + "workspaces": ["packages/vectorlite-win32-x64", "packages/vectorlite-linux-x64", "packages/vectorlite-darwin-x64", "packages/vectorlite-darwin-arm64", "packages/vectorlite"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/README.md b/bindings/nodejs/packages/README.md new file mode 100644 index 0000000..36f97c3 --- /dev/null +++ b/bindings/nodejs/packages/README.md @@ -0,0 +1 @@ +This folder hosts nodejs bindings for vectorlite. `vectorlite.[so|dll|dylib]` is copied to their own platform dependent package folder. Please check ci.yml for details. \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-arm64/package.json b/bindings/nodejs/packages/vectorlite-darwin-arm64/package.json new file mode 100644 index 0000000..b8f6bb6 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-arm64/package.json @@ -0,0 +1,12 @@ +{ + "name": "vectorlite-darwin-arm64", + "version": "0.1.0", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["darwin"], + "cpu": ["arm64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js b/bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-x64/package.json b/bindings/nodejs/packages/vectorlite-darwin-x64/package.json new file mode 100644 index 0000000..cd9dfaf --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-x64/package.json @@ -0,0 +1,12 @@ +{ + "name": "vectorlite-darwin-x64", + "version": "0.1.0", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["darwin"], + "cpu": ["x64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js b/bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-linux-x64/package.json b/bindings/nodejs/packages/vectorlite-linux-x64/package.json new file mode 100644 index 0000000..1249be5 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-linux-x64/package.json @@ -0,0 +1,12 @@ +{ + "name": "vectorlite-linux-x64", + "version": "0.1.0", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["linux"], + "cpu": ["x64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-linux-x64/src/index.js b/bindings/nodejs/packages/vectorlite-linux-x64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-linux-x64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-win32-x64/package.json b/bindings/nodejs/packages/vectorlite-win32-x64/package.json new file mode 100644 index 0000000..f109c69 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-win32-x64/package.json @@ -0,0 +1,12 @@ +{ + "name": "vectorlite-win32-x64", + "version": "0.1.0", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["win32"], + "cpu": ["x64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-win32-x64/src/index.js b/bindings/nodejs/packages/vectorlite-win32-x64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-win32-x64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite/README.md b/bindings/nodejs/packages/vectorlite/README.md new file mode 100644 index 0000000..09bc504 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/README.md @@ -0,0 +1 @@ +# `vectorlite` for nodejs \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite/package.json b/bindings/nodejs/packages/vectorlite/package.json new file mode 100644 index 0000000..88ef686 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/package.json @@ -0,0 +1,22 @@ +{ + "name": "vectorlite", + "version": "0.1.0", + "main": "src/index.js", + "files": ["src"], + "scripts": { + "test": "node test/test.js" + }, + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "devDependencies": { + "better-sqlite3": "^11.1.2" + }, + "optionalDependencies": { + "vectorlite-darwin-x64": "0.1.0", + "vectorlite-darwin-arm64": "0.1.0", + "vectorlite-linux-x64": "file://../vectorlite-linux-x64", + "vectorlite-win32-x64": "0.1.0" + } +} diff --git a/bindings/nodejs/packages/vectorlite/src/index.js b/bindings/nodejs/packages/vectorlite/src/index.js new file mode 100644 index 0000000..e9d0f54 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/src/index.js @@ -0,0 +1,28 @@ +const os = require('os'); + +const supportedPlatformsAndArchs = { + 'darwin-x64': 'vectorlite-darwin-x64', + 'darwin-arm64': 'vectorlite-darwin-arm64', + 'linux-x64': 'vectorlite-linux-x64', + 'win32-x64': 'vectorlite-win32-x64', +}; + +const platformAndArch = `${os.platform()}-${os.arch()}`; + +let vectorlitePathCache = undefined; + +// Returns path to the vectorlite shared library +function vectorlitePath() { + if (vectorlitePathCache) { + return vectorlitePathCache; + } + const packageName = supportedPlatformsAndArchs[platformAndArch]; + if (!packageName) { + throw new Error(`Platform ${platformAndArch} is not supported`); + } + const package = require(packageName); + vectorlitePathCache = package.vectorlitePath(); + return vectorlitePathCache; +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite/test/test.js b/bindings/nodejs/packages/vectorlite/test/test.js new file mode 100644 index 0000000..3a213ea --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/test/test.js @@ -0,0 +1,19 @@ +const sqlite3 = require('better-sqlite3'); +const vectorlite = require('../src/index.js'); + +const db = new sqlite3(':memory:'); +db.loadExtension(vectorlite.vectorlitePath()); + +console.log(db.prepare('select vectorlite_info()').all()); + +db.exec('create virtual table test using vectorlite(vec float32[10], hnsw(max_elements=100));') + +// insert a json vector +db.prepare('insert into test(rowid, vec) values (?, vector_from_json(?))').run([0, JSON.stringify(Array.from({length: 10}, () => Math.random()))]); +// insert a raw vector +db.prepare('insert into test(rowid, vec) values (?, ?)').run([1, Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +const result = db.run('select rowid from test where knn_search(vec, knn_param(?, 2))') + .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +console.log(result); \ No newline at end of file diff --git a/extract_wheels.sh b/extract_wheels.sh new file mode 100644 index 0000000..496004f --- /dev/null +++ b/extract_wheels.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +for wheel in dist/*.whl; do + unziped_dir=$wheel.unzipped + unzip $wheel -d $unziped_dir + + case "$wheel" in + *linux*x86_64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.so bindings/nodejs/packages/vectorlite-linux-x64/src + ;; + *win*amd64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.dll bindings/nodejs/packages/vectorlite-win32-x64/src + ;; + *macosx*arm64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.dylib bindings/nodejs/packages/vectorlite-darwin-arm64/src + ;; + *macosx*x86_64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.dylib bindings/nodejs/packages/vectorlite-darwin-x64/src + ;; + *) + echo "Unknown wheel type: $wheel" + exit 1 + ;; + esac +done +