From 4f6535c99ba5972719c5dac0b98373c4a6c4cb0c Mon Sep 17 00:00:00 2001 From: Swoorup Joshi Date: Wed, 5 Jun 2024 02:42:26 +1000 Subject: [PATCH] Organise workspace crates and consolidate Cargo.toml manifest properties/dependencies (#285) * Organise workspace crates and unify Cargo.toml manifests * Update code coverage and contributing.md * double choco openssl install timeout * num_integer is not optional * move httpfs later...? * before * include dir * add note * fix memory leak in api usage test * specify crates explicitly * swap action * I guess it just doesnt work with workspace members * swap again * swap yet again * ignore unpublished change * typo --------- Co-authored-by: Max Gabrielsson Co-authored-by: Max Gabrielsson --- .github/.codecov.yml | 4 +- .github/workflows/release.yaml | 6 +- .github/workflows/rust.yaml | 6 +- .gitignore | 6 +- .gitmodules | 4 +- CONTRIBUTING.md | 8 +- Cargo.toml | 130 +++++++----------- crates/duckdb-loadable-macros/Cargo.toml | 22 +++ crates/duckdb-loadable-macros/LICENSE | 1 + crates/duckdb-loadable-macros/README.md | 1 + .../duckdb-loadable-macros}/src/lib.rs | 0 crates/duckdb/Cargo.toml | 93 +++++++++++++ .../examples}/Movies_Social_metadata.xlsx | Bin .../duckdb/examples}/appender.rs | 0 {examples => crates/duckdb/examples}/basic.rs | 0 .../duckdb/examples}/date.xlsx | Bin .../duckdb/examples}/hello-ext/main.rs | 0 .../duckdb/examples}/int32_decimal.parquet | Bin .../duckdb/examples}/parquet.rs | 0 {src => crates/duckdb/src}/appender/arrow.rs | 0 {src => crates/duckdb/src}/appender/mod.rs | 0 {src => crates/duckdb/src}/appender_params.rs | 0 {src => crates/duckdb/src}/arrow_batch.rs | 0 {src => crates/duckdb/src}/cache.rs | 0 {src => crates/duckdb/src}/column.rs | 0 {src => crates/duckdb/src}/config.rs | 0 {src => crates/duckdb/src}/error.rs | 0 {src => crates/duckdb/src}/extension.rs | 0 .../duckdb/src}/inner_connection.rs | 0 {src => crates/duckdb/src}/lib.rs | 2 +- {src => crates/duckdb/src}/params.rs | 0 .../duckdb/src}/polars_dataframe.rs | 0 {src => crates/duckdb/src}/pragma.rs | 0 {src => crates/duckdb/src}/r2d2.rs | 0 {src => crates/duckdb/src}/raw_statement.rs | 0 {src => crates/duckdb/src}/row.rs | 3 +- {src => crates/duckdb/src}/statement.rs | 0 {src => crates/duckdb/src}/test_all_types.rs | 0 {src => crates/duckdb/src}/transaction.rs | 0 {src => crates/duckdb/src}/types/chrono.rs | 0 {src => crates/duckdb/src}/types/from_sql.rs | 0 {src => crates/duckdb/src}/types/mod.rs | 0 .../duckdb/src}/types/serde_json.rs | 0 {src => crates/duckdb/src}/types/to_sql.rs | 0 {src => crates/duckdb/src}/types/url.rs | 0 {src => crates/duckdb/src}/types/value.rs | 0 {src => crates/duckdb/src}/types/value_ref.rs | 6 +- {src => crates/duckdb/src}/util/mod.rs | 0 {src => crates/duckdb/src}/util/small_cstr.rs | 0 {src => crates/duckdb/src}/vtab/arrow.rs | 0 {src => crates/duckdb/src}/vtab/data_chunk.rs | 0 {src => crates/duckdb/src}/vtab/excel.rs | 0 {src => crates/duckdb/src}/vtab/function.rs | 0 .../duckdb/src}/vtab/logical_type.rs | 0 {src => crates/duckdb/src}/vtab/mod.rs | 0 {src => crates/duckdb/src}/vtab/value.rs | 0 {src => crates/duckdb/src}/vtab/vector.rs | 0 crates/libduckdb-sys/Cargo.toml | 42 ++++++ crates/libduckdb-sys/LICENSE | 1 + crates/libduckdb-sys/README.md | 1 + .../libduckdb-sys}/bindgen.sh | 0 .../libduckdb-sys}/build.rs | 19 +-- .../libduckdb-sys}/duckdb.tar.gz | Bin .../libduckdb-sys}/openssl/cfgs.rs | 0 .../libduckdb-sys}/openssl/expando.c | 0 .../libduckdb-sys}/openssl/find_normal.rs | 0 .../libduckdb-sys}/openssl/find_vendored.rs | 0 .../libduckdb-sys}/openssl/mod.rs | 0 .../libduckdb-sys}/openssl/run_bindgen.rs | 0 .../src/bindgen_bundled_version.rs | 0 .../libduckdb-sys}/src/error.rs | 0 .../libduckdb-sys}/src/lib.rs | 21 ++- .../libduckdb-sys}/src/raw_statement.rs | 0 .../libduckdb-sys}/update_sources.py | 0 .../libduckdb-sys}/upgrade.sh | 0 .../libduckdb-sys}/wrapper.h | 0 duckdb-loadable-macros/Cargo.toml | 22 --- duckdb-loadable-macros/LICENSE | 1 - duckdb-loadable-macros/README.md | 1 - libduckdb-sys/Cargo.toml | 42 ------ libduckdb-sys/LICENSE | 1 - libduckdb-sys/README.md | 1 - libduckdb-sys/duckdb-sources | 1 - 83 files changed, 249 insertions(+), 196 deletions(-) create mode 100644 crates/duckdb-loadable-macros/Cargo.toml create mode 120000 crates/duckdb-loadable-macros/LICENSE create mode 120000 crates/duckdb-loadable-macros/README.md rename {duckdb-loadable-macros => crates/duckdb-loadable-macros}/src/lib.rs (100%) create mode 100644 crates/duckdb/Cargo.toml rename {examples => crates/duckdb/examples}/Movies_Social_metadata.xlsx (100%) rename {examples => crates/duckdb/examples}/appender.rs (100%) rename {examples => crates/duckdb/examples}/basic.rs (100%) rename {examples => crates/duckdb/examples}/date.xlsx (100%) rename {examples => crates/duckdb/examples}/hello-ext/main.rs (100%) rename {examples => crates/duckdb/examples}/int32_decimal.parquet (100%) rename {examples => crates/duckdb/examples}/parquet.rs (100%) rename {src => crates/duckdb/src}/appender/arrow.rs (100%) rename {src => crates/duckdb/src}/appender/mod.rs (100%) rename {src => crates/duckdb/src}/appender_params.rs (100%) rename {src => crates/duckdb/src}/arrow_batch.rs (100%) rename {src => crates/duckdb/src}/cache.rs (100%) rename {src => crates/duckdb/src}/column.rs (100%) rename {src => crates/duckdb/src}/config.rs (100%) rename {src => crates/duckdb/src}/error.rs (100%) rename {src => crates/duckdb/src}/extension.rs (100%) rename {src => crates/duckdb/src}/inner_connection.rs (100%) rename {src => crates/duckdb/src}/lib.rs (99%) rename {src => crates/duckdb/src}/params.rs (100%) rename {src => crates/duckdb/src}/polars_dataframe.rs (100%) rename {src => crates/duckdb/src}/pragma.rs (100%) rename {src => crates/duckdb/src}/r2d2.rs (100%) rename {src => crates/duckdb/src}/raw_statement.rs (100%) rename {src => crates/duckdb/src}/row.rs (99%) rename {src => crates/duckdb/src}/statement.rs (100%) rename {src => crates/duckdb/src}/test_all_types.rs (100%) rename {src => crates/duckdb/src}/transaction.rs (100%) rename {src => crates/duckdb/src}/types/chrono.rs (100%) rename {src => crates/duckdb/src}/types/from_sql.rs (100%) rename {src => crates/duckdb/src}/types/mod.rs (100%) rename {src => crates/duckdb/src}/types/serde_json.rs (100%) rename {src => crates/duckdb/src}/types/to_sql.rs (100%) rename {src => crates/duckdb/src}/types/url.rs (100%) rename {src => crates/duckdb/src}/types/value.rs (100%) rename {src => crates/duckdb/src}/types/value_ref.rs (98%) rename {src => crates/duckdb/src}/util/mod.rs (100%) rename {src => crates/duckdb/src}/util/small_cstr.rs (100%) rename {src => crates/duckdb/src}/vtab/arrow.rs (100%) rename {src => crates/duckdb/src}/vtab/data_chunk.rs (100%) rename {src => crates/duckdb/src}/vtab/excel.rs (100%) rename {src => crates/duckdb/src}/vtab/function.rs (100%) rename {src => crates/duckdb/src}/vtab/logical_type.rs (100%) rename {src => crates/duckdb/src}/vtab/mod.rs (100%) rename {src => crates/duckdb/src}/vtab/value.rs (100%) rename {src => crates/duckdb/src}/vtab/vector.rs (100%) create mode 100644 crates/libduckdb-sys/Cargo.toml create mode 120000 crates/libduckdb-sys/LICENSE create mode 120000 crates/libduckdb-sys/README.md rename {libduckdb-sys => crates/libduckdb-sys}/bindgen.sh (100%) rename {libduckdb-sys => crates/libduckdb-sys}/build.rs (98%) rename {libduckdb-sys => crates/libduckdb-sys}/duckdb.tar.gz (100%) rename {libduckdb-sys => crates/libduckdb-sys}/openssl/cfgs.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/openssl/expando.c (100%) rename {libduckdb-sys => crates/libduckdb-sys}/openssl/find_normal.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/openssl/find_vendored.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/openssl/mod.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/openssl/run_bindgen.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/src/bindgen_bundled_version.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/src/error.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/src/lib.rs (93%) rename {libduckdb-sys => crates/libduckdb-sys}/src/raw_statement.rs (100%) rename {libduckdb-sys => crates/libduckdb-sys}/update_sources.py (100%) rename {libduckdb-sys => crates/libduckdb-sys}/upgrade.sh (100%) rename {libduckdb-sys => crates/libduckdb-sys}/wrapper.h (100%) delete mode 100644 duckdb-loadable-macros/Cargo.toml delete mode 120000 duckdb-loadable-macros/LICENSE delete mode 120000 duckdb-loadable-macros/README.md delete mode 100644 libduckdb-sys/Cargo.toml delete mode 120000 libduckdb-sys/LICENSE delete mode 120000 libduckdb-sys/README.md delete mode 160000 libduckdb-sys/duckdb-sources diff --git a/.github/.codecov.yml b/.github/.codecov.yml index ab126bc2..e8c58cf3 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -1,6 +1,6 @@ ignore: - - "libduckdb-sys/duckdb" - - "libduckdb-sys/src/bindgen_bundled_version.rs" + - "crates/libduckdb-sys/duckdb" + - "crates/libduckdb-sys/src/bindgen_bundled_version.rs" coverage: status: project: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 065f1737..91f4c6ae 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,13 +20,13 @@ jobs: components: 'rustfmt, clippy' # cargo publish - - uses: wangfenjin/publish-crates@main - name: cargo publish + - name: publish crates + uses: katyo/publish-crates@v2 with: path: './' args: --no-verify --allow-dirty --all-features registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} - ignore-published: true + ignore-unpublished-changes: true # create release - name: "Build Changelog" diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 0ddf6416..3928a586 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -145,10 +145,10 @@ jobs: # as the other tests have them. RUST_BACKTRACE: "0" run: cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu - - uses: wangfenjin/publish-crates@main - name: cargo publish --dry-run + - name: publish crates --dry-run + uses: katyo/publish-crates@v2 with: path: './' args: --allow-dirty --all-features dry-run: true - ignore-published: true + ignore-unpublished-changes: true diff --git a/.gitignore b/.gitignore index 0b6b9228..02483800 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,6 @@ Cargo.lock *.db -libduckdb-sys/duckdb-sources/ -libduckdb-sys/duckdb/ -libduckdb-sys/._duckdb +crates/libduckdb-sys/duckdb-sources/ +crates/libduckdb-sys/duckdb/ +crates/libduckdb-sys/._duckdb diff --git a/.gitmodules b/.gitmodules index 4754e802..c35364a0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ -[submodule "libduckdb-sys/duckdb-sources"] - path = libduckdb-sys/duckdb-sources +[submodule "crates/libduckdb-sys/duckdb-sources"] + path = crates/libduckdb-sys/duckdb-sources url = https://github.com/duckdb/duckdb update = none diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bac02655..16bc8bed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,24 +43,24 @@ export DUCKDB_INCLUDE_DIR=~/duckdb-lib Use the exported library and header: ```shell -cd ~/github/duckdb-rs/libduckdb-sys +cd ~/github/duckdb-rs/crates/libduckdb-sys cargo test --features buildtime_bindgen ``` Use the bundled header file: ```shell -cd ~/github/duckdb-rs/libduckdb-sys +cd ~/github/duckdb-rs/crates/libduckdb-sys cargo test --features bundled ``` -Currently in [github actions](https://github.com/wangfenjin/duckdb-rs/actions), we always use the bundled file for testing. So if you change the header in duckdb-cpp repo, you need to make the PR merged and updated the [bundled-file](https://github.com/wangfenjin/duckdb-rs/tree/main/libduckdb-sys/duckdb). +Currently in [github actions](https://github.com/wangfenjin/duckdb-rs/actions), we always use the bundled file for testing. So if you change the header in duckdb-cpp repo, you need to make the PR merged and updated the [bundled-file](https://github.com/wangfenjin/duckdb-rs/tree/main/crates/libduckdb-sys/duckdb). You can generated the amalgamated file by: ```shell cd ~/github/duckdb mkdir -p build/amaldebug python scripts/amalgamation.py -cp src/amalgamation/duckdb.cpp src/include/duckdb.h src/amalgamation/duckdb.hpp ../duckdb-rs/libduckdb-sys/duckdb/ +cp src/amalgamation/duckdb.cpp src/include/duckdb.h src/amalgamation/duckdb.hpp ../duckdb-rs/crates/libduckdb-sys/duckdb/ ``` ### duckdb-rs diff --git a/Cargo.toml b/Cargo.toml index 99c4f03a..5f176778 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,15 @@ -[package] -name = "duckdb" +[workspace] +resolver = "2" +members = [ + "crates/duckdb", + "crates/libduckdb-sys", + "crates/duckdb-loadable-macros" +] + +[workspace.package] version = "0.10.2" authors = ["wangfenjin "] edition = "2021" -description = "Ergonomic wrapper for DuckDB" repository = "https://github.com/wangfenjin/duckdb-rs" homepage = "https://github.com/wangfenjin/duckdb-rs" documentation = "http://docs.rs/duckdb/" @@ -12,90 +18,46 @@ keywords = ["duckdb", "database", "ffi"] license = "MIT" categories = ["database"] -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -name = "duckdb" - -[workspace] -members = ["libduckdb-sys", "duckdb-loadable-macros"] - -[features] -default = [] -bundled = ["libduckdb-sys/bundled"] -httpfs = ["libduckdb-sys/httpfs", "bundled"] -json = ["libduckdb-sys/json", "bundled"] -parquet = ["libduckdb-sys/parquet", "bundled"] -vtab = [] -vtab-loadable = ["vtab", "duckdb-loadable-macros"] -vtab-excel = ["vtab", "calamine"] -vtab-arrow = ["vtab", "num"] -appender-arrow = ["vtab-arrow"] -vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"] -extensions-full = ["httpfs", "json", "parquet", "vtab-full"] -buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"] -modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"] -polars = ["dep:polars"] -chrono = ["dep:chrono", "num-integer"] - -[dependencies] -# time = { version = "0.3.2", features = ["formatting", "parsing"], optional = true } -hashlink = "0.8" -chrono = { version = "0.4.22", optional = true } -serde_json = { version = "1.0", optional = true } -csv = { version = "1.1", optional = true } -url = { version = "2.1", optional = true } -lazy_static = { version = "1.4", optional = true } -byteorder = { version = "1.3", features = ["i128"], optional = true } +[workspace.dependencies] +duckdb = { version = "0.10.2", path = "crates/duckdb" } +libduckdb-sys = { version = "0.10.2", path = "crates/libduckdb-sys" } +duckdb-loadable-macros = { version = "0.1.1", path = "crates/duckdb-loadable-macros" } +autocfg = "1.0" +bindgen = { version = "0.69", default-features = false } +byteorder = "1.3" +calamine = "0.22.0" +cast = "0.3" +cc = "1.0" +chrono = "0.4.22" +csv = "1.1" +doc-comment = "0.3" fallible-iterator = "0.3" fallible-streaming-iterator = "0.1" +flate2 = "1.0" +hashlink = "0.8" +lazy_static = "1.4" memchr = "2.3" -uuid = { version = "1.0", optional = true } -smallvec = "1.6.1" -cast = { version = "0.3", features = ["std"] } -arrow = { version = "51", default-features = false, features = ["prettyprint", "ffi"] } +num = { version = "0.4", default-features = false } +pkg-config = "0.3.24" +polars = "0.35.4" +polars-core = "0.35.4" +pretty_assertions = "1.4.0" +proc-macro2 = "1.0.56" +quote = "1.0.21" +r2d2 = "0.8.9" +rand = "0.8.3" +regex = "1.6" rust_decimal = "1.14" -strum = { version = "0.25", features = ["derive"] } -r2d2 = { version = "0.8.9", optional = true } -calamine = { version = "0.22.0", optional = true } -num = { version = "0.4", optional = true, default-features = false, features = ["std"] } -duckdb-loadable-macros = { version = "0.1.1", path="./duckdb-loadable-macros", optional = true } -polars = { version = "0.35.4", features = ["dtype-full"], optional = true} -num-integer = {version = "0.1.46", optional = true} - -[dev-dependencies] -doc-comment = "0.3" +serde = "1.0" +serde_json = "1.0" +smallvec = "1.6.1" +strum = "0.25" +syn = "2.0.15" +tar = "0.4.38" +tempdir = "0.3.7" tempfile = "3.1.0" -lazy_static = "1.4" -regex = "1.6" -uuid = { version = "1.0", features = ["v4"] } unicase = "2.6.0" -rand = "0.8.3" -tempdir = "0.3.7" -polars-core = "0.35.4" -pretty_assertions = "1.4.0" -# criterion = "0.3" - -# [[bench]] -# name = "data_types" -# harness = false - -[dependencies.libduckdb-sys] -path = "libduckdb-sys" -version = "0.10.2" - - -[package.metadata.docs.rs] -features = ['vtab', 'chrono'] -all-features = false -no-default-features = true -default-target = "x86_64-unknown-linux-gnu" - -[package.metadata.playground] -features = [] -all-features = false - -[[example]] -name = "hello-ext" -crate-type = ["cdylib"] -required-features = ["vtab-loadable"] +url = "2.1" +uuid = "1.0" +vcpkg = "0.2" +arrow = { version = "51", default-features = false } diff --git a/crates/duckdb-loadable-macros/Cargo.toml b/crates/duckdb-loadable-macros/Cargo.toml new file mode 100644 index 00000000..98ccaaf0 --- /dev/null +++ b/crates/duckdb-loadable-macros/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "duckdb-loadable-macros" +version = "0.1.1" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +keywords.workspace = true +readme.workspace = true +categories = ["external-ffi-bindings", "database"] +description = "Native bindings to the libduckdb library, C API; build loadable extensions" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true, features = ["extra-traits", "full", "fold", "parsing"] } + +[lib] +proc-macro = true diff --git a/crates/duckdb-loadable-macros/LICENSE b/crates/duckdb-loadable-macros/LICENSE new file mode 120000 index 00000000..30cff740 --- /dev/null +++ b/crates/duckdb-loadable-macros/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/duckdb-loadable-macros/README.md b/crates/duckdb-loadable-macros/README.md new file mode 120000 index 00000000..fe840054 --- /dev/null +++ b/crates/duckdb-loadable-macros/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/duckdb-loadable-macros/src/lib.rs b/crates/duckdb-loadable-macros/src/lib.rs similarity index 100% rename from duckdb-loadable-macros/src/lib.rs rename to crates/duckdb-loadable-macros/src/lib.rs diff --git a/crates/duckdb/Cargo.toml b/crates/duckdb/Cargo.toml new file mode 100644 index 00000000..5e45b701 --- /dev/null +++ b/crates/duckdb/Cargo.toml @@ -0,0 +1,93 @@ +[package] +name = "duckdb" +version = "0.10.2" +authors.workspace = true +edition.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +readme.workspace = true +keywords.workspace = true +license.workspace = true +categories.workspace = true +description = "Ergonomic wrapper for DuckDB" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +name = "duckdb" + +[features] +default = [] +bundled = ["libduckdb-sys/bundled"] +httpfs = ["libduckdb-sys/httpfs", "bundled"] +json = ["libduckdb-sys/json", "bundled"] +parquet = ["libduckdb-sys/parquet", "bundled"] +vtab = [] +vtab-loadable = ["vtab", "duckdb-loadable-macros"] +vtab-excel = ["vtab", "calamine"] +vtab-arrow = ["vtab", "num"] +appender-arrow = ["vtab-arrow"] +vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"] +extensions-full = ["httpfs", "json", "parquet", "vtab-full"] +buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"] +modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"] +polars = ["dep:polars"] + +[dependencies] +libduckdb-sys = { workspace = true } +hashlink = { workspace = true } +chrono = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +csv = { workspace = true, optional = true } +url = { workspace = true, optional = true } +lazy_static = { workspace = true, optional = true } +byteorder = { workspace = true, features = ["i128"], optional = true } +fallible-iterator = { workspace = true } +fallible-streaming-iterator = { workspace = true } +memchr = { workspace = true } +uuid = { workspace = true, optional = true } +smallvec = { workspace = true } +cast = { workspace = true, features = ["std"] } +arrow = { workspace = true, features = ["prettyprint", "ffi"] } +rust_decimal = { workspace = true } +strum = { workspace = true, features = ["derive"] } +r2d2 = { workspace = true, optional = true } +calamine = { workspace = true, optional = true } +num = { workspace = true, features = ["std"], optional = true } +duckdb-loadable-macros = { workspace = true, optional = true } +polars = { workspace = true, features = ["dtype-full"], optional = true } +num-integer = {version = "0.1.46"} + +[dev-dependencies] +doc-comment = { workspace = true } +tempfile = { workspace = true } +lazy_static = { workspace = true } +regex = { workspace = true } +uuid = { workspace = true, features = ["v4"] } +unicase = { workspace = true } +rand = { workspace = true } +tempdir = { workspace = true } +polars-core = { workspace = true } +pretty_assertions = { workspace = true } +# criterion = "0.3" + +# [[bench]] +# name = "data_types" +# harness = false + + +[package.metadata.docs.rs] +features = [] +all-features = false +no-default-features = true +default-target = "x86_64-unknown-linux-gnu" + +[package.metadata.playground] +features = [] +all-features = false + +[[example]] +name = "hello-ext" +crate-type = ["cdylib"] +required-features = ["vtab-loadable"] diff --git a/examples/Movies_Social_metadata.xlsx b/crates/duckdb/examples/Movies_Social_metadata.xlsx similarity index 100% rename from examples/Movies_Social_metadata.xlsx rename to crates/duckdb/examples/Movies_Social_metadata.xlsx diff --git a/examples/appender.rs b/crates/duckdb/examples/appender.rs similarity index 100% rename from examples/appender.rs rename to crates/duckdb/examples/appender.rs diff --git a/examples/basic.rs b/crates/duckdb/examples/basic.rs similarity index 100% rename from examples/basic.rs rename to crates/duckdb/examples/basic.rs diff --git a/examples/date.xlsx b/crates/duckdb/examples/date.xlsx similarity index 100% rename from examples/date.xlsx rename to crates/duckdb/examples/date.xlsx diff --git a/examples/hello-ext/main.rs b/crates/duckdb/examples/hello-ext/main.rs similarity index 100% rename from examples/hello-ext/main.rs rename to crates/duckdb/examples/hello-ext/main.rs diff --git a/examples/int32_decimal.parquet b/crates/duckdb/examples/int32_decimal.parquet similarity index 100% rename from examples/int32_decimal.parquet rename to crates/duckdb/examples/int32_decimal.parquet diff --git a/examples/parquet.rs b/crates/duckdb/examples/parquet.rs similarity index 100% rename from examples/parquet.rs rename to crates/duckdb/examples/parquet.rs diff --git a/src/appender/arrow.rs b/crates/duckdb/src/appender/arrow.rs similarity index 100% rename from src/appender/arrow.rs rename to crates/duckdb/src/appender/arrow.rs diff --git a/src/appender/mod.rs b/crates/duckdb/src/appender/mod.rs similarity index 100% rename from src/appender/mod.rs rename to crates/duckdb/src/appender/mod.rs diff --git a/src/appender_params.rs b/crates/duckdb/src/appender_params.rs similarity index 100% rename from src/appender_params.rs rename to crates/duckdb/src/appender_params.rs diff --git a/src/arrow_batch.rs b/crates/duckdb/src/arrow_batch.rs similarity index 100% rename from src/arrow_batch.rs rename to crates/duckdb/src/arrow_batch.rs diff --git a/src/cache.rs b/crates/duckdb/src/cache.rs similarity index 100% rename from src/cache.rs rename to crates/duckdb/src/cache.rs diff --git a/src/column.rs b/crates/duckdb/src/column.rs similarity index 100% rename from src/column.rs rename to crates/duckdb/src/column.rs diff --git a/src/config.rs b/crates/duckdb/src/config.rs similarity index 100% rename from src/config.rs rename to crates/duckdb/src/config.rs diff --git a/src/error.rs b/crates/duckdb/src/error.rs similarity index 100% rename from src/error.rs rename to crates/duckdb/src/error.rs diff --git a/src/extension.rs b/crates/duckdb/src/extension.rs similarity index 100% rename from src/extension.rs rename to crates/duckdb/src/extension.rs diff --git a/src/inner_connection.rs b/crates/duckdb/src/inner_connection.rs similarity index 100% rename from src/inner_connection.rs rename to crates/duckdb/src/inner_connection.rs diff --git a/src/lib.rs b/crates/duckdb/src/lib.rs similarity index 99% rename from src/lib.rs rename to crates/duckdb/src/lib.rs index 81e2a5d9..f86ad9c1 100644 --- a/src/lib.rs +++ b/crates/duckdb/src/lib.rs @@ -566,7 +566,7 @@ impl fmt::Debug for Connection { } #[cfg(doctest)] -doc_comment::doctest!("../README.md"); +doc_comment::doctest!("../../../README.md"); #[cfg(test)] mod test { diff --git a/src/params.rs b/crates/duckdb/src/params.rs similarity index 100% rename from src/params.rs rename to crates/duckdb/src/params.rs diff --git a/src/polars_dataframe.rs b/crates/duckdb/src/polars_dataframe.rs similarity index 100% rename from src/polars_dataframe.rs rename to crates/duckdb/src/polars_dataframe.rs diff --git a/src/pragma.rs b/crates/duckdb/src/pragma.rs similarity index 100% rename from src/pragma.rs rename to crates/duckdb/src/pragma.rs diff --git a/src/r2d2.rs b/crates/duckdb/src/r2d2.rs similarity index 100% rename from src/r2d2.rs rename to crates/duckdb/src/r2d2.rs diff --git a/src/raw_statement.rs b/crates/duckdb/src/raw_statement.rs similarity index 100% rename from src/raw_statement.rs rename to crates/duckdb/src/raw_statement.rs diff --git a/src/row.rs b/crates/duckdb/src/row.rs similarity index 99% rename from src/row.rs rename to crates/duckdb/src/row.rs index d1ac7905..cfdec186 100644 --- a/src/row.rs +++ b/crates/duckdb/src/row.rs @@ -3,9 +3,8 @@ use std::{convert, sync::Arc}; use super::{Error, Result, Statement}; use crate::types::{self, EnumType, FromSql, FromSqlError, ListType, ValueRef}; -use arrow::array::DictionaryArray; use arrow::{ - array::{self, Array, ArrayRef, ListArray, StructArray}, + array::{self, Array, ArrayRef, DictionaryArray, ListArray, StructArray}, datatypes::*, }; use fallible_iterator::FallibleIterator; diff --git a/src/statement.rs b/crates/duckdb/src/statement.rs similarity index 100% rename from src/statement.rs rename to crates/duckdb/src/statement.rs diff --git a/src/test_all_types.rs b/crates/duckdb/src/test_all_types.rs similarity index 100% rename from src/test_all_types.rs rename to crates/duckdb/src/test_all_types.rs diff --git a/src/transaction.rs b/crates/duckdb/src/transaction.rs similarity index 100% rename from src/transaction.rs rename to crates/duckdb/src/transaction.rs diff --git a/src/types/chrono.rs b/crates/duckdb/src/types/chrono.rs similarity index 100% rename from src/types/chrono.rs rename to crates/duckdb/src/types/chrono.rs diff --git a/src/types/from_sql.rs b/crates/duckdb/src/types/from_sql.rs similarity index 100% rename from src/types/from_sql.rs rename to crates/duckdb/src/types/from_sql.rs diff --git a/src/types/mod.rs b/crates/duckdb/src/types/mod.rs similarity index 100% rename from src/types/mod.rs rename to crates/duckdb/src/types/mod.rs diff --git a/src/types/serde_json.rs b/crates/duckdb/src/types/serde_json.rs similarity index 100% rename from src/types/serde_json.rs rename to crates/duckdb/src/types/serde_json.rs diff --git a/src/types/to_sql.rs b/crates/duckdb/src/types/to_sql.rs similarity index 100% rename from src/types/to_sql.rs rename to crates/duckdb/src/types/to_sql.rs diff --git a/src/types/url.rs b/crates/duckdb/src/types/url.rs similarity index 100% rename from src/types/url.rs rename to crates/duckdb/src/types/url.rs diff --git a/src/types/value.rs b/crates/duckdb/src/types/value.rs similarity index 100% rename from src/types/value.rs rename to crates/duckdb/src/types/value.rs diff --git a/src/types/value_ref.rs b/crates/duckdb/src/types/value_ref.rs similarity index 98% rename from src/types/value_ref.rs rename to crates/duckdb/src/types/value_ref.rs index 60ef8c7d..db06c462 100644 --- a/src/types/value_ref.rs +++ b/crates/duckdb/src/types/value_ref.rs @@ -4,8 +4,10 @@ use crate::types::{FromSqlError, FromSqlResult}; use crate::Row; use rust_decimal::prelude::*; -use arrow::array::{Array, ArrayRef, DictionaryArray, LargeListArray, ListArray}; -use arrow::datatypes::{UInt16Type, UInt32Type, UInt8Type}; +use arrow::{ + array::{Array, ArrayRef, DictionaryArray, LargeListArray, ListArray}, + datatypes::{UInt16Type, UInt32Type, UInt8Type}, +}; /// An absolute length of time in seconds, milliseconds, microseconds or nanoseconds. /// Copy from arrow::datatypes::TimeUnit diff --git a/src/util/mod.rs b/crates/duckdb/src/util/mod.rs similarity index 100% rename from src/util/mod.rs rename to crates/duckdb/src/util/mod.rs diff --git a/src/util/small_cstr.rs b/crates/duckdb/src/util/small_cstr.rs similarity index 100% rename from src/util/small_cstr.rs rename to crates/duckdb/src/util/small_cstr.rs diff --git a/src/vtab/arrow.rs b/crates/duckdb/src/vtab/arrow.rs similarity index 100% rename from src/vtab/arrow.rs rename to crates/duckdb/src/vtab/arrow.rs diff --git a/src/vtab/data_chunk.rs b/crates/duckdb/src/vtab/data_chunk.rs similarity index 100% rename from src/vtab/data_chunk.rs rename to crates/duckdb/src/vtab/data_chunk.rs diff --git a/src/vtab/excel.rs b/crates/duckdb/src/vtab/excel.rs similarity index 100% rename from src/vtab/excel.rs rename to crates/duckdb/src/vtab/excel.rs diff --git a/src/vtab/function.rs b/crates/duckdb/src/vtab/function.rs similarity index 100% rename from src/vtab/function.rs rename to crates/duckdb/src/vtab/function.rs diff --git a/src/vtab/logical_type.rs b/crates/duckdb/src/vtab/logical_type.rs similarity index 100% rename from src/vtab/logical_type.rs rename to crates/duckdb/src/vtab/logical_type.rs diff --git a/src/vtab/mod.rs b/crates/duckdb/src/vtab/mod.rs similarity index 100% rename from src/vtab/mod.rs rename to crates/duckdb/src/vtab/mod.rs diff --git a/src/vtab/value.rs b/crates/duckdb/src/vtab/value.rs similarity index 100% rename from src/vtab/value.rs rename to crates/duckdb/src/vtab/value.rs diff --git a/src/vtab/vector.rs b/crates/duckdb/src/vtab/vector.rs similarity index 100% rename from src/vtab/vector.rs rename to crates/duckdb/src/vtab/vector.rs diff --git a/crates/libduckdb-sys/Cargo.toml b/crates/libduckdb-sys/Cargo.toml new file mode 100644 index 00000000..207bdf56 --- /dev/null +++ b/crates/libduckdb-sys/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "libduckdb-sys" +version = "0.10.2" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +keywords.workspace = true +readme.workspace = true +build = "build.rs" +categories = ["external-ffi-bindings", "database"] +description = "Native bindings to the libduckdb library, C API" +exclude = ["duckdb-sources"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["vcpkg", "pkg-config"] +bundled = ["cc"] +buildtime_bindgen = ["bindgen", "pkg-config", "vcpkg"] + +httpfs = ["bundled"] +json = ["bundled"] +parquet = ["bundled"] +extensions-full = ["httpfs", "json", "parquet"] + +[dependencies] + +[build-dependencies] +autocfg = { workspace = true } +bindgen = { workspace = true, features = ["runtime"], optional = true } +flate2 = { workspace = true } +pkg-config = { workspace = true, optional = true } +cc = { workspace = true, features = ["parallel"], optional = true } +vcpkg = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tar = { workspace = true } + +[dev-dependencies] +arrow = { workspace = true, features = ["ffi"] } diff --git a/crates/libduckdb-sys/LICENSE b/crates/libduckdb-sys/LICENSE new file mode 120000 index 00000000..30cff740 --- /dev/null +++ b/crates/libduckdb-sys/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/libduckdb-sys/README.md b/crates/libduckdb-sys/README.md new file mode 120000 index 00000000..fe840054 --- /dev/null +++ b/crates/libduckdb-sys/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/libduckdb-sys/bindgen.sh b/crates/libduckdb-sys/bindgen.sh similarity index 100% rename from libduckdb-sys/bindgen.sh rename to crates/libduckdb-sys/bindgen.sh diff --git a/libduckdb-sys/build.rs b/crates/libduckdb-sys/build.rs similarity index 98% rename from libduckdb-sys/build.rs rename to crates/libduckdb-sys/build.rs index 92031acb..1ae4533a 100644 --- a/libduckdb-sys/build.rs +++ b/crates/libduckdb-sys/build.rs @@ -120,14 +120,6 @@ mod build_bundled { let mut cfg = cc::Build::new(); - #[cfg(feature = "httpfs")] - { - if let Ok((_, openssl_include_dir)) = super::openssl::get_openssl_v2() { - cfg.include(openssl_include_dir); - } - add_extension(&mut cfg, &manifest, "httpfs", &mut cpp_files, &mut include_dirs); - } - #[cfg(feature = "parquet")] add_extension(&mut cfg, &manifest, "parquet", &mut cpp_files, &mut include_dirs); @@ -146,6 +138,16 @@ mod build_bundled { cfg.include(lib_name); + // Note: dont move this, the link order is important and we need to make + // sure we link openssl after duckdb + #[cfg(feature = "httpfs")] + { + if let Ok((_, openssl_include_dir)) = super::openssl::get_openssl_v2() { + cfg.include(openssl_include_dir); + } + add_extension(&mut cfg, &manifest, "httpfs", &mut cpp_files, &mut include_dirs); + } + cfg.includes(include_dirs.iter().map(|x| format!("{}/{}", lib_name, x))); for f in cpp_files { @@ -163,7 +165,6 @@ mod build_bundled { if win_target() { cfg.define("DUCKDB_BUILD_LIBRARY", None); } - cfg.compile(lib_name); println!("cargo:lib_dir={out_dir}"); } diff --git a/libduckdb-sys/duckdb.tar.gz b/crates/libduckdb-sys/duckdb.tar.gz similarity index 100% rename from libduckdb-sys/duckdb.tar.gz rename to crates/libduckdb-sys/duckdb.tar.gz diff --git a/libduckdb-sys/openssl/cfgs.rs b/crates/libduckdb-sys/openssl/cfgs.rs similarity index 100% rename from libduckdb-sys/openssl/cfgs.rs rename to crates/libduckdb-sys/openssl/cfgs.rs diff --git a/libduckdb-sys/openssl/expando.c b/crates/libduckdb-sys/openssl/expando.c similarity index 100% rename from libduckdb-sys/openssl/expando.c rename to crates/libduckdb-sys/openssl/expando.c diff --git a/libduckdb-sys/openssl/find_normal.rs b/crates/libduckdb-sys/openssl/find_normal.rs similarity index 100% rename from libduckdb-sys/openssl/find_normal.rs rename to crates/libduckdb-sys/openssl/find_normal.rs diff --git a/libduckdb-sys/openssl/find_vendored.rs b/crates/libduckdb-sys/openssl/find_vendored.rs similarity index 100% rename from libduckdb-sys/openssl/find_vendored.rs rename to crates/libduckdb-sys/openssl/find_vendored.rs diff --git a/libduckdb-sys/openssl/mod.rs b/crates/libduckdb-sys/openssl/mod.rs similarity index 100% rename from libduckdb-sys/openssl/mod.rs rename to crates/libduckdb-sys/openssl/mod.rs diff --git a/libduckdb-sys/openssl/run_bindgen.rs b/crates/libduckdb-sys/openssl/run_bindgen.rs similarity index 100% rename from libduckdb-sys/openssl/run_bindgen.rs rename to crates/libduckdb-sys/openssl/run_bindgen.rs diff --git a/libduckdb-sys/src/bindgen_bundled_version.rs b/crates/libduckdb-sys/src/bindgen_bundled_version.rs similarity index 100% rename from libduckdb-sys/src/bindgen_bundled_version.rs rename to crates/libduckdb-sys/src/bindgen_bundled_version.rs diff --git a/libduckdb-sys/src/error.rs b/crates/libduckdb-sys/src/error.rs similarity index 100% rename from libduckdb-sys/src/error.rs rename to crates/libduckdb-sys/src/error.rs diff --git a/libduckdb-sys/src/lib.rs b/crates/libduckdb-sys/src/lib.rs similarity index 93% rename from libduckdb-sys/src/lib.rs rename to crates/libduckdb-sys/src/lib.rs index d7547535..ae57cadf 100644 --- a/libduckdb-sys/src/lib.rs +++ b/crates/libduckdb-sys/src/lib.rs @@ -33,18 +33,15 @@ mod tests { ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}, }; - unsafe fn print_int_result(mut result: duckdb_result) { - for i in 0..duckdb_column_count(&mut result) { - print!( - "{} ", - CStr::from_ptr(duckdb_column_name(&mut result, i)).to_string_lossy() - ); + unsafe fn print_int_result(result: &mut duckdb_result) { + for i in 0..duckdb_column_count(result) { + print!("{} ", CStr::from_ptr(duckdb_column_name(result, i)).to_string_lossy()); } println!(); // print the data of the result - for row_idx in 0..duckdb_row_count(&mut result) { - for col_idx in 0..duckdb_column_count(&mut result) { - let val = duckdb_value_int32(&mut result, col_idx, row_idx); + for row_idx in 0..duckdb_row_count(result) { + for col_idx in 0..duckdb_column_count(result) { + let val = duckdb_value_int32(result, col_idx, row_idx); print!("{val} "); } println!(); @@ -162,7 +159,7 @@ mod tests { } assert_eq!(duckdb_row_count(&mut result), 3); assert_eq!(duckdb_column_count(&mut result), 2); - print_int_result(result); + print_int_result(&mut result); duckdb_destroy_result(&mut result); // test prepare @@ -179,7 +176,7 @@ mod tests { } assert_eq!(duckdb_row_count(&mut result), 2); assert_eq!(duckdb_column_count(&mut result), 2); - print_int_result(result); + print_int_result(&mut result); duckdb_destroy_result(&mut result); // test bind params again @@ -191,7 +188,7 @@ mod tests { } assert_eq!(duckdb_row_count(&mut result), 1); assert_eq!(duckdb_column_count(&mut result), 2); - print_int_result(result); + print_int_result(&mut result); duckdb_destroy_result(&mut result); duckdb_destroy_prepare(&mut stmt); diff --git a/libduckdb-sys/src/raw_statement.rs b/crates/libduckdb-sys/src/raw_statement.rs similarity index 100% rename from libduckdb-sys/src/raw_statement.rs rename to crates/libduckdb-sys/src/raw_statement.rs diff --git a/libduckdb-sys/update_sources.py b/crates/libduckdb-sys/update_sources.py similarity index 100% rename from libduckdb-sys/update_sources.py rename to crates/libduckdb-sys/update_sources.py diff --git a/libduckdb-sys/upgrade.sh b/crates/libduckdb-sys/upgrade.sh similarity index 100% rename from libduckdb-sys/upgrade.sh rename to crates/libduckdb-sys/upgrade.sh diff --git a/libduckdb-sys/wrapper.h b/crates/libduckdb-sys/wrapper.h similarity index 100% rename from libduckdb-sys/wrapper.h rename to crates/libduckdb-sys/wrapper.h diff --git a/duckdb-loadable-macros/Cargo.toml b/duckdb-loadable-macros/Cargo.toml deleted file mode 100644 index 30c40ab2..00000000 --- a/duckdb-loadable-macros/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "duckdb-loadable-macros" -version = "0.1.1" -authors = ["wangfenjin "] -edition = "2021" -license = "MIT" -repository = "https://github.com/wangfenjin/duckdb-rs" -homepage = "https://github.com/wangfenjin/duckdb-rs" -keywords = ["duckdb", "ffi", "database"] -readme = "README.md" -categories = ["external-ffi-bindings", "database"] -description = "Native bindings to the libduckdb library, C API; build loadable extensions" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -proc-macro2 = { version = "1.0.56" } -quote = { version = "1.0.21" } -syn = { version = "2.0.15", features = [ "extra-traits", "full", "fold", "parsing" ] } - -[lib] -proc-macro = true diff --git a/duckdb-loadable-macros/LICENSE b/duckdb-loadable-macros/LICENSE deleted file mode 120000 index ea5b6064..00000000 --- a/duckdb-loadable-macros/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../LICENSE \ No newline at end of file diff --git a/duckdb-loadable-macros/README.md b/duckdb-loadable-macros/README.md deleted file mode 120000 index 32d46ee8..00000000 --- a/duckdb-loadable-macros/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/libduckdb-sys/Cargo.toml b/libduckdb-sys/Cargo.toml deleted file mode 100644 index 7465f90f..00000000 --- a/libduckdb-sys/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -name = "libduckdb-sys" -version = "0.10.2" -authors = ["wangfenjin "] -edition = "2021" -build = "build.rs" -license = "MIT" -repository = "https://github.com/wangfenjin/duckdb-rs" -homepage = "https://github.com/wangfenjin/duckdb-rs" -keywords = ["duckdb", "ffi", "database"] -readme = "README.md" -categories = ["external-ffi-bindings", "database"] -description = "Native bindings to the libduckdb library, C API" -exclude = ["duckdb-sources"] - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[features] -default = ["vcpkg", "pkg-config"] -bundled = ["cc"] -buildtime_bindgen = ["bindgen", "pkg-config", "vcpkg"] - -httpfs = ["bundled"] -json = ["bundled"] -parquet = ["bundled"] -extensions-full = ["httpfs", "json", "parquet"] - -[dependencies] - -[build-dependencies] -autocfg = "1.0" -bindgen = { version = "0.69", optional = true, default-features = false, features = ["runtime"] } -flate2 = "1.0" -pkg-config = { version = "0.3.24", optional = true } -cc = { version = "1.0", features = ["parallel"], optional = true } -vcpkg = { version = "0.2", optional = true } -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0" } -tar = "0.4.38" - -[dev-dependencies] -arrow = { version = "51", default-features = false, features = ["ffi"] } diff --git a/libduckdb-sys/LICENSE b/libduckdb-sys/LICENSE deleted file mode 120000 index ea5b6064..00000000 --- a/libduckdb-sys/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../LICENSE \ No newline at end of file diff --git a/libduckdb-sys/README.md b/libduckdb-sys/README.md deleted file mode 120000 index 32d46ee8..00000000 --- a/libduckdb-sys/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/libduckdb-sys/duckdb-sources b/libduckdb-sys/duckdb-sources deleted file mode 160000 index 1601d94f..00000000 --- a/libduckdb-sys/duckdb-sources +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1601d94f94a7e0d2eb805a94803eb1e3afbbe4ed