Organise workspace crates and consolidate Cargo.toml manifest propert…

…ies/dependencies (#285)

* Organise workspace crates and unify Cargo.toml manifests

* Update code coverage and contributing.md

* double choco openssl install timeout

* num_integer is not optional

* move httpfs later...?

* before

* include dir

* add note

* fix memory leak in api usage test

* specify crates explicitly

* swap action

* I guess it just doesnt work with workspace members

* swap again

* swap yet again

* ignore unpublished change

* typo

---------

Co-authored-by: Max Gabrielsson <[email protected]>
Co-authored-by: Max Gabrielsson <[email protected]>

.github/.codecov.yml

@@ -1,6 +1,6 @@
 ignore:
-  - "libduckdb-sys/duckdb"
-  - "libduckdb-sys/src/bindgen_bundled_version.rs"
+  - "crates/libduckdb-sys/duckdb"
+  - "crates/libduckdb-sys/src/bindgen_bundled_version.rs"
 coverage:
   status:
     project:

.github/workflows/release.yaml

@@ -20,13 +20,13 @@ jobs:
           components: 'rustfmt, clippy'
 
       # cargo publish
-      - uses: wangfenjin/publish-crates@main
-        name: cargo publish
+      - name: publish crates
+        uses: katyo/publish-crates@v2
         with:
           path: './'
           args: --no-verify --allow-dirty --all-features
           registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          ignore-published: true
+          ignore-unpublished-changes: true
 
       # create release
       - name: "Build Changelog"

.github/workflows/rust.yaml

@@ -145,10 +145,10 @@ jobs:
           # as the other tests have them.
           RUST_BACKTRACE: "0"
         run: cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu
-      - uses: wangfenjin/publish-crates@main
-        name: cargo publish --dry-run
+      - name: publish crates --dry-run
+        uses: katyo/publish-crates@v2
         with:
           path: './'
           args: --allow-dirty --all-features
           dry-run: true
-          ignore-published: true
+          ignore-unpublished-changes: true

.gitignore

@@ -27,6 +27,6 @@ Cargo.lock
 
 *.db
 
-libduckdb-sys/duckdb-sources/
-libduckdb-sys/duckdb/
-libduckdb-sys/._duckdb
+crates/libduckdb-sys/duckdb-sources/
+crates/libduckdb-sys/duckdb/
+crates/libduckdb-sys/._duckdb

.gitmodules

@@ -1,4 +1,4 @@
-[submodule "libduckdb-sys/duckdb-sources"]
-	path = libduckdb-sys/duckdb-sources
+[submodule "crates/libduckdb-sys/duckdb-sources"]
+	path = crates/libduckdb-sys/duckdb-sources
 	url = https://github.com/duckdb/duckdb
 	update = none

CONTRIBUTING.md

@@ -43,24 +43,24 @@ export DUCKDB_INCLUDE_DIR=~/duckdb-lib
 Use the exported library and header:
 
 ```shell
-cd ~/github/duckdb-rs/libduckdb-sys
+cd ~/github/duckdb-rs/crates/libduckdb-sys
 cargo test --features buildtime_bindgen
 ```
 
 Use the bundled header file:
 ```shell
-cd ~/github/duckdb-rs/libduckdb-sys
+cd ~/github/duckdb-rs/crates/libduckdb-sys
 cargo test --features bundled
 ```
 
-Currently in [github actions](https://github.com/wangfenjin/duckdb-rs/actions), we always use the bundled file for testing. So if you change the header in duckdb-cpp repo, you need to make the PR merged and updated the [bundled-file](https://github.com/wangfenjin/duckdb-rs/tree/main/libduckdb-sys/duckdb).
+Currently in [github actions](https://github.com/wangfenjin/duckdb-rs/actions), we always use the bundled file for testing. So if you change the header in duckdb-cpp repo, you need to make the PR merged and updated the [bundled-file](https://github.com/wangfenjin/duckdb-rs/tree/main/crates/libduckdb-sys/duckdb).
 You can generated the amalgamated file by:
 
 ```shell
 cd ~/github/duckdb
 mkdir -p build/amaldebug
 python scripts/amalgamation.py
-cp src/amalgamation/duckdb.cpp src/include/duckdb.h src/amalgamation/duckdb.hpp ../duckdb-rs/libduckdb-sys/duckdb/
+cp src/amalgamation/duckdb.cpp src/include/duckdb.h src/amalgamation/duckdb.hpp ../duckdb-rs/crates/libduckdb-sys/duckdb/
 ```
 
 ### duckdb-rs

Cargo.toml

@@ -1,9 +1,15 @@
-[package]
-name = "duckdb"
+[workspace]
+resolver = "2"
+members = [
+    "crates/duckdb",
+    "crates/libduckdb-sys",
+    "crates/duckdb-loadable-macros"
+]
+
+[workspace.package]
 version = "0.10.2"
 authors = ["wangfenjin <[email protected]>"]
 edition = "2021"
-description = "Ergonomic wrapper for DuckDB"
 repository = "https://github.com/wangfenjin/duckdb-rs"
 homepage = "https://github.com/wangfenjin/duckdb-rs"
 documentation = "http://docs.rs/duckdb/"
@@ -12,90 +18,46 @@ keywords = ["duckdb", "database", "ffi"]
 license = "MIT"
 categories = ["database"]
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[lib]
-name = "duckdb"
-
-[workspace]
-members = ["libduckdb-sys", "duckdb-loadable-macros"]
-
-[features]
-default = []
-bundled = ["libduckdb-sys/bundled"]
-httpfs = ["libduckdb-sys/httpfs", "bundled"]
-json = ["libduckdb-sys/json", "bundled"]
-parquet = ["libduckdb-sys/parquet", "bundled"]
-vtab = []
-vtab-loadable = ["vtab", "duckdb-loadable-macros"]
-vtab-excel = ["vtab", "calamine"]
-vtab-arrow = ["vtab", "num"]
-appender-arrow = ["vtab-arrow"]
-vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"]
-extensions-full = ["httpfs", "json", "parquet", "vtab-full"]
-buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
-modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"]
-polars = ["dep:polars"]
-chrono = ["dep:chrono", "num-integer"]
-
-[dependencies]
-# time = { version = "0.3.2", features = ["formatting", "parsing"], optional = true }
-hashlink = "0.8"
-chrono = { version = "0.4.22", optional = true }
-serde_json = { version = "1.0", optional = true }
-csv = { version = "1.1", optional = true }
-url = { version = "2.1", optional = true }
-lazy_static = { version = "1.4", optional = true }
-byteorder = { version = "1.3", features = ["i128"], optional = true }
+[workspace.dependencies]
+duckdb = { version = "0.10.2", path = "crates/duckdb" }
+libduckdb-sys = { version = "0.10.2", path = "crates/libduckdb-sys" }
+duckdb-loadable-macros = { version = "0.1.1", path = "crates/duckdb-loadable-macros" }
+autocfg = "1.0"
+bindgen = { version = "0.69", default-features = false }
+byteorder = "1.3"
+calamine = "0.22.0"
+cast = "0.3"
+cc = "1.0"
+chrono = "0.4.22"
+csv = "1.1"
+doc-comment = "0.3"
 fallible-iterator = "0.3"
 fallible-streaming-iterator = "0.1"
+flate2 = "1.0"
+hashlink = "0.8"
+lazy_static = "1.4"
 memchr = "2.3"
-uuid = { version = "1.0", optional = true }
-smallvec = "1.6.1"
-cast = { version = "0.3", features = ["std"] }
-arrow = { version = "51", default-features = false, features = ["prettyprint", "ffi"] }
+num = { version = "0.4", default-features = false }
+pkg-config = "0.3.24"
+polars = "0.35.4"
+polars-core = "0.35.4"
+pretty_assertions = "1.4.0"
+proc-macro2 = "1.0.56"
+quote = "1.0.21"
+r2d2 = "0.8.9"
+rand = "0.8.3"
+regex = "1.6"
 rust_decimal = "1.14"
-strum = { version = "0.25", features = ["derive"] }
-r2d2 = { version = "0.8.9", optional = true }
-calamine = { version = "0.22.0", optional = true }
-num = { version = "0.4", optional = true, default-features = false, features = ["std"] }
-duckdb-loadable-macros = { version = "0.1.1", path="./duckdb-loadable-macros", optional = true }
-polars = { version = "0.35.4", features = ["dtype-full"], optional = true}
-num-integer = {version = "0.1.46", optional = true}
-
-[dev-dependencies]
-doc-comment = "0.3"
+serde = "1.0"
+serde_json = "1.0"
+smallvec = "1.6.1"
+strum = "0.25"
+syn = "2.0.15"
+tar = "0.4.38"
+tempdir = "0.3.7"
 tempfile = "3.1.0"
-lazy_static = "1.4"
-regex = "1.6"
-uuid = { version = "1.0", features = ["v4"] }
 unicase = "2.6.0"
-rand = "0.8.3"
-tempdir = "0.3.7"
-polars-core = "0.35.4"
-pretty_assertions = "1.4.0"
-# criterion = "0.3"
-
-# [[bench]]
-# name = "data_types"
-# harness = false
-
-[dependencies.libduckdb-sys]
-path = "libduckdb-sys"
-version = "0.10.2"
-
-
-[package.metadata.docs.rs]
-features = ['vtab', 'chrono']
-all-features = false
-no-default-features = true
-default-target = "x86_64-unknown-linux-gnu"
-
-[package.metadata.playground]
-features = []
-all-features = false
-
-[[example]]
-name = "hello-ext"
-crate-type = ["cdylib"]
-required-features = ["vtab-loadable"]
+url = "2.1"
+uuid = "1.0"
+vcpkg = "0.2"
+arrow = { version = "51", default-features = false }

crates/duckdb-loadable-macros/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "duckdb-loadable-macros"
+version = "0.1.1"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+readme.workspace = true
+categories = ["external-ffi-bindings", "database"]
+description = "Native bindings to the libduckdb library, C API; build loadable extensions"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+proc-macro2 = { workspace = true }
+quote = { workspace = true }
+syn = { workspace = true, features = ["extra-traits", "full", "fold", "parsing"] }
+
+[lib]
+proc-macro = true

crates/duckdb-loadable-macros/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

crates/duckdb-loadable-macros/README.md

@@ -0,0 +1 @@
+../../README.md

crates/duckdb/Cargo.toml

@@ -0,0 +1,93 @@
+[package]
+name = "duckdb"
+version = "0.10.2"
+authors.workspace = true
+edition.workspace = true
+repository.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+readme.workspace = true
+keywords.workspace = true
+license.workspace = true
+categories.workspace = true
+description = "Ergonomic wrapper for DuckDB"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+name = "duckdb"
+
+[features]
+default = []
+bundled = ["libduckdb-sys/bundled"]
+httpfs = ["libduckdb-sys/httpfs", "bundled"]
+json = ["libduckdb-sys/json", "bundled"]
+parquet = ["libduckdb-sys/parquet", "bundled"]
+vtab = []
+vtab-loadable = ["vtab", "duckdb-loadable-macros"]
+vtab-excel = ["vtab", "calamine"]
+vtab-arrow = ["vtab", "num"]
+appender-arrow = ["vtab-arrow"]
+vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"]
+extensions-full = ["httpfs", "json", "parquet", "vtab-full"]
+buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
+modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"]
+polars = ["dep:polars"]
+
+[dependencies]
+libduckdb-sys = { workspace = true }
+hashlink = { workspace = true }
+chrono = { workspace = true, optional = true }
+serde_json = { workspace = true, optional = true }
+csv = { workspace = true, optional = true }
+url = { workspace = true, optional = true }
+lazy_static = { workspace = true, optional = true }
+byteorder = { workspace = true, features = ["i128"], optional = true }
+fallible-iterator = { workspace = true }
+fallible-streaming-iterator = { workspace = true }
+memchr = { workspace = true }
+uuid = { workspace = true, optional = true }
+smallvec = { workspace = true }
+cast = { workspace = true, features = ["std"] }
+arrow = { workspace = true, features = ["prettyprint", "ffi"] }
+rust_decimal = { workspace = true }
+strum = { workspace = true, features = ["derive"] }
+r2d2 = { workspace = true, optional = true }
+calamine = { workspace = true, optional = true }
+num = { workspace = true, features = ["std"], optional = true }
+duckdb-loadable-macros = { workspace = true, optional = true }
+polars = { workspace = true, features = ["dtype-full"], optional = true }
+num-integer = {version = "0.1.46"}
+
+[dev-dependencies]
+doc-comment = { workspace = true }
+tempfile = { workspace = true }
+lazy_static = { workspace = true }
+regex = { workspace = true }
+uuid = { workspace = true, features = ["v4"] }
+unicase = { workspace = true }
+rand = { workspace = true }
+tempdir = { workspace = true }
+polars-core = { workspace = true }
+pretty_assertions = { workspace = true }
+# criterion = "0.3"
+
+# [[bench]]
+# name = "data_types"
+# harness = false
+
+
+[package.metadata.docs.rs]
+features = []
+all-features = false
+no-default-features = true
+default-target = "x86_64-unknown-linux-gnu"
+
+[package.metadata.playground]
+features = []
+all-features = false
+
+[[example]]
+name = "hello-ext"
+crate-type = ["cdylib"]
+required-features = ["vtab-loadable"]

src/lib.rs → crates/duckdb/src/lib.rs

@@ -566,7 +566,7 @@ impl fmt::Debug for Connection {
 }
 
 #[cfg(doctest)]
-doc_comment::doctest!("../README.md");
+doc_comment::doctest!("../../../README.md");
 
 #[cfg(test)]
 mod test {

src/row.rs → crates/duckdb/src/row.rs

@@ -3,9 +3,8 @@ use std::{convert, sync::Arc};
 use super::{Error, Result, Statement};
 use crate::types::{self, EnumType, FromSql, FromSqlError, ListType, ValueRef};
 
-use arrow::array::DictionaryArray;
 use arrow::{
-    array::{self, Array, ArrayRef, ListArray, StructArray},
+    array::{self, Array, ArrayRef, DictionaryArray, ListArray, StructArray},
     datatypes::*,
 };
 use fallible_iterator::FallibleIterator;

src/types/value_ref.rs → crates/duckdb/src/types/value_ref.rs

@@ -4,8 +4,10 @@ use crate::types::{FromSqlError, FromSqlResult};
 use crate::Row;
 use rust_decimal::prelude::*;
 
-use arrow::array::{Array, ArrayRef, DictionaryArray, LargeListArray, ListArray};
-use arrow::datatypes::{UInt16Type, UInt32Type, UInt8Type};
+use arrow::{
+    array::{Array, ArrayRef, DictionaryArray, LargeListArray, ListArray},
+    datatypes::{UInt16Type, UInt32Type, UInt8Type},
+};
 
 /// An absolute length of time in seconds, milliseconds, microseconds or nanoseconds.
 /// Copy from arrow::datatypes::TimeUnit