wip: use sqlite extension structure that supports attach/detach

CMakeLists.txt

@@ -1,22 +1,23 @@
 cmake_minimum_required(VERSION 2.8.12)
-
-# Set extension name here
 set(TARGET_NAME odbc_scanner)
-
 set(EXTENSION_NAME ${TARGET_NAME}_extension)
 project(${TARGET_NAME})
 
-include_directories(src/include)
-
-set(
-  EXTENSION_SOURCES
-  src/odbc_scan.cpp
-  src/odbc_scanner_extension.cpp
-)
-add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES})
+# include_directories(src/include)
+#
+# set(
+#   EXTENSION_SOURCES
+#   src/odbc_scan.cpp
+#   src/odbc_scanner_extension.cpp
+# )
+# add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES})
+add_subdirectory(src)
+set(EXTENSION_OBJECT_FILES ${ALL_OBJECT_FILES})
+add_library(${EXTENSION_NAME} STATIC ${EXTENSION_OBJECT_FILES})
 
 set(PARAMETERS "-warnings")
-build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES})
+# build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES})
+build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_OBJECT_FILES})
 
 # nix store location of ODBC_CONFIG is passed in as CLIENT_FLAGS to root level Makefile
 # set(ODBC_CONFIG /nix/store/xs4bg5404nsjarivdzxszq0z0pn2ckzv-unixODBC-2.3.11/bin/odbc_config)

Makefile

@@ -23,7 +23,11 @@ BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_TPCH_EXTENSION=1 -DBUILD_PARQUET_
 CLIENT_FLAGS :=
 
 # These flags will make DuckDB build the extension
-EXTENSION_FLAGS=-DDUCKDB_OOT_EXTENSION_NAMES="odbc_scanner" -DDUCKDB_OOT_EXTENSION_ODBC_SCANNER_PATH="$(PROJ_DIR)" -DDUCKDB_OOT_EXTENSION_ODBC_SCANNER_SHOULD_LINK="TRUE" -DDUCKDB_OOT_EXTENSION_ODBC_SCANNER_INCLUDE_PATH="$(PROJ_DIR)src/include"
+EXTENSION_FLAGS=\
+								-DDUCKDB_OOT_EXTENSION_NAMES="odbc_scanner" \
+								-DDUCKDB_OOT_EXTENSION_ODBC_SCANNER_PATH="$(PROJ_DIR)" \
+								-DDUCKDB_OOT_EXTENSION_ODBC_SCANNER_SHOULD_LINK="TRUE" \
+								-DDUCKDB_OOT_EXTENSION_ODBC_SCANNER_INCLUDE_PATH="$(PROJ_DIR)src/include"
 
 pull:
 	git submodule init

src/CMakeLists.txt

@@ -0,0 +1,18 @@
+include_directories(include)
+
+add_subdirectory(storage)
+
+add_library(
+  odbc_ext_library OBJECT
+  odbc_db.cpp
+  odbc_scanner_extension.cpp
+  odbc_scan.cpp
+  odbc_stmt.cpp
+  odbc_storage.cpp
+  odbc_utils.cpp
+)
+set(
+  ALL_OBJECT_FILES
+  ${ALL_OBJECT_FILES} $<TARGET_OBJECTS:odbc_ext_library>
+  PARENT_SCOPE
+)

src/include/odbc_db.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "odbc_utils.hpp"
+
+namespace duckdb {
+class OdbcStatement;
+struct IndexInfo;
+
+class OdbcDB {
+public:
+	OdbcDB();
+	// OdbcDB(sqlite3 *db);
+	~OdbcDB();
+	// disable copy constructors
+	OdbcDB(const OdbcDB &other) = delete;
+	OdbcDB &operator=(const OdbcDB &) = delete;
+	//! enable move constructors
+	OdbcDB(OdbcDB &&other) noexcept;
+	OdbcDB &operator=(OdbcDB &&) noexcept;
+
+	// sqlite3 *db;
+
+public:
+	static OdbcDB Open(const string &path, bool is_read_only = true, bool is_shared = false);
+	bool TryPrepare(const string &query, OdbcStatement &result);
+	OdbcStatement Prepare(const string &query);
+	void Execute(const string &query);
+	vector<string> GetTables();
+
+	vector<string> GetEntries(string entry_type);
+	CatalogType GetEntryType(const string &name);
+	void GetTableInfo(const string &table_name, ColumnList &columns, vector<unique_ptr<Constraint>> &constraints,
+	                  bool all_varchar);
+	void GetViewInfo(const string &view_name, string &sql);
+	void GetIndexInfo(const string &index_name, string &sql, string &table_name);
+	idx_t RunPragma(string pragma_name);
+	//! Gets the max row id of a table, returns false if the table does not have a rowid column
+	bool GetMaxRowId(const string &table_name, idx_t &row_id);
+	bool ColumnExists(const string &table_name, const string &column_name);
+	vector<IndexInfo> GetIndexInfo(const string &table_name);
+
+	bool IsOpen();
+	void Close();
+};
+
+} // namespace duckdb

src/include/odbc_stmt.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include "odbc_utils.hpp"
+
+#include <cstddef>
+
+namespace duckdb {
+
+class OdbcStatement {
+public:
+	OdbcStatement();
+	// OdbcStatement(sqlite3 *db, sqlite3_stmt *stmt);
+	~OdbcStatement();
+	// disable copy constructors
+	OdbcStatement(const OdbcStatement &other) = delete;
+	OdbcStatement &operator=(const OdbcStatement &) = delete;
+	//! enable move constructors
+	OdbcStatement(OdbcStatement &&other) noexcept;
+	OdbcStatement &operator=(OdbcStatement &&) noexcept;
+
+	// sqlite3 *db;
+	// sqlite3_stmt *stmt;
+
+public:
+	int Step();
+	template <class T>
+	T GetValue(idx_t col) {
+		throw InternalException("Unsupported type for OdbcStatement::GetValue");
+	}
+	template <class T>
+	void Bind(idx_t col, T value) {
+		throw InternalException("Unsupported type for OdbcStatement::Bind");
+	}
+	void BindText(idx_t col, const string_t &value);
+	void BindValue(Vector &col, idx_t c, idx_t r);
+	int GetType(idx_t col);
+	bool IsOpen();
+	void Close();
+	// void CheckTypeMatches(sqlite3_value *val, int sqlite_column_type, int expected_type, idx_t col_idx);
+	// void CheckTypeIsFloatOrInteger(sqlite3_value *val, int sqlite_column_type, idx_t col_idx);
+	void Reset();
+};
+
+template <>
+string OdbcStatement::GetValue(idx_t col);
+template <>
+int OdbcStatement::GetValue(idx_t col);
+template <>
+int64_t OdbcStatement::GetValue(idx_t col);
+// template <>
+// sqlite3_value *OdbcStatement::GetValue(idx_t col);
+
+template <>
+void OdbcStatement::Bind(idx_t col, int32_t value);
+template <>
+void OdbcStatement::Bind(idx_t col, int64_t value);
+template <>
+void OdbcStatement::Bind(idx_t col, double value);
+template <>
+void OdbcStatement::Bind(idx_t col, std::nullptr_t value);
+
+} // namespace duckdb

src/include/odbc_storage.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "duckdb/storage/storage_extension.hpp"
+
+namespace duckdb {
+
+class OdbcStorageExtension : public StorageExtension {
+public:
+	OdbcStorageExtension();
+};
+
+} // namespace duckdb

src/include/odbc_utils.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "duckdb.hpp"
+// #include "sqlite3.h"
+
+namespace duckdb {
+
+class OdbcUtils {
+public:
+	// static void Check(int rc, sqlite3 *db);
+	static string TypeToString(int sqlite_type);
+	static LogicalType TypeToLogicalType(const string &sqlite_type);
+	static string SanitizeString(const string &table_name);
+	static string SanitizeIdentifier(const string &table_name);
+	static LogicalType ToOdbcType(const LogicalType &input);
+	string ToOdbcTypeAlias(const LogicalType &input);
+};
+
+} // namespace duckdb

src/include/storage/odbc_catalog.hpp

@@ -0,0 +1,73 @@
+#pragma once
+
+#include "duckdb/catalog/catalog.hpp"
+#include "duckdb/common/enums/access_mode.hpp"
+#include "odbc_db.hpp"
+
+namespace duckdb {
+class OdbcSchemaEntry;
+
+class OdbcCatalog : public Catalog {
+public:
+	explicit OdbcCatalog(AttachedDatabase &db_p, const string &path, AccessMode access_mode);
+	~OdbcCatalog();
+
+	string path;
+	AccessMode access_mode;
+
+public:
+	void Initialize(bool load_builtin) override;
+	string GetCatalogType() override {
+		return "odbc";
+	}
+
+	optional_ptr<CatalogEntry> CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) override;
+
+	void ScanSchemas(ClientContext &context, std::function<void(SchemaCatalogEntry &)> callback) override;
+
+	optional_ptr<SchemaCatalogEntry> GetSchema(CatalogTransaction transaction, const string &schema_name,
+	                                           OnEntryNotFound if_not_found,
+	                                           QueryErrorContext error_context = QueryErrorContext()) override;
+
+	OdbcSchemaEntry &GetMainSchema() {
+		return *main_schema;
+	}
+
+	unique_ptr<PhysicalOperator> PlanInsert(ClientContext &context, LogicalInsert &op,
+	                                        unique_ptr<PhysicalOperator> plan) override;
+	unique_ptr<PhysicalOperator> PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op,
+	                                               unique_ptr<PhysicalOperator> plan) override;
+	unique_ptr<PhysicalOperator> PlanDelete(ClientContext &context, LogicalDelete &op,
+	                                        unique_ptr<PhysicalOperator> plan) override;
+	unique_ptr<PhysicalOperator> PlanUpdate(ClientContext &context, LogicalUpdate &op,
+	                                        unique_ptr<PhysicalOperator> plan) override;
+	unique_ptr<LogicalOperator> BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table,
+	                                            unique_ptr<LogicalOperator> plan) override;
+
+	DatabaseSize GetDatabaseSize(ClientContext &context) override;
+
+	//! Whether or not this is an in-memory Odbc database
+	bool InMemory() override;
+	string GetDBPath() override;
+
+	//! Returns a reference to the in-memory database (if any)
+	OdbcDB *GetInMemoryDatabase();
+	//! Release the in-memory database (if there is any)
+	void ReleaseInMemoryDatabase();
+
+private:
+	void DropSchema(ClientContext &context, DropInfo &info) override;
+
+private:
+	unique_ptr<OdbcSchemaEntry> main_schema;
+	//! Whether or not the database is in-memory
+	bool in_memory;
+	//! In-memory database - if any
+	OdbcDB in_memory_db;
+	//! The lock maintaing access to the in-memory database
+	mutex in_memory_lock;
+	//! Whether or not there is any active transaction on the in-memory database
+	bool active_in_memory;
+};
+
+} // namespace duckdb

src/include/storage/odbc_delete.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "duckdb/execution/physical_operator.hpp"
+
+namespace duckdb {
+
+class OdbcDelete : public PhysicalOperator {
+public:
+	OdbcDelete(LogicalOperator &op, TableCatalogEntry &table);
+
+	//! The table to delete from
+	TableCatalogEntry &table;
+
+public:
+	// Source interface
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
+
+public:
+	// Sink interface
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+
+	bool ParallelSink() const override {
+		return false;
+	}
+
+	string GetName() const override;
+	string ParamsToString() const override;
+};
+
+} // namespace duckdb

src/include/storage/odbc_index.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "duckdb/execution/physical_operator.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
+
+namespace duckdb {
+
+//! PhysicalCreateSequence represents a CREATE SEQUENCE command
+class OdbcCreateIndex : public PhysicalOperator {
+public:
+	explicit OdbcCreateIndex(unique_ptr<CreateIndexInfo> info, TableCatalogEntry &table);
+
+	unique_ptr<CreateIndexInfo> info;
+	TableCatalogEntry &table;
+
+public:
+	// Source interface
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
+};
+
+} // namespace duckdb

src/include/storage/odbc_index_entry.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
+
+namespace duckdb {
+
+class OdbcIndexEntry : public IndexCatalogEntry {
+public:
+  OdbcIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &info, string table_name);
+
+  string table_name;
+
+public:
+  string GetSchemaName() const override;
+  string GetTableName() const override;
+};
+
+} // namespace duckdb