Skip to content

Commit

Permalink
perf(memtables): avoid converting to pyarrow for all memtables
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud committed Jul 24, 2024
1 parent 74ef444 commit a6c0525
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 1 deletion.
2 changes: 1 addition & 1 deletion ibis/backends/duckdb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1578,7 +1578,7 @@ def _register_in_memory_table(self, op: ops.InMemoryTable) -> None:
self.con.table(name)
except (duckdb.CatalogException, duckdb.InvalidInputException):
# only register if we haven't already done so
self.con.register(name, op.data.to_pyarrow(op.schema))
self.con.register(name, op.data.to_native())

def _register_udfs(self, expr: ir.Expr) -> None:
con = self.con
Expand Down
8 changes: 8 additions & 0 deletions ibis/formats/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,14 @@ def to_pyarrow(self, schema: Schema) -> pa.Table: # pragma: no cover
def to_polars(self, schema: Schema) -> pl.DataFrame: # pragma: no cover
"""Convert this input to a Polars DataFrame."""

def to_native(self) -> T:
"""Convert to underlying native format.
Useful when the backend can handle the in-memory data structure
directly.
"""
return self.obj

def to_pyarrow_bytes(self, schema: Schema) -> bytes:
import pyarrow as pa
import pyarrow_hotfix # noqa: F401
Expand Down

0 comments on commit a6c0525

Please sign in to comment.