Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

coerce Arrow BigInt to Number #2164

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/options.js
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ function maybeTypedArrowify(vector, type) {
? vector
: (type === undefined || type === Array) && isArrowDateType(vector.type)
? coerceDates(vector.toArray())
: (type === undefined || type === Array) && isArrowBigIntType(vector.type)
? (type ?? Float64Array).from(vector.toArray(), Number)
: maybeTypedArrayify(vector.toArray(), type);
}

Expand Down Expand Up @@ -625,6 +627,7 @@ function isArrowVector(value) {
// Apache Arrow now represents dates as numbers. We currently only support
// implicit coercion to JavaScript Date objects when the numbers represent
// milliseconds since Unix epoch.
// https://github.com/apache/arrow/blob/cd50c324882ab1419d1728e9adad20d47b185508/js/src/enum.ts#L52-L72
function isArrowDateType(type) {
return (
type &&
Expand All @@ -633,3 +636,12 @@ function isArrowDateType(type) {
type.unit === 1 // millisecond
);
}

// https://github.com/apache/arrow/blob/cd50c324882ab1419d1728e9adad20d47b185508/js/src/enum.ts#L52-L72
function isArrowBigIntType(type) {
return (
type &&
type.typeId === 2 && // int
type.bitWidth >= 64
);
}
173 changes: 173 additions & 0 deletions test/output/bigintNormalize.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
19 changes: 19 additions & 0 deletions test/plots/bigint.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import * as Plot from "@observablehq/plot";
import * as d3 from "d3";
import * as Arrow from "apache-arrow";

const integers = d3.range(40).map((int) => ({
big1: BigInt(int),
Expand All @@ -25,3 +26,21 @@ export async function bigintOrdinal() {
export async function bigintStack() {
return Plot.barY(integers, {x: (d, i) => i % 5, y: "big1"}).plot();
}

export async function bigintNormalize() {
const heights = await d3.csv("data/athletes.csv").then((data) => data.map((d) => d.height));
const table = Arrow.tableFromJSON(
d3.groups(heights, (d) => (d ? +d : NaN)).map(([height, {length}]) => ({height, frequency: BigInt(length)}))
);
return Plot.plot({
height: 500,
x: {percent: true, grid: true},
marks: [
Plot.ruleX([0]),
Plot.ruleY(
table,
Plot.normalizeX("sum", {strokeWidth: 2, y: "height", x: "frequency", tip: {format: {y: ".2f"}}})
)
]
});
}