[front] feature: Use dust-app for parsing table headers

connectors/src/connectors/google_drive/temporal/spreadsheets.ts

@@ -1,7 +1,6 @@
 import type { ModelId } from "@dust-tt/types";
 import {
   getGoogleSheetTableId,
-  getSanitizedHeaders,
   InvalidStructuredDataHeaderError,
   slugify,
 } from "@dust-tt/types";
@@ -80,38 +79,29 @@ async function upsertTable(
     },
     truncate: true,
     parents: [tableId, ...parents],
+    useAppForHeaderDetection: true,
   });
 
   logger.info(loggerArgs, "[Spreadsheet] Table upserted.");
 }
 
 function findDataRangeAndSelectRows(allRows: string[][]): string[][] {
   // Find the first row with data to determine the range.
-  const firstNonEmptyRow = allRows.find((row) =>
+  const nonEmptyRow = allRows.filter((row) =>
     row.some((cell) => cell.trim() !== "")
   );
-  if (!firstNonEmptyRow) {
-    return []; // No data found.
-  }
-
-  // Identify the range of data: Start at the first non-empty cell and end at the nearest following empty cell or row end.
-  const startIndex = firstNonEmptyRow.findIndex((cell) => cell.trim() !== "");
-  let endIndex = firstNonEmptyRow.findIndex(
-    (cell, idx) => idx > startIndex && cell.trim() === ""
-  );
-  if (endIndex === -1) {
-    endIndex = firstNonEmptyRow.length;
-  }
 
-  // Select only rows and columns within the data range.
-  return allRows
-    .map((row) => row.slice(startIndex, endIndex))
-    .filter((row) => row.some((cell) => cell.trim() !== ""));
+  return nonEmptyRow;
 }
 
 function getValidRows(allRows: string[][], loggerArgs: object): string[][] {
   const filteredRows = findDataRangeAndSelectRows(allRows);
 
+  const maxCols = filteredRows.reduce(
+    (acc, row) => (row.length > acc ? row.length : acc),
+    0
+  );
+
   // We assume that the first row is always the headers.
   // Headers are used to assert the number of cells per row.
   const [rawHeaders] = filteredRows;
@@ -124,25 +114,13 @@ function getValidRows(allRows: string[][], loggerArgs: object): string[][] {
   }
 
   try {
-    const headers = getSanitizedHeaders(rawHeaders);
-
-    const validRows: string[][] = filteredRows.map((row, index) => {
-      // Return raw headers.
-      if (index === 0) {
-        return headers;
-      }
-
+    const validRows: string[][] = filteredRows.map((row) => {
       // If a row has less cells than headers, we fill the gap with empty strings.
-      if (row.length < headers.length) {
-        const shortfall = headers.length - row.length;
+      if (row.length < maxCols) {
+        const shortfall = maxCols - row.length;
         return [...row, ...Array(shortfall).fill("")];
       }
 
-      // If a row has more cells than headers we truncate the row.
-      if (row.length > headers.length) {
-        return row.slice(0, headers.length);
-      }
-
       return row;
     });
 

connectors/src/connectors/microsoft/temporal/spreadsheets.ts

@@ -1,5 +1,5 @@
 import type { Result } from "@dust-tt/types";
-import { Err, getSanitizedHeaders, Ok, slugify } from "@dust-tt/types";
+import { Err, Ok, slugify } from "@dust-tt/types";
 import type { Client } from "@microsoft/microsoft-graph-client";
 import { stringify } from "csv-stringify/sync";
 
@@ -99,6 +99,7 @@ async function upsertTable(
     },
     truncate: true,
     parents,
+    useAppForHeaderDetection: true,
   });
 
   logger.info(loggerArgs, "[Spreadsheet] Table upserted.");
@@ -181,7 +182,7 @@ async function processSheet({
 
   // Assuming the first line as headers, at least one additional data line is required.
   if (rawHeaders && rows.length > 1) {
-    const headers = getSanitizedHeaders(rawHeaders);
+    const headers = rawHeaders;
 
     const parents = [
       worksheetInternalId,

connectors/src/connectors/notion/lib/notion_api.ts

@@ -5,11 +5,7 @@ import type {
   ParsedNotionPage,
   PropertyKeys,
 } from "@dust-tt/types";
-import {
-  assertNever,
-  cacheWithRedis,
-  getSanitizedHeaders,
-} from "@dust-tt/types";
+import { assertNever, cacheWithRedis } from "@dust-tt/types";
 import type { LogLevel } from "@notionhq/client";
 import {
   APIResponseError,
@@ -1009,7 +1005,7 @@ export async function renderDatabaseFromPages({
     )
   );
 
-  const sanitizedHeaders = getSanitizedHeaders(header);
+  const sanitizedHeaders = header;
 
   let csv = await new Promise<string>((resolve, reject) => {
     stringify(

connectors/src/lib/data_sources.ts

@@ -563,6 +563,7 @@ export async function upsertTableFromCsv({
   loggerArgs,
   truncate,
   parents,
+  useAppForHeaderDetection,
 }: {
   dataSourceConfig: DataSourceConfig;
   tableId: string;
@@ -572,6 +573,7 @@ export async function upsertTableFromCsv({
   loggerArgs?: Record<string, string | number>;
   truncate: boolean;
   parents: string[];
+  useAppForHeaderDetection?: boolean;
 }) {
   const localLogger = logger.child({ ...loggerArgs, tableId, tableName });
   const statsDTags = [
@@ -599,6 +601,7 @@ export async function upsertTableFromCsv({
     tableId,
     truncate,
     async: true,
+    useAppForHeaderDetection,
   };
   const dustRequestConfig: AxiosRequestConfig = {
     headers: {

front/components/data_source/DocumentOrTableUploadOrEditModal.tsx

@@ -8,6 +8,7 @@ import {
   Modal,
   Page,
   PlusIcon,
+  SparklesIcon,
   Spinner,
   TrashIcon,
 } from "@dust-tt/sparkle";
@@ -17,8 +18,8 @@ import type {
   CoreAPILightDocument,
   DataSourceViewType,
   LightContentNode,
-  LightWorkspaceType,
   PlanType,
+  WorkspaceType,
 } from "@dust-tt/types";
 import {
   BIG_FILE_SIZE,
@@ -42,7 +43,7 @@ interface DocumentOrTableUploadOrEditModalProps {
   dataSourceView: DataSourceViewType;
   isOpen: boolean;
   onClose: (save: boolean) => void;
-  owner: LightWorkspaceType;
+  owner: WorkspaceType;
   plan: PlanType;
   totalNodesCount: number;
   viewType: ContentNodesViewType;
@@ -98,7 +99,8 @@ export function DocumentOrTableUploadOrEditModal({
   const [uploading, setUploading] = useState(false);
   const [isBigFile, setIsBigFile] = useState(false);
   const [developerOptionsVisible, setDeveloperOptionsVisible] = useState(false);
-
+  const [useAppForHeaderDetection, setUseAppForHeaderDetection] =
+    useState(false);
   const isTable = viewType == "tables";
   const initialId = contentNode?.internalId;
 
@@ -195,6 +197,7 @@ export function DocumentOrTableUploadOrEditModal({
         parents: [],
         truncate: false,
         async: false,
+        useAppForHeaderDetection,
       });
 
       const res = await fetch(endpoint, {
@@ -478,6 +481,29 @@ export function DocumentOrTableUploadOrEditModal({
                 )}
               </div>
 
+              {isTable &&
+                owner.flags.includes("use_app_for_header_detection") && (
+                  <div>
+                    <Page.SectionHeader
+                      title="Enable header detection"
+                      description={
+                        "Use the LLM model to detect headers in the CSV file."
+                      }
+                      action={{
+                        label: useAppForHeaderDetection ? "Disable" : "Enable",
+                        variant: useAppForHeaderDetection
+                          ? "primary"
+                          : "tertiary",
+                        icon: SparklesIcon,
+                        onClick: () =>
+                          setUseAppForHeaderDetection(
+                            !useAppForHeaderDetection
+                          ),
+                      }}
+                    />
+                  </div>
+                )}
+
               {!isTable && (
                 <div>
                   <Page.SectionHeader

front/lib/api/config.ts

@@ -82,7 +82,8 @@ const config = {
       url:
         EnvironmentConfig.getOptionalEnvVariable("DUST_PROD_API") ??
         PRODUCTION_DUST_API,
-      nodeEnv: EnvironmentConfig.getEnvVariable("NODE_ENV"),
+      nodeEnv:
+        EnvironmentConfig.getOptionalEnvVariable("NODE_ENV") || "development",
     };
   },
   getOAuthAPIConfig: (): { url: string; apiKey: string | null } => {

front/lib/api/data_sources.ts

@@ -367,6 +367,7 @@ export async function upsertTable({
   async,
   dataSource,
   auth,
+  useAppForHeaderDetection,
 }: {
   tableId?: string | null;
   name: string;
@@ -379,6 +380,7 @@ export async function upsertTable({
   async: boolean;
   dataSource: DataSourceResource;
   auth: Authenticator;
+  useAppForHeaderDetection?: boolean;
 }) {
   const nonNullTableId = tableId ?? generateLegacyModelSId();
   const tableParents: string[] = parents ?? [];
@@ -387,9 +389,17 @@ export async function upsertTable({
     tableParents.push(nonNullTableId);
   }
 
+  const useAppForHeaderDetectionFlag = auth
+    .getNonNullableWorkspace()
+    .flags.includes("use_app_for_header_detection");
+
+  const useApp = !!useAppForHeaderDetection && useAppForHeaderDetectionFlag;
+
   if (async) {
     // Ensure the CSV is valid before enqueuing the upsert.
-    const csvRowsRes = csv ? await rowsFromCsv(csv) : null;
+    const csvRowsRes = csv
+      ? await rowsFromCsv({ auth, csv, useAppForHeaderDetection: useApp })
+      : null;
     if (csvRowsRes?.isErr()) {
       return csvRowsRes;
     }
@@ -406,6 +416,7 @@ export async function upsertTable({
         tableParents,
         csv: csv ?? null,
         truncate,
+        useAppForHeaderDetection: useApp,
       },
     });
     if (enqueueRes.isErr()) {
@@ -426,6 +437,7 @@ export async function upsertTable({
     tableParents,
     csv: csv ?? null,
     truncate,
+    useAppForHeaderDetection: useApp,
   });
 
   return tableRes;