Revert "[front] feature: Use dust-app for parsing table headers (#7077)"

This reverts commit 722579d.

connectors/src/connectors/google_drive/temporal/spreadsheets.ts

@@ -1,6 +1,7 @@
 import type { ModelId } from "@dust-tt/types";
 import {
   getGoogleSheetTableId,
+  getSanitizedHeaders,
   InvalidStructuredDataHeaderError,
   slugify,
 } from "@dust-tt/types";
@@ -79,29 +80,38 @@ async function upsertTable(
     },
     truncate: true,
     parents: [tableId, ...parents],
-    useAppForHeaderDetection: true,
   });
 
   logger.info(loggerArgs, "[Spreadsheet] Table upserted.");
 }
 
 function findDataRangeAndSelectRows(allRows: string[][]): string[][] {
   // Find the first row with data to determine the range.
-  const nonEmptyRow = allRows.filter((row) =>
+  const firstNonEmptyRow = allRows.find((row) =>
     row.some((cell) => cell.trim() !== "")
   );
+  if (!firstNonEmptyRow) {
+    return []; // No data found.
+  }
+
+  // Identify the range of data: Start at the first non-empty cell and end at the nearest following empty cell or row end.
+  const startIndex = firstNonEmptyRow.findIndex((cell) => cell.trim() !== "");
+  let endIndex = firstNonEmptyRow.findIndex(
+    (cell, idx) => idx > startIndex && cell.trim() === ""
+  );
+  if (endIndex === -1) {
+    endIndex = firstNonEmptyRow.length;
+  }
 
-  return nonEmptyRow;
+  // Select only rows and columns within the data range.
+  return allRows
+    .map((row) => row.slice(startIndex, endIndex))
+    .filter((row) => row.some((cell) => cell.trim() !== ""));
 }
 
 function getValidRows(allRows: string[][], loggerArgs: object): string[][] {
   const filteredRows = findDataRangeAndSelectRows(allRows);
 
-  const maxCols = filteredRows.reduce(
-    (acc, row) => (row.length > acc ? row.length : acc),
-    0
-  );
-
   // We assume that the first row is always the headers.
   // Headers are used to assert the number of cells per row.
   const [rawHeaders] = filteredRows;
@@ -114,13 +124,25 @@ function getValidRows(allRows: string[][], loggerArgs: object): string[][] {
   }
 
   try {
-    const validRows: string[][] = filteredRows.map((row) => {
+    const headers = getSanitizedHeaders(rawHeaders);
+
+    const validRows: string[][] = filteredRows.map((row, index) => {
+      // Return raw headers.
+      if (index === 0) {
+        return headers;
+      }
+
       // If a row has less cells than headers, we fill the gap with empty strings.
-      if (row.length < maxCols) {
-        const shortfall = maxCols - row.length;
+      if (row.length < headers.length) {
+        const shortfall = headers.length - row.length;
         return [...row, ...Array(shortfall).fill("")];
       }
 
+      // If a row has more cells than headers we truncate the row.
+      if (row.length > headers.length) {
+        return row.slice(0, headers.length);
+      }
+
       return row;
     });
 

connectors/src/connectors/microsoft/temporal/spreadsheets.ts

@@ -1,5 +1,5 @@
 import type { Result } from "@dust-tt/types";
-import { Err, Ok, slugify } from "@dust-tt/types";
+import { Err, getSanitizedHeaders, Ok, slugify } from "@dust-tt/types";
 import type { Client } from "@microsoft/microsoft-graph-client";
 import { stringify } from "csv-stringify/sync";
 
@@ -99,7 +99,6 @@ async function upsertTable(
     },
     truncate: true,
     parents,
-    useAppForHeaderDetection: true,
   });
 
   logger.info(loggerArgs, "[Spreadsheet] Table upserted.");
@@ -182,7 +181,7 @@ async function processSheet({
 
   // Assuming the first line as headers, at least one additional data line is required.
   if (rawHeaders && rows.length > 1) {
-    const headers = rawHeaders;
+    const headers = getSanitizedHeaders(rawHeaders);
 
     const parents = [
       worksheetInternalId,

connectors/src/connectors/notion/lib/notion_api.ts

@@ -5,7 +5,11 @@ import type {
   ParsedNotionPage,
   PropertyKeys,
 } from "@dust-tt/types";
-import { assertNever, cacheWithRedis } from "@dust-tt/types";
+import {
+  assertNever,
+  cacheWithRedis,
+  getSanitizedHeaders,
+} from "@dust-tt/types";
 import type { LogLevel } from "@notionhq/client";
 import {
   APIResponseError,
@@ -1005,7 +1009,7 @@ export async function renderDatabaseFromPages({
     )
   );
 
-  const sanitizedHeaders = header;
+  const sanitizedHeaders = getSanitizedHeaders(header);
 
   let csv = await new Promise<string>((resolve, reject) => {
     stringify(

connectors/src/lib/data_sources.ts

@@ -563,7 +563,6 @@ export async function upsertTableFromCsv({
   loggerArgs,
   truncate,
   parents,
-  useAppForHeaderDetection,
 }: {
   dataSourceConfig: DataSourceConfig;
   tableId: string;
@@ -573,7 +572,6 @@ export async function upsertTableFromCsv({
   loggerArgs?: Record<string, string | number>;
   truncate: boolean;
   parents: string[];
-  useAppForHeaderDetection?: boolean;
 }) {
   const localLogger = logger.child({ ...loggerArgs, tableId, tableName });
   const statsDTags = [
@@ -601,7 +599,6 @@ export async function upsertTableFromCsv({
     tableId,
     truncate,
     async: true,
-    useAppForHeaderDetection,
   };
   const dustRequestConfig: AxiosRequestConfig = {
     headers: {

front/components/data_source/DocumentOrTableUploadOrEditModal.tsx

@@ -8,7 +8,6 @@ import {
   Modal,
   Page,
   PlusIcon,
-  SparklesIcon,
   Spinner,
   TrashIcon,
 } from "@dust-tt/sparkle";
@@ -18,8 +17,8 @@ import type {
   CoreAPILightDocument,
   DataSourceViewType,
   LightContentNode,
+  LightWorkspaceType,
   PlanType,
-  WorkspaceType,
 } from "@dust-tt/types";
 import {
   BIG_FILE_SIZE,
@@ -43,7 +42,7 @@ interface DocumentOrTableUploadOrEditModalProps {
   dataSourceView: DataSourceViewType;
   isOpen: boolean;
   onClose: (save: boolean) => void;
-  owner: WorkspaceType;
+  owner: LightWorkspaceType;
   plan: PlanType;
   totalNodesCount: number;
   viewType: ContentNodesViewType;
@@ -99,8 +98,7 @@ export function DocumentOrTableUploadOrEditModal({
   const [uploading, setUploading] = useState(false);
   const [isBigFile, setIsBigFile] = useState(false);
   const [developerOptionsVisible, setDeveloperOptionsVisible] = useState(false);
-  const [useAppForHeaderDetection, setUseAppForHeaderDetection] =
-    useState(false);
+
   const isTable = viewType == "tables";
   const initialId = contentNode?.internalId;
 
@@ -197,7 +195,6 @@ export function DocumentOrTableUploadOrEditModal({
         parents: [],
         truncate: false,
         async: false,
-        useAppForHeaderDetection,
       });
 
       const res = await fetch(endpoint, {
@@ -481,29 +478,6 @@ export function DocumentOrTableUploadOrEditModal({
                 )}
               </div>
 
-              {isTable &&
-                owner.flags.includes("use_app_for_header_detection") && (
-                  <div>
-                    <Page.SectionHeader
-                      title="Enable header detection"
-                      description={
-                        "Use the LLM model to detect headers in the CSV file."
-                      }
-                      action={{
-                        label: useAppForHeaderDetection ? "Disable" : "Enable",
-                        variant: useAppForHeaderDetection
-                          ? "primary"
-                          : "tertiary",
-                        icon: SparklesIcon,
-                        onClick: () =>
-                          setUseAppForHeaderDetection(
-                            !useAppForHeaderDetection
-                          ),
-                      }}
-                    />
-                  </div>
-                )}
-
               {!isTable && (
                 <div>
                   <Page.SectionHeader

front/lib/api/config.ts

@@ -82,8 +82,7 @@ const config = {
       url:
         EnvironmentConfig.getOptionalEnvVariable("DUST_PROD_API") ??
         PRODUCTION_DUST_API,
-      nodeEnv:
-        EnvironmentConfig.getOptionalEnvVariable("NODE_ENV") || "development",
+      nodeEnv: EnvironmentConfig.getEnvVariable("NODE_ENV"),
     };
   },
   getOAuthAPIConfig: (): { url: string; apiKey: string | null } => {

front/lib/api/data_sources.ts

@@ -367,7 +367,6 @@ export async function upsertTable({
   async,
   dataSource,
   auth,
-  useAppForHeaderDetection,
 }: {
   tableId?: string | null;
   name: string;
@@ -380,7 +379,6 @@ export async function upsertTable({
   async: boolean;
   dataSource: DataSourceResource;
   auth: Authenticator;
-  useAppForHeaderDetection?: boolean;
 }) {
   const nonNullTableId = tableId ?? generateLegacyModelSId();
   const tableParents: string[] = parents ?? [];
@@ -389,17 +387,9 @@ export async function upsertTable({
     tableParents.push(nonNullTableId);
   }
 
-  const useAppForHeaderDetectionFlag = auth
-    .getNonNullableWorkspace()
-    .flags.includes("use_app_for_header_detection");
-
-  const useApp = !!useAppForHeaderDetection && useAppForHeaderDetectionFlag;
-
   if (async) {
     // Ensure the CSV is valid before enqueuing the upsert.
-    const csvRowsRes = csv
-      ? await rowsFromCsv({ auth, csv, useAppForHeaderDetection: useApp })
-      : null;
+    const csvRowsRes = csv ? await rowsFromCsv(csv) : null;
     if (csvRowsRes?.isErr()) {
       return csvRowsRes;
     }
@@ -416,7 +406,6 @@ export async function upsertTable({
         tableParents,
         csv: csv ?? null,
         truncate,
-        useAppForHeaderDetection: useApp,
       },
     });
     if (enqueueRes.isErr()) {
@@ -437,7 +426,6 @@ export async function upsertTable({
     tableParents,
     csv: csv ?? null,
     truncate,
-    useAppForHeaderDetection: useApp,
   });
 
   return tableRes;