Skip to content

Commit

Permalink
Merge branch 'main' into flav/viz-draft
Browse files Browse the repository at this point in the history
  • Loading branch information
flvndvd committed Jul 25, 2024
2 parents e0c2b67 + 83edeb4 commit 2609255
Show file tree
Hide file tree
Showing 41 changed files with 568 additions and 816 deletions.
60 changes: 30 additions & 30 deletions connectors/migrations/20230606_deprecate_nango_connection_id.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,30 @@
import { Op } from "sequelize";

import { sequelizeConnection } from "@connectors/resources/storage";
import { ConnectorModel } from "@connectors/resources/storage/models/connector_model";

async function main() {
await ConnectorModel.update(
{
connectionId: sequelizeConnection.col("nangoConnectionId"),
},
{
// @ts-expect-error `connectionId` has been made non-nullable
where: {
connectionId: {
[Op.eq]: null,
},
},
}
);
}

main()
.then(() => {
console.log("Done");
process.exit(0);
})
.catch((e) => {
console.error(e);
process.exit(1);
});
// import { Op } from "sequelize";
//
// import { sequelizeConnection } from "@connectors/resources/storage";
// import { ConnectorModel } from "@connectors/resources/storage/models/connector_model";
//
// async function main() {
// await ConnectorModel.update(
// {
// connectionId: sequelizeConnection.col("nangoConnectionId"),
// },
// {
// // @ts-expect-error `connectionId` has been made non-nullable
// where: {
// connectionId: {
// [Op.eq]: null,
// },
// },
// }
// );
// }
//
// main()
// .then(() => {
// console.log("Done");
// process.exit(0);
// })
// .catch((e) => {
// console.error(e);
// process.exit(1);
// });
2 changes: 2 additions & 0 deletions connectors/src/connectors/github/lib/github_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,8 @@ const EXTENSION_WHITELIST = [
".sql",
".kt",
".kts",
".gradle",
".xml",
];

const SUFFIX_BLACKLIST = [".min.js", ".min.css"];
Expand Down
10 changes: 6 additions & 4 deletions connectors/src/connectors/microsoft/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@ import {
getFilesAndFolders,
getSites,
getTeams,
internalIdFromTypeAndPath,
typeAndPathFromInternalId,
} from "@connectors/connectors/microsoft/lib/graph_api";
import type { MicrosoftNodeType } from "@connectors/connectors/microsoft/lib/types";
import {
getSiteNodesToSync,
internalIdFromTypeAndPath,
typeAndPathFromInternalId,
} from "@connectors/connectors/microsoft/lib/utils";
import {
getRootNodesToSync,
populateDeltas,
} from "@connectors/connectors/microsoft/temporal/activities";
import {
Expand Down Expand Up @@ -309,7 +311,7 @@ export class MicrosoftConnectorManager extends BaseConnectorManager<null> {

await MicrosoftRootResource.batchMakeNew(newResourcesBlobs);

const nodesToSync = await getSiteNodesToSync(this.connectorId);
const nodesToSync = await getRootNodesToSync(this.connectorId);

// poupulates deltas for the nodes so that if incremental sync starts before
// fullsync populated, there's no error
Expand Down
4 changes: 3 additions & 1 deletion connectors/src/connectors/microsoft/lib/content_nodes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import {
getDriveInternalId,
getDriveItemInternalId,
getSiteAPIPath,
} from "@connectors/connectors/microsoft/lib/graph_api";
import {
internalIdFromTypeAndPath,
typeAndPathFromInternalId,
} from "@connectors/connectors/microsoft/lib/graph_api";
} from "@connectors/connectors/microsoft/lib/utils";
import type { MicrosoftNodeResource } from "@connectors/resources/microsoft_resource";

export function getRootNodes(): ContentNode[] {
Expand Down
51 changes: 4 additions & 47 deletions connectors/src/connectors/microsoft/lib/graph_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ import { assertNever } from "@dust-tt/types";
import type { Client } from "@microsoft/microsoft-graph-client";
import type * as MicrosoftGraph from "@microsoft/microsoft-graph-types";

import type { MicrosoftNodeType } from "@connectors/connectors/microsoft/lib/types";
import type { MicrosoftNode } from "@connectors/connectors/microsoft/lib/types";
import { isValidNodeType } from "@connectors/connectors/microsoft/lib/types";
import {
internalIdFromTypeAndPath,
typeAndPathFromInternalId,
} from "@connectors/connectors/microsoft/lib/utils";

export async function getSites(
client: Client,
Expand Down Expand Up @@ -408,51 +410,6 @@ export function itemToMicrosoftNode<T extends keyof MicrosoftEntityMapping>(
}
}

export function internalIdFromTypeAndPath({
nodeType,
itemAPIPath,
}: {
nodeType: MicrosoftNodeType;
itemAPIPath: string;
}): string {
let stringId = "";
if (nodeType === "sites-root" || nodeType === "teams-root") {
stringId = nodeType;
} else {
stringId = `${nodeType}/${itemAPIPath}`;
}
// encode to base64url so the internal id is URL-friendly
return "microsoft-" + Buffer.from(stringId).toString("base64url");
}

export function typeAndPathFromInternalId(internalId: string): {
nodeType: MicrosoftNodeType;
itemAPIPath: string;
} {
if (!internalId.startsWith("microsoft-")) {
throw new Error(`Invalid internal id: ${internalId}`);
}

// decode from base64url
const decodedId = Buffer.from(
internalId.slice("microsoft-".length),
"base64url"
).toString();

if (decodedId === "sites-root" || decodedId === "teams-root") {
return { nodeType: decodedId, itemAPIPath: "" };
}

const [nodeType, ...resourcePathArr] = decodedId.split("/");
if (!nodeType || !isValidNodeType(nodeType)) {
throw new Error(
`Invalid internal id: ${decodedId} with nodeType: ${nodeType}`
);
}

return { nodeType, itemAPIPath: resourcePathArr.join("/") };
}

export function getDriveItemInternalId(item: MicrosoftGraph.DriveItem) {
const { parentReference } = item;

Expand Down
59 changes: 59 additions & 0 deletions connectors/src/connectors/microsoft/lib/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import type { MicrosoftNodeType } from "./types";
import { isValidNodeType } from "./types";

export function internalIdFromTypeAndPath({
nodeType,
itemAPIPath,
}: {
nodeType: MicrosoftNodeType;
itemAPIPath: string;
}): string {
let stringId = "";
if (nodeType === "sites-root" || nodeType === "teams-root") {
stringId = nodeType;
} else {
stringId = `${nodeType}/${itemAPIPath}`;
}
// encode to base64url so the internal id is URL-friendly
return "microsoft-" + Buffer.from(stringId).toString("base64url");
}

export function typeAndPathFromInternalId(internalId: string): {
nodeType: MicrosoftNodeType;
itemAPIPath: string;
} {
if (!internalId.startsWith("microsoft-")) {
throw new Error(`Invalid internal id: ${internalId}`);
}

// decode from base64url
const decodedId = Buffer.from(
internalId.slice("microsoft-".length),
"base64url"
).toString();

if (decodedId === "sites-root" || decodedId === "teams-root") {
return { nodeType: decodedId, itemAPIPath: "" };
}

const [nodeType, ...resourcePathArr] = decodedId.split("/");
if (!nodeType || !isValidNodeType(nodeType)) {
throw new Error(
`Invalid internal id: ${decodedId} with nodeType: ${nodeType}`
);
}

return { nodeType, itemAPIPath: resourcePathArr.join("/") };
}

export function getDriveInternalIdFromItemId(itemId: string) {
const { itemAPIPath } = typeAndPathFromInternalId(itemId);
if (!itemAPIPath.startsWith("/drives/")) {
throw new Error("Unexpected: no drive id for item");
}
const parts = itemAPIPath.split("/");
return internalIdFromTypeAndPath({
nodeType: "drive",
itemAPIPath: `/drives/${parts[2]}`,
});
}
Loading

0 comments on commit 2609255

Please sign in to comment.