From bfec3d324f67d407eb72ecf7b2006d35735f57af Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Mon, 8 Jun 2026 09:05:31 +0000 Subject: [PATCH 1/9] feat: add BigQuery connector schema, migrations and Zod validation types Phase 1 of the BigQuery warehouse connector. Adds Prisma models for BigQueryConnection, BigQuerySync and BigQuerySyncRun with four supporting enums, two Prisma migrations, typed JSON column via PrismaJson namespace, Zod schemas (zBigQuerySyncConfig + column mapping variants) in @openpanel/validation, and the @google-cloud/bigquery dependency. --- packages/db/package.json | 1 + .../migration.sql | 2 + .../migration.sql | 76 ++++++ packages/db/prisma/schema.prisma | 96 ++++++- packages/db/src/types.ts | 4 +- packages/validation/src/index.ts | 40 +++ pnpm-lock.yaml | 246 +++++++++++++++++- 7 files changed, 445 insertions(+), 20 deletions(-) mode change 100644 => 100755 packages/db/package.json create mode 100644 packages/db/prisma/migrations/20260607120000_add_event_meta_description/migration.sql create mode 100644 packages/db/prisma/migrations/20260608090217_add_bigquery_connector/migration.sql mode change 100644 => 100755 packages/db/src/types.ts mode change 100644 => 100755 packages/validation/src/index.ts mode change 100644 => 100755 pnpm-lock.yaml diff --git a/packages/db/package.json b/packages/db/package.json old mode 100644 new mode 100755 index 634a264b1..58b4ff42f --- a/packages/db/package.json +++ b/packages/db/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "@clickhouse/client": "^1.18.5", + "@google-cloud/bigquery": "^7.9.1", "@openpanel/common": "workspace:*", "@openpanel/constants": "workspace:*", "@openpanel/json": "workspace:*", diff --git a/packages/db/prisma/migrations/20260607120000_add_event_meta_description/migration.sql b/packages/db/prisma/migrations/20260607120000_add_event_meta_description/migration.sql new file mode 100644 index 000000000..7ab277bb8 --- /dev/null +++ b/packages/db/prisma/migrations/20260607120000_add_event_meta_description/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."event_meta" ADD COLUMN "description" TEXT; diff --git a/packages/db/prisma/migrations/20260608090217_add_bigquery_connector/migration.sql b/packages/db/prisma/migrations/20260608090217_add_bigquery_connector/migration.sql new file mode 100644 index 000000000..90443c2c0 --- /dev/null +++ b/packages/db/prisma/migrations/20260608090217_add_bigquery_connector/migration.sql @@ -0,0 +1,76 @@ +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncMappingType" AS ENUM ('events', 'profiles'); + +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncMode" AS ENUM ('append', 'full'); + +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncSchedule" AS ENUM ('hourly', 'daily', 'weekly'); + +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncRunStatus" AS ENUM ('pending', 'running', 'completed', 'failed'); + +-- CreateTable +CREATE TABLE "public"."bigquery_connections" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "projectId" TEXT NOT NULL, + "gcpProjectId" TEXT NOT NULL, + "serviceAccountEmail" TEXT NOT NULL, + "serviceAccountJsonEncrypted" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "bigquery_connections_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."bigquery_syncs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "connectionId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "displayName" TEXT NOT NULL, + "dataset" TEXT NOT NULL, + "tableName" TEXT NOT NULL, + "mappingType" "public"."BigQuerySyncMappingType" NOT NULL, + "syncMode" "public"."BigQuerySyncMode" NOT NULL, + "schedule" "public"."BigQuerySyncSchedule" NOT NULL, + "columnMapping" JSONB NOT NULL, + "lastCursor" TEXT, + "lastSyncedAt" TIMESTAMP(3), + "lastSyncStatus" TEXT, + "lastSyncError" TEXT, + "isEnabled" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "bigquery_syncs_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."bigquery_sync_runs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "syncId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "status" "public"."BigQuerySyncRunStatus" NOT NULL DEFAULT 'pending', + "rowCount" INTEGER NOT NULL DEFAULT 0, + "errorMessage" TEXT, + "startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TIMESTAMP(3), + + CONSTRAINT "bigquery_sync_runs_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "bigquery_connections_projectId_key" ON "public"."bigquery_connections"("projectId"); + +-- AddForeignKey +ALTER TABLE "public"."bigquery_connections" ADD CONSTRAINT "bigquery_connections_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."bigquery_syncs" ADD CONSTRAINT "bigquery_syncs_connectionId_fkey" FOREIGN KEY ("connectionId") REFERENCES "public"."bigquery_connections"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."bigquery_syncs" ADD CONSTRAINT "bigquery_syncs_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."bigquery_sync_runs" ADD CONSTRAINT "bigquery_sync_runs_syncId_fkey" FOREIGN KEY ("syncId") REFERENCES "public"."bigquery_syncs"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index a171dc654..238308603 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -251,8 +251,10 @@ model Project { notificationRules NotificationRule[] notifications Notification[] imports Import[] - gscConnection GscConnection? - cohorts Cohort[] + gscConnection GscConnection? + bigQueryConnection BigQueryConnection? + bigQuerySyncs BigQuerySync[] + cohorts Cohort[] // When deleteAt > now(), the project will be deleted deleteAt DateTime? @@ -500,12 +502,13 @@ model ShareWidget { } model EventMeta { - id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid - name String - conversion Boolean? - color String? - icon String? - projectId String + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + name String + conversion Boolean? + color String? + icon String? + description String? + projectId String project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) createdAt DateTime @default(now()) @@ -701,6 +704,83 @@ model GscConnection { @@map("gsc_connections") } +enum BigQuerySyncMappingType { + events + profiles +} + +enum BigQuerySyncMode { + append + full +} + +enum BigQuerySyncSchedule { + hourly + daily + weekly +} + +enum BigQuerySyncRunStatus { + pending + running + completed + failed +} + +model BigQueryConnection { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + projectId String @unique + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + gcpProjectId String + serviceAccountEmail String + serviceAccountJsonEncrypted String + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + syncs BigQuerySync[] + + @@map("bigquery_connections") +} + +model BigQuerySync { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + connectionId String @db.Uuid + connection BigQueryConnection @relation(fields: [connectionId], references: [id], onDelete: Cascade) + projectId String + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + displayName String + dataset String + tableName String + mappingType BigQuerySyncMappingType + syncMode BigQuerySyncMode + schedule BigQuerySyncSchedule + /// [IBigQueryColumnMapping] + columnMapping Json + lastCursor String? + lastSyncedAt DateTime? + lastSyncStatus String? + lastSyncError String? + isEnabled Boolean @default(true) + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + runs BigQuerySyncRun[] + + @@map("bigquery_syncs") +} + +model BigQuerySyncRun { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + syncId String @db.Uuid + sync BigQuerySync @relation(fields: [syncId], references: [id], onDelete: Cascade) + projectId String + status BigQuerySyncRunStatus @default(pending) + rowCount Int @default(0) + errorMessage String? + startedAt DateTime @default(now()) + completedAt DateTime? + + @@map("bigquery_sync_runs") +} + model EmailUnsubscribe { id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid email String diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts old mode 100644 new mode 100755 index aacd6d1e7..8e05d74e2 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -1,11 +1,12 @@ import type { CohortDefinition, + IBigQueryColumnMapping as IBigQueryColumnMappingType, IImportConfig, IIntegrationConfig, INotificationRuleConfig, + InsightPayload, IProjectFilters, IWidgetOptions, - InsightPayload, } from '@openpanel/validation'; import type { IClickhouseBotEvent, @@ -27,6 +28,7 @@ declare global { type IPrismaClickhouseProfile = IClickhouseProfile; type IPrismaClickhouseBotEvent = IClickhouseBotEvent; type IPrismaCohortDefinition = CohortDefinition; + type IBigQueryColumnMapping = IBigQueryColumnMappingType; // Each ChatMessage row stores one Better Agent `ConversationItem` // (message, tool call, or tool result) as JSON. Typed as `unknown[]` // here to avoid pulling `@better-agent/core` into @openpanel/db's diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts old mode 100644 new mode 100755 index 4b2fd87cb..93e256a9b --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -692,6 +692,46 @@ export const zCreateImport = z.object({ export type ICreateImport = z.infer; +export const zBigQueryColumnMappingEvents = z.object({ + eventName: z.string().optional(), + profileId: z.string().optional(), + deviceId: z.string().optional(), + timestamp: z.string().optional(), + insertTime: z.string().optional(), +}); + +export const zBigQueryColumnMappingProfiles = z.object({ + profileIdColumn: z.string().min(1), + firstName: z.string().optional(), + lastName: z.string().optional(), + email: z.string().optional(), + avatar: z.string().optional(), +}); + +export const zBigQuerySyncConfig = z.object({ + displayName: z.string().min(1).max(100), + dataset: z.string().min(1), + tableName: z.string().min(1), + mappingType: z.enum(['events', 'profiles']), + syncMode: z.enum(['append', 'full']), + schedule: z.enum(['hourly', 'daily', 'weekly']), + columnMapping: z.union([ + zBigQueryColumnMappingEvents, + zBigQueryColumnMappingProfiles, + ]), +}); + +export type IBigQueryColumnMappingEvents = z.infer< + typeof zBigQueryColumnMappingEvents +>; +export type IBigQueryColumnMappingProfiles = z.infer< + typeof zBigQueryColumnMappingProfiles +>; +export type IBigQueryColumnMapping = + | IBigQueryColumnMappingEvents + | IBigQueryColumnMappingProfiles; +export type IBigQuerySyncConfig = z.infer; + export * from './types.insights'; export * from './types.validation'; export * from './track.validation'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml old mode 100644 new mode 100755 index a7e168cf0..f5abed03d --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1214,6 +1214,9 @@ importers: '@clickhouse/client': specifier: ^1.18.5 version: 1.18.5 + '@google-cloud/bigquery': + specifier: ^7.9.1 + version: 7.9.4 '@openpanel/common': specifier: workspace:* version: link:../common @@ -4827,6 +4830,30 @@ packages: '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} + '@google-cloud/bigquery@7.9.4': + resolution: {integrity: sha512-C7jeI+9lnCDYK3cRDujcBsPgiwshWKn/f0BiaJmClplfyosCLfWE83iGQ0eKH113UZzjR9c9q7aZQg0nU388sw==} + engines: {node: '>=14.0.0'} + + '@google-cloud/common@5.0.2': + resolution: {integrity: sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==} + engines: {node: '>=14.0.0'} + + '@google-cloud/paginator@5.0.2': + resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==} + engines: {node: '>=14.0.0'} + + '@google-cloud/precise-date@4.0.0': + resolution: {integrity: sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==} + engines: {node: '>=14.0.0'} + + '@google-cloud/projectify@4.0.0': + resolution: {integrity: sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==} + engines: {node: '>=14.0.0'} + + '@google-cloud/promisify@4.0.0': + resolution: {integrity: sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==} + engines: {node: '>=14'} + '@graphql-typed-document-node/core@3.2.0': resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==} peerDependencies: @@ -9370,6 +9397,10 @@ packages: '@types/react-dom': optional: true + '@tootallnate/once@2.0.1': + resolution: {integrity: sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ==} + engines: {node: '>= 10'} + '@trpc-limiter/core@1.0.0': resolution: {integrity: sha512-Wjq2oTCmCdwNbZKRfKpzpl1Um9QXGE8OHXOab+EUSj5Wk+26I/V6Vs2p0VBFAz6eEsBn36982cCC/vaaznA8+Q==} engines: {node: '>=16'} @@ -9466,6 +9497,9 @@ packages: '@types/bunyan@1.8.11': resolution: {integrity: sha512-758fRH7umIMk5qt5ELmRMff4mLDlN+xyYzC+dkPTdKwbSkJFvz6xwyScrytPU0QIBbRRwbiE8/BIg8bpajerNQ==} + '@types/caseless@0.12.5': + resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==} + '@types/compression@1.7.5': resolution: {integrity: sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==} @@ -9844,6 +9878,9 @@ packages: '@types/request-ip@0.0.41': resolution: {integrity: sha512-Qzz0PM2nSZej4lsLzzNfADIORZhhxO7PED0fXpg4FjXiHuJ/lMyUg+YFF5q8x9HPZH3Gl6N+NOM8QZjItNgGKg==} + '@types/request@2.48.13': + resolution: {integrity: sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==} + '@types/resolve@1.20.2': resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} @@ -9889,6 +9926,9 @@ packages: '@types/through@0.0.33': resolution: {integrity: sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==} + '@types/tough-cookie@4.0.5': + resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} + '@types/triple-beam@1.3.5': resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} @@ -9949,6 +9989,7 @@ packages: '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} + deprecated: Potential CWE-502 - Update to 1.3.1 or higher '@unhead/vue@2.0.19': resolution: {integrity: sha512-7BYjHfOaoZ9+ARJkT10Q2TjnTUqDXmMpfakIAsD/hXiuff1oqWg1xeXT5+MomhNcC15HbiABpbbBmITLSHxdKg==} @@ -10634,6 +10675,9 @@ packages: resolution: {integrity: sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==} engines: {node: '>=0.6'} + big.js@6.2.2: + resolution: {integrity: sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==} + bignumber.js@9.1.2: resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} @@ -12697,6 +12741,10 @@ packages: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} + form-data@2.5.5: + resolution: {integrity: sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==} + engines: {node: '>= 0.12'} + form-data@3.0.4: resolution: {integrity: sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==} engines: {node: '>= 6'} @@ -13160,6 +13208,10 @@ packages: peerDependencies: csstype: ^3.0.10 + google-auth-library@9.15.1: + resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} + engines: {node: '>=14'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -13183,6 +13235,10 @@ packages: peerDependencies: ioredis: '>=5' + gtoken@7.1.0: + resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} + engines: {node: '>=14.0.0'} + gzip-size@6.0.0: resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} engines: {node: '>=10'} @@ -13355,6 +13411,9 @@ packages: resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} engines: {node: '>=18'} + html-entities@2.6.0: + resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==} + html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} @@ -13385,6 +13444,10 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} + http-proxy-agent@5.0.0: + resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==} + engines: {node: '>= 6'} + http-proxy-agent@7.0.2: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} @@ -13930,6 +13993,10 @@ packages: resolution: {integrity: sha512-jv+8jaWCl0g2lSBkNSVXdzfBA0npK1HGC2KtWM9FumFRoGS94g3NbCCLVnCYHLjp4GrW2KZeeSTMo5ddtznmGw==} engines: {node: '>=18'} + is@3.3.2: + resolution: {integrity: sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==} + engines: {node: '>= 0.4'} + isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} @@ -14158,9 +14225,15 @@ packages: jwa@1.4.1: resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + jws@3.2.2: resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + kafkajs@2.2.4: resolution: {integrity: sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==} engines: {node: '>=14.0.0'} @@ -16822,6 +16895,7 @@ packages: recharts@2.15.4: resolution: {integrity: sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==} engines: {node: '>=14'} + deprecated: 1.x and 2.x branches are no longer active. Bump to Recharts v3 to receive latest features and bugfixes. See https://github.com/recharts/recharts/wiki/3.0-migration-guide peerDependencies: react: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 @@ -17096,6 +17170,10 @@ packages: retext@9.0.0: resolution: {integrity: sha512-sbMDcpHCNjvlheSgMfEcVrZko3cDzdbe1x/e7G66dFp0Ff7Mldvi2uv6JkJQzdRcvLYE8CA8Oe8siQx8ZOgTcA==} + retry-request@7.0.2: + resolution: {integrity: sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==} + engines: {node: '>=14'} + retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -17714,6 +17792,9 @@ packages: resolution: {integrity: sha512-uyQK/mx5QjHun80FLJTfaWE7JtwfRMKBLkMne6udYOmvH0CawotVa7TfgYHzAnpphn4+TweIx1QKMnRIbipmUg==} engines: {node: '>= 0.10.0'} + stream-events@1.0.5: + resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==} + stream-shift@1.0.3: resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==} @@ -17827,6 +17908,9 @@ packages: structured-headers@0.4.1: resolution: {integrity: sha512-0MP/Cxx5SzeeZ10p/bZI0S6MpgD+yxAhi1BOQ34jgnMXsCq3j1t6tQnZu+KdlL7dvJTLT3g9xN8tl10TqgFMcg==} + stubs@3.0.0: + resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} + style-mod@4.1.3: resolution: {integrity: sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==} @@ -17976,6 +18060,10 @@ packages: tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + teeny-request@9.0.0: + resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==} + engines: {node: '>=14'} + temp-dir@1.0.0: resolution: {integrity: sha512-xZFXEGbG7SNC3itwBzI3RYjq/cEhBkx2hJuKGIUOcEULmkQExXiHat2z/qkISYsuR+IKumhEfKKbV5qXmhICFQ==} engines: {node: '>=4'} @@ -21102,7 +21190,7 @@ snapshots: dependencies: '@babel/template': 7.28.6 '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -22203,7 +22291,7 @@ snapshots: '@babel/core': 7.28.3 '@babel/helper-module-transforms': 7.28.6(@babel/core@7.28.3) '@babel/helper-plugin-utils': 7.28.6 - '@babel/helper-validator-identifier': 7.28.5 + '@babel/helper-validator-identifier': 7.29.7 '@babel/traverse': 7.29.0 transitivePeerDependencies: - supports-color @@ -22214,7 +22302,7 @@ snapshots: '@babel/core': 7.28.5 '@babel/helper-module-transforms': 7.28.6(@babel/core@7.28.5) '@babel/helper-plugin-utils': 7.28.6 - '@babel/helper-validator-identifier': 7.28.5 + '@babel/helper-validator-identifier': 7.29.7 '@babel/traverse': 7.29.0 transitivePeerDependencies: - supports-color @@ -22583,7 +22671,7 @@ snapshots: '@babel/helper-module-imports': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.28.5) - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -22594,7 +22682,7 @@ snapshots: '@babel/helper-module-imports': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0) - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -23061,7 +23149,7 @@ snapshots: dependencies: '@babel/core': 7.28.3 '@babel/helper-plugin-utils': 7.28.6 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 esutils: 2.0.3 optional: true @@ -23069,7 +23157,7 @@ snapshots: dependencies: '@babel/core': 7.28.5 '@babel/helper-plugin-utils': 7.28.6 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 esutils: 2.0.3 '@babel/preset-react@7.28.5(@babel/core@7.28.5)': @@ -24238,8 +24326,8 @@ snapshots: dependencies: '@babel/core': 7.29.0 '@babel/generator': 7.29.1 - '@babel/parser': 7.29.2 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 '@expo/config': 8.5.4 '@expo/env': 0.2.3 '@expo/json-file': 8.3.3 @@ -24484,6 +24572,49 @@ snapshots: '@gar/promisify@1.1.3': {} + '@google-cloud/bigquery@7.9.4': + dependencies: + '@google-cloud/common': 5.0.2 + '@google-cloud/paginator': 5.0.2 + '@google-cloud/precise-date': 4.0.0 + '@google-cloud/promisify': 4.0.0 + arrify: 2.0.1 + big.js: 6.2.2 + duplexify: 4.1.3 + extend: 3.0.2 + is: 3.3.2 + stream-events: 1.0.5 + uuid: 9.0.1 + transitivePeerDependencies: + - encoding + - supports-color + + '@google-cloud/common@5.0.2': + dependencies: + '@google-cloud/projectify': 4.0.0 + '@google-cloud/promisify': 4.0.0 + arrify: 2.0.1 + duplexify: 4.1.3 + extend: 3.0.2 + google-auth-library: 9.15.1 + html-entities: 2.6.0 + retry-request: 7.0.2 + teeny-request: 9.0.0 + transitivePeerDependencies: + - encoding + - supports-color + + '@google-cloud/paginator@5.0.2': + dependencies: + arrify: 2.0.1 + extend: 3.0.2 + + '@google-cloud/precise-date@4.0.0': {} + + '@google-cloud/projectify@4.0.0': {} + + '@google-cloud/promisify@4.0.0': {} + '@graphql-typed-document-node/core@3.2.0(graphql@15.8.0)': dependencies: graphql: 15.8.0 @@ -29914,6 +30045,8 @@ snapshots: '@types/react': 19.2.7 '@types/react-dom': 19.2.3(@types/react@19.2.7) + '@tootallnate/once@2.0.1': {} + '@trpc-limiter/core@1.0.0(@trpc/client@11.17.0(@trpc/server@11.17.0(typescript@5.9.3))(typescript@5.9.3))(@trpc/server@11.17.0(typescript@5.9.3))': dependencies: '@trpc/client': 11.17.0(@trpc/server@11.17.0(typescript@5.9.3))(typescript@5.9.3) @@ -30017,6 +30150,8 @@ snapshots: dependencies: '@types/node': 20.19.24 + '@types/caseless@0.12.5': {} + '@types/compression@1.7.5': dependencies: '@types/express': 5.0.3 @@ -30461,6 +30596,13 @@ snapshots: dependencies: '@types/node': 20.19.24 + '@types/request@2.48.13': + dependencies: + '@types/caseless': 0.12.5 + '@types/node': 20.19.24 + '@types/tough-cookie': 4.0.5 + form-data: 2.5.5 + '@types/resolve@1.20.2': {} '@types/retry@0.12.0': {} @@ -30511,6 +30653,8 @@ snapshots: dependencies: '@types/node': 20.19.24 + '@types/tough-cookie@4.0.5': {} + '@types/triple-beam@1.3.5': {} '@types/tsscmp@1.0.2': {} @@ -31596,6 +31740,8 @@ snapshots: big-integer@1.6.52: {} + big.js@6.2.2: {} + bignumber.js@9.1.2: {} binary-extensions@2.2.0: {} @@ -33058,14 +33204,14 @@ snapshots: duplexify@3.7.1: dependencies: - end-of-stream: 1.4.4 + end-of-stream: 1.4.5 inherits: 2.0.4 readable-stream: 2.3.8 stream-shift: 1.0.3 duplexify@4.1.3: dependencies: - end-of-stream: 1.4.4 + end-of-stream: 1.4.5 inherits: 2.0.4 readable-stream: 3.6.2 stream-shift: 1.0.3 @@ -34335,6 +34481,15 @@ snapshots: combined-stream: 1.0.8 mime-types: 2.1.35 + form-data@2.5.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + safe-buffer: 5.2.1 + form-data@3.0.4: dependencies: asynckit: 0.4.0 @@ -34863,6 +35018,18 @@ snapshots: dependencies: csstype: 3.2.3 + google-auth-library@9.15.1: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 6.7.1 + gcp-metadata: 6.1.0 + gtoken: 7.1.0 + jws: 4.0.1 + transitivePeerDependencies: + - encoding + - supports-color + gopd@1.2.0: {} graceful-fs@4.2.11: {} @@ -34879,6 +35046,14 @@ snapshots: cron-parser: 4.9.0 ioredis: 5.8.2 + gtoken@7.1.0: + dependencies: + gaxios: 6.7.1 + jws: 4.0.1 + transitivePeerDependencies: + - encoding + - supports-color + gzip-size@6.0.0: dependencies: duplexer: 0.1.2 @@ -35153,6 +35328,8 @@ snapshots: dependencies: whatwg-encoding: 3.1.1 + html-entities@2.6.0: {} + html-escaper@3.0.3: {} html-to-text@9.0.5: @@ -35199,6 +35376,14 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + http-proxy-agent@5.0.0: + dependencies: + '@tootallnate/once': 2.0.1 + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 @@ -35717,6 +35902,8 @@ snapshots: dependencies: system-architecture: 0.1.0 + is@3.3.2: {} + isarray@1.0.0: {} isarray@2.0.5: {} @@ -36040,11 +36227,22 @@ snapshots: ecdsa-sig-formatter: 1.0.11 safe-buffer: 5.2.1 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + jws@3.2.2: dependencies: jwa: 1.4.1 safe-buffer: 5.2.1 + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + kafkajs@2.2.4: {} katex@0.16.21: @@ -39942,6 +40140,15 @@ snapshots: retext-stringify: 4.0.0 unified: 11.0.5 + retry-request@7.0.2: + dependencies: + '@types/request': 2.48.13 + extend: 3.0.2 + teeny-request: 9.0.0 + transitivePeerDependencies: + - encoding + - supports-color + retry@0.13.1: {} reusify@1.0.4: {} @@ -40699,6 +40906,10 @@ snapshots: stream-buffers@2.2.0: {} + stream-events@1.0.5: + dependencies: + stubs: 3.0.0 + stream-shift@1.0.3: {} streamsearch@1.1.0: {} @@ -40837,6 +41048,8 @@ snapshots: structured-headers@0.4.1: {} + stubs@3.0.0: {} + style-mod@4.1.3: {} style-to-js@1.1.21: @@ -41045,6 +41258,17 @@ snapshots: dependencies: bintrees: 1.0.2 + teeny-request@9.0.0: + dependencies: + http-proxy-agent: 5.0.0 + https-proxy-agent: 5.0.1 + node-fetch: 2.7.0 + stream-events: 1.0.5 + uuid: 9.0.1 + transitivePeerDependencies: + - encoding + - supports-color + temp-dir@1.0.0: {} temp-dir@2.0.0: {} From 9d593ae231b6ff51a5c50082ffe58841077526fa Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Mon, 8 Jun 2026 09:15:40 +0000 Subject: [PATCH 2/9] fix: harden BigQuery column mapping schemas after type audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add zBqColRef validator for column references (supports dot-notation for STRUCT nested fields like user.profile.email) - Add mappingType discriminator to both mapping schemas so the union is discriminated and TypeScript can narrow the type cleanly - Add superRefine cross-validation: append mode events syncs must declare an insertTime column (the TIMESTAMP cursor) - Update plan with verified BigQuery Node.js client type mappings: INT64 needs wrapIntegers:true, TIMESTAMP/.value not .toISOString(), DATETIME has no timezone, BYTES→Buffer, Big for NUMERIC/BIGNUMERIC --- packages/validation/src/index.ts | 73 ++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index 93e256a9b..7c1608db9 100755 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -692,35 +692,62 @@ export const zCreateImport = z.object({ export type ICreateImport = z.infer; +// Validates a BigQuery column reference: simple name or dot-notation for STRUCT +// fields (e.g. "user.profile.email"). Allows alphanumeric and underscores per segment. +const zBqColRef = z + .string() + .min(1) + .regex( + /^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*$/, + 'Column reference must be a valid BigQuery field name or dot-notation path (e.g. user.email)', + ); + export const zBigQueryColumnMappingEvents = z.object({ - eventName: z.string().optional(), - profileId: z.string().optional(), - deviceId: z.string().optional(), - timestamp: z.string().optional(), - insertTime: z.string().optional(), + mappingType: z.literal('events'), + eventName: zBqColRef.optional(), + profileId: zBqColRef.optional(), + deviceId: zBqColRef.optional(), + timestamp: zBqColRef.optional(), + // Required for append mode — must point to a TIMESTAMP/DATETIME column + insertTime: zBqColRef.optional(), }); export const zBigQueryColumnMappingProfiles = z.object({ - profileIdColumn: z.string().min(1), - firstName: z.string().optional(), - lastName: z.string().optional(), - email: z.string().optional(), - avatar: z.string().optional(), -}); - -export const zBigQuerySyncConfig = z.object({ - displayName: z.string().min(1).max(100), - dataset: z.string().min(1), - tableName: z.string().min(1), - mappingType: z.enum(['events', 'profiles']), - syncMode: z.enum(['append', 'full']), - schedule: z.enum(['hourly', 'daily', 'weekly']), - columnMapping: z.union([ - zBigQueryColumnMappingEvents, - zBigQueryColumnMappingProfiles, - ]), + mappingType: z.literal('profiles'), + profileIdColumn: zBqColRef, + firstName: zBqColRef.optional(), + lastName: zBqColRef.optional(), + email: zBqColRef.optional(), + avatar: zBqColRef.optional(), }); +export const zBigQuerySyncConfig = z + .object({ + displayName: z.string().min(1).max(100), + dataset: z.string().min(1), + tableName: z.string().min(1), + syncMode: z.enum(['append', 'full']), + schedule: z.enum(['hourly', 'daily', 'weekly']), + columnMapping: z.discriminatedUnion('mappingType', [ + zBigQueryColumnMappingEvents, + zBigQueryColumnMappingProfiles, + ]), + }) + .superRefine((data, ctx) => { + if ( + data.syncMode === 'append' && + data.columnMapping.mappingType === 'events' && + !data.columnMapping.insertTime + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['columnMapping', 'insertTime'], + message: + 'insertTime column is required for append mode — it must point to a TIMESTAMP column used as the incremental cursor', + }); + } + }); + export type IBigQueryColumnMappingEvents = z.infer< typeof zBigQueryColumnMappingEvents >; From 40e68002fbaa03f1fdeba7a59cccc98d85e1440f Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Mon, 8 Jun 2026 09:34:37 +0000 Subject: [PATCH 3/9] fix: allow multiple named BigQuery connections per project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-world orgs connect multiple data sources to one project (e.g. jm-ebg and jm-ebg-cdp on the same ROAS project). The original @unique on projectId wrongly enforced a single connection per project. - Remove @unique from BigQueryConnection.projectId - Add name String field (user label, e.g. "CDP Source") - Replace with @@unique([projectId, name]) — names unique within project - Change Project.bigQueryConnection? → bigQueryConnections[] --- .../migration.sql | 12 ++++++++++++ packages/db/prisma/schema.prisma | 10 ++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 packages/db/prisma/migrations/20260608091000_bigquery_multi_connection/migration.sql diff --git a/packages/db/prisma/migrations/20260608091000_bigquery_multi_connection/migration.sql b/packages/db/prisma/migrations/20260608091000_bigquery_multi_connection/migration.sql new file mode 100644 index 000000000..003bde308 --- /dev/null +++ b/packages/db/prisma/migrations/20260608091000_bigquery_multi_connection/migration.sql @@ -0,0 +1,12 @@ +-- Allow multiple BigQuery connections per project (named connections) +-- Drop single-column unique index on projectId +DROP INDEX IF EXISTS "public"."bigquery_connections_projectId_key"; + +-- Add name column (default '' for any pre-existing rows during dev) +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "name" TEXT NOT NULL DEFAULT ''; + +-- Remove the default now that column exists +ALTER TABLE "public"."bigquery_connections" ALTER COLUMN "name" DROP DEFAULT; + +-- Add composite unique constraint: name must be unique within a project +CREATE UNIQUE INDEX "bigquery_connections_projectId_name_key" ON "public"."bigquery_connections"("projectId", "name"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 238308603..1f35fe155 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -251,9 +251,9 @@ model Project { notificationRules NotificationRule[] notifications Notification[] imports Import[] - gscConnection GscConnection? - bigQueryConnection BigQueryConnection? - bigQuerySyncs BigQuerySync[] + gscConnection GscConnection? + bigQueryConnections BigQueryConnection[] + bigQuerySyncs BigQuerySync[] cohorts Cohort[] // When deleteAt > now(), the project will be deleted @@ -729,7 +729,8 @@ enum BigQuerySyncRunStatus { model BigQueryConnection { id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid - projectId String @unique + projectId String + name String project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) gcpProjectId String serviceAccountEmail String @@ -738,6 +739,7 @@ model BigQueryConnection { updatedAt DateTime @default(now()) @updatedAt syncs BigQuerySync[] + @@unique([projectId, name]) @@map("bigquery_connections") } From 7920219dbbb42d541ed4f4467535f9ad4ba8b7c0 Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Mon, 8 Jun 2026 09:49:09 +0000 Subject: [PATCH 4/9] harden(bigquery): production-grade schema fields and Zod validators Schema additions: - BigQueryConnection: gcpRegion (GDPR region compliance), lastTestedAt/lastTestStatus (connection health) - BigQuerySync: lastSyncStatus typed as enum (was String?), errorRetryCount+isErrorPaused circuit breaker, partitionFilter for cost-safe full-refresh on partitioned tables - BigQuerySyncRun: rowCount BigInt (INT max ~2.1B insufficient), bytesProcessed for cost tracking Zod additions: - zGcpProjectId: GCP project ID format regex (rejects project numbers and display names) - zBqIdentifier: dataset/table name validator (no hyphens, per BQ naming rules) - zServiceAccountJson: SA JSON structure check (rejects authorized_user creds before encryption) - zBigQueryConnectionCreate: connection creation schema with name/region/SA JSON - zBigQuerySyncConfig: dataset/tableName now use zBqIdentifier, partitionFilter field added --- .../migration.sql | 16 +++ packages/db/prisma/schema.prisma | 19 ++- packages/validation/src/index.ts | 121 +++++++++++++++++- 3 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 packages/db/prisma/migrations/20260608120000_bigquery_harden/migration.sql diff --git a/packages/db/prisma/migrations/20260608120000_bigquery_harden/migration.sql b/packages/db/prisma/migrations/20260608120000_bigquery_harden/migration.sql new file mode 100644 index 000000000..ae3eaac88 --- /dev/null +++ b/packages/db/prisma/migrations/20260608120000_bigquery_harden/migration.sql @@ -0,0 +1,16 @@ +-- Phase 1 hardening: production-grade fields for BigQuery connector + +-- BigQueryConnection: region (GDPR compliance) + connection health tracking +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "gcpRegion" TEXT NOT NULL DEFAULT 'US'; +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "lastTestedAt" TIMESTAMP(3); +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "lastTestStatus" BOOLEAN; + +-- BigQuerySync: typed status enum, circuit-breaker fields, partition filter +ALTER TABLE "public"."bigquery_syncs" ALTER COLUMN "lastSyncStatus" TYPE "public"."BigQuerySyncRunStatus" USING "lastSyncStatus"::"public"."BigQuerySyncRunStatus"; +ALTER TABLE "public"."bigquery_syncs" ADD COLUMN "errorRetryCount" INTEGER NOT NULL DEFAULT 0; +ALTER TABLE "public"."bigquery_syncs" ADD COLUMN "isErrorPaused" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "public"."bigquery_syncs" ADD COLUMN "partitionFilter" TEXT; + +-- BigQuerySyncRun: BigInt rowCount (INT max ~2.1B insufficient for large tables), bytes for cost tracking +ALTER TABLE "public"."bigquery_sync_runs" ALTER COLUMN "rowCount" TYPE BIGINT USING "rowCount"::BIGINT; +ALTER TABLE "public"."bigquery_sync_runs" ADD COLUMN "bytesProcessed" BIGINT; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 1f35fe155..9ecccf710 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -733,8 +733,11 @@ model BigQueryConnection { name String project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) gcpProjectId String + gcpRegion String @default("US") serviceAccountEmail String serviceAccountJsonEncrypted String + lastTestedAt DateTime? + lastTestStatus Boolean? createdAt DateTime @default(now()) updatedAt DateTime @default(now()) @updatedAt syncs BigQuerySync[] @@ -759,9 +762,12 @@ model BigQuerySync { columnMapping Json lastCursor String? lastSyncedAt DateTime? - lastSyncStatus String? + lastSyncStatus BigQuerySyncRunStatus? lastSyncError String? isEnabled Boolean @default(true) + errorRetryCount Int @default(0) + isErrorPaused Boolean @default(false) + partitionFilter String? createdAt DateTime @default(now()) updatedAt DateTime @default(now()) @updatedAt runs BigQuerySyncRun[] @@ -774,11 +780,12 @@ model BigQuerySyncRun { syncId String @db.Uuid sync BigQuerySync @relation(fields: [syncId], references: [id], onDelete: Cascade) projectId String - status BigQuerySyncRunStatus @default(pending) - rowCount Int @default(0) - errorMessage String? - startedAt DateTime @default(now()) - completedAt DateTime? + status BigQuerySyncRunStatus @default(pending) + rowCount BigInt @default(0) + bytesProcessed BigInt? + errorMessage String? + startedAt DateTime @default(now()) + completedAt DateTime? @@map("bigquery_sync_runs") } diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index 7c1608db9..7d55c98d7 100755 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -692,8 +692,115 @@ export const zCreateImport = z.object({ export type ICreateImport = z.infer; -// Validates a BigQuery column reference: simple name or dot-notation for STRUCT -// fields (e.g. "user.profile.email"). Allows alphanumeric and underscores per segment. +// BigQuery identifier validators — enforce BQ naming rules at form-save time, not query time + +// GCP project ID: lowercase letters, digits, hyphens; 6-30 chars; must start/end with letter or digit +const zGcpProjectId = z + .string() + .min(6, 'GCP project ID must be at least 6 characters') + .max(30, 'GCP project ID must be at most 30 characters') + .regex( + /^[a-z][a-z0-9\-]{4,28}[a-z0-9]$/, + 'GCP project ID must start with a lowercase letter, contain only lowercase letters, digits, and hyphens, and end with a letter or digit', + ); + +// BigQuery dataset/table names: letters, digits, underscores only (no hyphens without backtick quoting) +const zBqIdentifier = z + .string() + .min(1) + .max(1024) + .regex( + /^[a-zA-Z0-9_]+$/, + 'BigQuery identifiers may only contain letters, digits, and underscores (no hyphens or spaces)', + ); + +// Service account JSON structure check — catches authorized_user creds pasted by mistake +export const zServiceAccountJson = z + .string() + .min(1) + .superRefine((val, ctx) => { + let parsed: unknown; + try { + parsed = JSON.parse(val); + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Must be valid JSON', + }); + return; + } + const sa = parsed as Record; + if (sa.type !== 'service_account') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'Must be a service account key (type: "service_account"). Application Default Credentials are not supported.', + }); + } + if (!sa.private_key || typeof sa.private_key !== 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Service account key is missing the private_key field', + }); + } + if (!sa.client_email || typeof sa.client_email !== 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Service account key is missing the client_email field', + }); + } + if (!sa.project_id || typeof sa.project_id !== 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Service account key is missing the project_id field', + }); + } + }); + +// Connection creation schema — used in Phase 2 tRPC router +export const zBigQueryConnectionCreate = z.object({ + name: z + .string() + .min(1) + .max(50) + .regex( + /^[a-zA-Z0-9 _\-]+$/, + 'Connection name may only contain letters, digits, spaces, hyphens, and underscores', + ), + serviceAccountJson: zServiceAccountJson, + gcpRegion: z + .enum([ + 'US', + 'EU', + 'us-central1', + 'us-east1', + 'us-east4', + 'us-west1', + 'us-west2', + 'us-west3', + 'us-west4', + 'europe-west1', + 'europe-west2', + 'europe-west3', + 'europe-west4', + 'europe-west6', + 'europe-north1', + 'asia-east1', + 'asia-east2', + 'asia-northeast1', + 'asia-northeast2', + 'asia-northeast3', + 'asia-south1', + 'asia-southeast1', + 'asia-southeast2', + 'australia-southeast1', + 'northamerica-northeast1', + 'southamerica-east1', + ]) + .default('US'), +}); + +// Column reference: simple field name or dot-notation path for STRUCT fields (e.g. user.profile.email) const zBqColRef = z .string() .min(1) @@ -708,7 +815,6 @@ export const zBigQueryColumnMappingEvents = z.object({ profileId: zBqColRef.optional(), deviceId: zBqColRef.optional(), timestamp: zBqColRef.optional(), - // Required for append mode — must point to a TIMESTAMP/DATETIME column insertTime: zBqColRef.optional(), }); @@ -724,10 +830,11 @@ export const zBigQueryColumnMappingProfiles = z.object({ export const zBigQuerySyncConfig = z .object({ displayName: z.string().min(1).max(100), - dataset: z.string().min(1), - tableName: z.string().min(1), + dataset: zBqIdentifier, + tableName: zBqIdentifier, syncMode: z.enum(['append', 'full']), schedule: z.enum(['hourly', 'daily', 'weekly']), + partitionFilter: z.string().max(500).optional(), columnMapping: z.discriminatedUnion('mappingType', [ zBigQueryColumnMappingEvents, zBigQueryColumnMappingProfiles, @@ -758,6 +865,10 @@ export type IBigQueryColumnMapping = | IBigQueryColumnMappingEvents | IBigQueryColumnMappingProfiles; export type IBigQuerySyncConfig = z.infer; +export type IBigQueryConnectionCreate = z.infer< + typeof zBigQueryConnectionCreate +>; +export { zGcpProjectId, zBqIdentifier }; export * from './types.insights'; export * from './types.validation'; From 632bde3c719054f5e7b2a29d4176d259677826a2 Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Mon, 8 Jun 2026 10:39:42 +0000 Subject: [PATCH 5/9] =?UTF-8?q?fix(bigquery):=20address=20CodeRabbit=20fin?= =?UTF-8?q?dings=20=E2=80=94=20referential=20integrity=20and=20dependency?= =?UTF-8?q?=20upgrade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Upgrade @google-cloud/bigquery from ^7.9.1 to ^8.3.1 (latest stable) - Add FK bigquery_sync_runs.projectId -> projects(id) (was missing, allowing orphan runs) - Add composite FK bigquery_syncs(projectId, connectionId) -> bigquery_connections(projectId, id) to prevent cross-tenant data: a sync can no longer reference a connection from a different project - Add UNIQUE INDEX bigquery_connections(projectId, id) to back the composite FK - Add CHECK(char_length(name) > 0) on bigquery_connections to enforce non-empty names at DB level - Backfill any dev rows with empty name using concat('connection_', id) --- packages/db/package.json | 2 +- .../migration.sql | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 packages/db/prisma/migrations/20260608140000_bigquery_referential_integrity/migration.sql diff --git a/packages/db/package.json b/packages/db/package.json index 58b4ff42f..85e5be97e 100755 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -14,7 +14,7 @@ }, "dependencies": { "@clickhouse/client": "^1.18.5", - "@google-cloud/bigquery": "^7.9.1", + "@google-cloud/bigquery": "^8.3.1", "@openpanel/common": "workspace:*", "@openpanel/constants": "workspace:*", "@openpanel/json": "workspace:*", diff --git a/packages/db/prisma/migrations/20260608140000_bigquery_referential_integrity/migration.sql b/packages/db/prisma/migrations/20260608140000_bigquery_referential_integrity/migration.sql new file mode 100644 index 000000000..3a661fcf9 --- /dev/null +++ b/packages/db/prisma/migrations/20260608140000_bigquery_referential_integrity/migration.sql @@ -0,0 +1,29 @@ +-- Fix 1: Add FK on bigquery_sync_runs.projectId (was missing, allowing orphan runs) +ALTER TABLE "public"."bigquery_sync_runs" + ADD CONSTRAINT "bigquery_sync_runs_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Fix 2: Cross-tenant vulnerability — enforce connection belongs to same project as sync +-- Requires a unique index on (projectId, id) in bigquery_connections to back the composite FK +CREATE UNIQUE INDEX "bigquery_connections_projectId_id_key" + ON "public"."bigquery_connections"("projectId", "id"); + +-- Add composite FK on bigquery_syncs: (projectId, connectionId) must match a connection +-- that belongs to the same project. This is an additional constraint on top of Prisma's +-- single-column connectionId FK — both coexist for defence in depth. +ALTER TABLE "public"."bigquery_syncs" + ADD CONSTRAINT "bigquery_syncs_projectId_connectionId_fkey" + FOREIGN KEY ("projectId", "connectionId") + REFERENCES "public"."bigquery_connections"("projectId", "id") + ON DELETE CASCADE ON UPDATE CASCADE; + +-- Fix 3: Prevent empty-string connection names at the DB level +-- (Zod already blocks this at the application layer; this is belt-and-suspenders) +-- Backfill any dev rows that got the empty-string default from the prior migration +UPDATE "public"."bigquery_connections" + SET "name" = concat('connection_', "id") + WHERE char_length("name") = 0; + +ALTER TABLE "public"."bigquery_connections" + ADD CONSTRAINT "bigquery_connections_name_nonempty_check" + CHECK (char_length("name") > 0); From ca03135f41e35483bdf51c090eb538eb50429495 Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Wed, 10 Jun 2026 19:43:38 +0000 Subject: [PATCH 6/9] =?UTF-8?q?feat(warehouse):=20Phase=201=20=E2=80=94=20?= =?UTF-8?q?generic=20warehouse=20schema,=20migrations,=20and=20Zod=20valid?= =?UTF-8?q?ators?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the foundational schema for a multi-provider Data Warehouse Connector (BigQuery first, extensible to Snowflake/Redshift/Databricks/Postgres). Three shared Prisma tables: - warehouse_connections: one row per named connection, any provider - warehouse_syncs: one sync job per connection - warehouse_sync_runs: one run record per execution Key design decisions: - configEncrypted (AES-256-GCM) + displayIdentifier/displayEmail for UI display without decryption - Composite FK warehouse_syncs(projectId,connectionId) → warehouse_connections(projectId,id) blocks cross-tenant exploit at DB level - Three sync modes: append (cursor), full (reload + stale cleanup), onetime (backfill) - cursorOverlapMinutes (default 10) rewinds append cursor to catch late-arriving rows - syncDelayMinutes (default 0) delays cron execution to allow BQ pipelines to land - Performance indexes on all FK columns (PostgreSQL does not create these automatically) - jsonProperties column mapping flattens a JSON column into event/profile properties Validation (packages/validation/src/index.ts): - zBigQueryWarehouseConfig + zWarehouseConfig discriminated union - zBigQuerySyncConfig with superRefine cross-field rules: schedule required for append/full, insertTime required for append, eventName/eventNameStatic mutually exclusive Migrations applied: 20260610115042–20260610115046 21/21 Zod validator probes pass. Zero schema drift (prisma migrate diff). --- .../migration.sql | 124 +++++++++ .../migration.sql | 6 + .../migration.sql | 24 ++ .../migration.sql | 8 + .../migration.sql | 12 + packages/db/prisma/schema.prisma | 135 +++++---- packages/db/src/types.ts | 4 +- packages/validation/src/index.ts | 147 ++++++---- pnpm-lock.yaml | 257 +++++++++--------- 9 files changed, 480 insertions(+), 237 deletions(-) create mode 100644 packages/db/prisma/migrations/20260610115042_warehouse_restructure/migration.sql create mode 100644 packages/db/prisma/migrations/20260610115043_warehouse_security_fks/migration.sql create mode 100644 packages/db/prisma/migrations/20260610115044_warehouse_phase1_finalize/migration.sql create mode 100644 packages/db/prisma/migrations/20260610115045_warehouse_onetime_mode/migration.sql create mode 100644 packages/db/prisma/migrations/20260610115046_warehouse_sync_overlap_delay/migration.sql diff --git a/packages/db/prisma/migrations/20260610115042_warehouse_restructure/migration.sql b/packages/db/prisma/migrations/20260610115042_warehouse_restructure/migration.sql new file mode 100644 index 000000000..23a9e367b --- /dev/null +++ b/packages/db/prisma/migrations/20260610115042_warehouse_restructure/migration.sql @@ -0,0 +1,124 @@ +-- Restructure: BigQuery-specific tables → generic Warehouse tables +-- Phase 1 tables are dev-only (no production data) — drop and recreate cleanly. +-- This makes the schema ready for BigQuery, Snowflake, Redshift, Databricks, Postgres. + +-- Drop old tables (CASCADE removes all FKs and indexes automatically) +DROP TABLE IF EXISTS "public"."bigquery_sync_runs"; +DROP TABLE IF EXISTS "public"."bigquery_syncs"; +DROP TABLE IF EXISTS "public"."bigquery_connections"; + +-- Drop old BigQuery-specific enums +DROP TYPE IF EXISTS "public"."BigQuerySyncMappingType"; +DROP TYPE IF EXISTS "public"."BigQuerySyncMode"; +DROP TYPE IF EXISTS "public"."BigQuerySyncSchedule"; +DROP TYPE IF EXISTS "public"."BigQuerySyncRunStatus"; + +-- WarehouseType: all supported warehouse providers +CREATE TYPE "public"."WarehouseType" AS ENUM ('bigquery', 'snowflake', 'redshift', 'databricks', 'postgres'); + +-- Shared sync enums (provider-agnostic) +CREATE TYPE "public"."WarehouseSyncMappingType" AS ENUM ('events', 'profiles'); +CREATE TYPE "public"."WarehouseSyncMode" AS ENUM ('append', 'full'); +CREATE TYPE "public"."WarehouseSyncSchedule" AS ENUM ('hourly', 'daily', 'weekly'); +CREATE TYPE "public"."WarehouseSyncRunStatus" AS ENUM ('pending', 'running', 'completed', 'failed'); + +-- warehouse_connections: one row per named connection, any provider +-- configEncrypted: AES-256-GCM encrypted JSON (shape validated by zWarehouseConfig discriminated union) +-- displayIdentifier: plain-text for UI display without decryption (GCP project ID / Snowflake account / etc.) +-- displayEmail: plain-text for UI display (SA email / username) +CREATE TABLE "public"."warehouse_connections" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "projectId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "type" "public"."WarehouseType" NOT NULL, + "configEncrypted" TEXT NOT NULL, + "displayIdentifier" TEXT, + "displayEmail" TEXT, + "lastTestedAt" TIMESTAMP(3), + "lastTestStatus" BOOLEAN, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "warehouse_connections_pkey" PRIMARY KEY ("id") +); + +CREATE TABLE "public"."warehouse_syncs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "connectionId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "displayName" TEXT NOT NULL, + "dataset" TEXT NOT NULL, + "tableName" TEXT NOT NULL, + "mappingType" "public"."WarehouseSyncMappingType" NOT NULL, + "syncMode" "public"."WarehouseSyncMode" NOT NULL, + "schedule" "public"."WarehouseSyncSchedule" NOT NULL, + "columnMapping" JSONB NOT NULL, + "lastCursor" TEXT, + "lastSyncedAt" TIMESTAMP(3), + "lastSyncStatus" "public"."WarehouseSyncRunStatus", + "lastSyncError" TEXT, + "isEnabled" BOOLEAN NOT NULL DEFAULT true, + "errorRetryCount" INTEGER NOT NULL DEFAULT 0, + "isErrorPaused" BOOLEAN NOT NULL DEFAULT false, + "partitionFilter" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "warehouse_syncs_pkey" PRIMARY KEY ("id") +); + +CREATE TABLE "public"."warehouse_sync_runs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "syncId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "status" "public"."WarehouseSyncRunStatus" NOT NULL DEFAULT 'pending', + "rowCount" BIGINT NOT NULL DEFAULT 0, + "bytesProcessed" BIGINT, + "errorMessage" TEXT, + "startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TIMESTAMP(3), + CONSTRAINT "warehouse_sync_runs_pkey" PRIMARY KEY ("id") +); + +-- Unique indexes +-- (projectId, name): connection name must be unique per project, across ALL warehouse types +CREATE UNIQUE INDEX "warehouse_connections_projectId_name_key" + ON "public"."warehouse_connections"("projectId", "name"); + +-- (projectId, id): backs the composite FK on warehouse_syncs for cross-tenant protection +CREATE UNIQUE INDEX "warehouse_connections_projectId_id_key" + ON "public"."warehouse_connections"("projectId", "id"); + +-- DB-level name nonempty (belt-and-suspenders; Zod blocks empty strings at app layer) +ALTER TABLE "public"."warehouse_connections" + ADD CONSTRAINT "warehouse_connections_name_nonempty_check" + CHECK (char_length("name") > 0); + +-- FKs on warehouse_connections +ALTER TABLE "public"."warehouse_connections" + ADD CONSTRAINT "warehouse_connections_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- FKs on warehouse_syncs +ALTER TABLE "public"."warehouse_syncs" + ADD CONSTRAINT "warehouse_syncs_connectionId_fkey" + FOREIGN KEY ("connectionId") REFERENCES "public"."warehouse_connections"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +ALTER TABLE "public"."warehouse_syncs" + ADD CONSTRAINT "warehouse_syncs_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Composite FK: prevents cross-tenant exploit where a sync in project A +-- could reference a connection belonging to project B +ALTER TABLE "public"."warehouse_syncs" + ADD CONSTRAINT "warehouse_syncs_projectId_connectionId_fkey" + FOREIGN KEY ("projectId", "connectionId") + REFERENCES "public"."warehouse_connections"("projectId", "id") + ON DELETE CASCADE ON UPDATE CASCADE; + +-- FKs on warehouse_sync_runs +ALTER TABLE "public"."warehouse_sync_runs" + ADD CONSTRAINT "warehouse_sync_runs_syncId_fkey" + FOREIGN KEY ("syncId") REFERENCES "public"."warehouse_syncs"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +ALTER TABLE "public"."warehouse_sync_runs" + ADD CONSTRAINT "warehouse_sync_runs_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20260610115043_warehouse_security_fks/migration.sql b/packages/db/prisma/migrations/20260610115043_warehouse_security_fks/migration.sql new file mode 100644 index 000000000..1a751c228 --- /dev/null +++ b/packages/db/prisma/migrations/20260610115043_warehouse_security_fks/migration.sql @@ -0,0 +1,6 @@ +-- Drop the single-field connectionId FK on warehouse_syncs. +-- The composite FK (projectId, connectionId) → warehouse_connections(projectId, id) +-- from 20260610115042_warehouse_restructure already enforces the same referential +-- integrity PLUS prevents cross-tenant access — keeping both would be redundant. +ALTER TABLE "public"."warehouse_syncs" + DROP CONSTRAINT "warehouse_syncs_connectionId_fkey"; diff --git a/packages/db/prisma/migrations/20260610115044_warehouse_phase1_finalize/migration.sql b/packages/db/prisma/migrations/20260610115044_warehouse_phase1_finalize/migration.sql new file mode 100644 index 000000000..566a54f6d --- /dev/null +++ b/packages/db/prisma/migrations/20260610115044_warehouse_phase1_finalize/migration.sql @@ -0,0 +1,24 @@ +-- Phase 1 finalizing migration: performance indexes + column additions +-- These were identified as missing after the initial restructure. + +-- Performance indexes on FK columns (PostgreSQL does NOT auto-create these). +-- Required for list queries: syncs-by-project, syncs-by-connection, runs-by-sync. +CREATE INDEX "warehouse_syncs_projectId_idx" + ON "public"."warehouse_syncs"("projectId"); + +CREATE INDEX "warehouse_syncs_connectionId_idx" + ON "public"."warehouse_syncs"("connectionId"); + +CREATE INDEX "warehouse_sync_runs_syncId_idx" + ON "public"."warehouse_sync_runs"("syncId"); + +-- failureCount: tracks rows that were fetched but failed to import individually +-- (bad data, type mismatch, etc.) — separate from rowCount (successfully written rows). +-- Matches Mixpanel's Sync History "Failures" column in the UI. +ALTER TABLE "public"."warehouse_sync_runs" + ADD COLUMN "failureCount" BIGINT NOT NULL DEFAULT 0; + +-- createdBy: userId of the person who created this sync. +-- Used for displaying creator avatar + name in the sync list (matching Mixpanel UI). +ALTER TABLE "public"."warehouse_syncs" + ADD COLUMN "createdBy" TEXT; diff --git a/packages/db/prisma/migrations/20260610115045_warehouse_onetime_mode/migration.sql b/packages/db/prisma/migrations/20260610115045_warehouse_onetime_mode/migration.sql new file mode 100644 index 000000000..cf63e5bc2 --- /dev/null +++ b/packages/db/prisma/migrations/20260610115045_warehouse_onetime_mode/migration.sql @@ -0,0 +1,8 @@ +-- Add 'onetime' sync mode: run once immediately, then disable automatically. +-- Used for historical backfills — import all historical data once, then set up +-- an Append sync for ongoing data. +ALTER TYPE "public"."WarehouseSyncMode" ADD VALUE 'onetime'; + +-- Make schedule nullable: onetime syncs have no recurring schedule. +ALTER TABLE "public"."warehouse_syncs" + ALTER COLUMN "schedule" DROP NOT NULL; diff --git a/packages/db/prisma/migrations/20260610115046_warehouse_sync_overlap_delay/migration.sql b/packages/db/prisma/migrations/20260610115046_warehouse_sync_overlap_delay/migration.sql new file mode 100644 index 000000000..96ae7e6ef --- /dev/null +++ b/packages/db/prisma/migrations/20260610115046_warehouse_sync_overlap_delay/migration.sql @@ -0,0 +1,12 @@ +-- cursorOverlapMinutes: overlap window for append-mode cursor. +-- Rewinds the cursor by this many minutes to catch late-arriving rows that share +-- the same insertTime as the previous high-water mark. Worker handles dedup via +-- eventId (sha256 hash or user-mapped column) so no duplicate events are created. +ALTER TABLE "public"."warehouse_syncs" + ADD COLUMN "cursorOverlapMinutes" INTEGER NOT NULL DEFAULT 10; + +-- syncDelayMinutes: delay after scheduled fire time before running. +-- Allows pipeline data to fully land in BigQuery before the sync executes. +-- e.g. a daily sync firing at midnight waits N minutes for the ETL job to finish. +ALTER TABLE "public"."warehouse_syncs" + ADD COLUMN "syncDelayMinutes" INTEGER NOT NULL DEFAULT 0; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 9ecccf710..3f3a61fc2 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -252,8 +252,9 @@ model Project { notifications Notification[] imports Import[] gscConnection GscConnection? - bigQueryConnections BigQueryConnection[] - bigQuerySyncs BigQuerySync[] + warehouseConnections WarehouseConnection[] + warehouseSyncs WarehouseSync[] + warehouseSyncRuns WarehouseSyncRun[] cohorts Cohort[] // When deleteAt > now(), the project will be deleted @@ -704,90 +705,108 @@ model GscConnection { @@map("gsc_connections") } -enum BigQuerySyncMappingType { +enum WarehouseType { + bigquery + snowflake + redshift + databricks + postgres +} + +enum WarehouseSyncMappingType { events profiles } -enum BigQuerySyncMode { +enum WarehouseSyncMode { append full + onetime } -enum BigQuerySyncSchedule { +enum WarehouseSyncSchedule { hourly daily weekly } -enum BigQuerySyncRunStatus { +enum WarehouseSyncRunStatus { pending running completed failed } -model BigQueryConnection { - id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid - projectId String - name String - project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) - gcpProjectId String - gcpRegion String @default("US") - serviceAccountEmail String - serviceAccountJsonEncrypted String - lastTestedAt DateTime? - lastTestStatus Boolean? - createdAt DateTime @default(now()) - updatedAt DateTime @default(now()) @updatedAt - syncs BigQuerySync[] +model WarehouseConnection { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + projectId String + name String + type WarehouseType + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + configEncrypted String + displayIdentifier String? + displayEmail String? + lastTestedAt DateTime? + lastTestStatus Boolean? + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + syncs WarehouseSync[] @@unique([projectId, name]) - @@map("bigquery_connections") + @@unique([projectId, id]) + @@map("warehouse_connections") } -model BigQuerySync { - id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid - connectionId String @db.Uuid - connection BigQueryConnection @relation(fields: [connectionId], references: [id], onDelete: Cascade) +model WarehouseSync { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + connectionId String @db.Uuid + projectId String + connection WarehouseConnection @relation(fields: [projectId, connectionId], references: [projectId, id], onDelete: Cascade) + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + displayName String + dataset String + tableName String + mappingType WarehouseSyncMappingType + syncMode WarehouseSyncMode + schedule WarehouseSyncSchedule? + /// [IWarehouseColumnMapping] + columnMapping Json + lastCursor String? + lastSyncedAt DateTime? + lastSyncStatus WarehouseSyncRunStatus? + lastSyncError String? + isEnabled Boolean @default(true) + errorRetryCount Int @default(0) + isErrorPaused Boolean @default(false) + partitionFilter String? + cursorOverlapMinutes Int @default(10) + syncDelayMinutes Int @default(0) + createdBy String? + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + runs WarehouseSyncRun[] + + @@index([projectId]) + @@index([connectionId]) + @@map("warehouse_syncs") +} + +model WarehouseSyncRun { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + syncId String @db.Uuid + sync WarehouseSync @relation(fields: [syncId], references: [id], onDelete: Cascade) projectId String - project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) - displayName String - dataset String - tableName String - mappingType BigQuerySyncMappingType - syncMode BigQuerySyncMode - schedule BigQuerySyncSchedule - /// [IBigQueryColumnMapping] - columnMapping Json - lastCursor String? - lastSyncedAt DateTime? - lastSyncStatus BigQuerySyncRunStatus? - lastSyncError String? - isEnabled Boolean @default(true) - errorRetryCount Int @default(0) - isErrorPaused Boolean @default(false) - partitionFilter String? - createdAt DateTime @default(now()) - updatedAt DateTime @default(now()) @updatedAt - runs BigQuerySyncRun[] - - @@map("bigquery_syncs") -} - -model BigQuerySyncRun { - id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid - syncId String @db.Uuid - sync BigQuerySync @relation(fields: [syncId], references: [id], onDelete: Cascade) - projectId String - status BigQuerySyncRunStatus @default(pending) - rowCount BigInt @default(0) + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + status WarehouseSyncRunStatus @default(pending) + rowCount BigInt @default(0) + failureCount BigInt @default(0) bytesProcessed BigInt? errorMessage String? - startedAt DateTime @default(now()) + startedAt DateTime @default(now()) completedAt DateTime? - @@map("bigquery_sync_runs") + @@index([syncId]) + @@map("warehouse_sync_runs") } model EmailUnsubscribe { diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 8e05d74e2..42b3a5211 100755 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -1,6 +1,6 @@ import type { CohortDefinition, - IBigQueryColumnMapping as IBigQueryColumnMappingType, + IWarehouseColumnMapping as IWarehouseColumnMappingType, IImportConfig, IIntegrationConfig, INotificationRuleConfig, @@ -28,7 +28,7 @@ declare global { type IPrismaClickhouseProfile = IClickhouseProfile; type IPrismaClickhouseBotEvent = IClickhouseBotEvent; type IPrismaCohortDefinition = CohortDefinition; - type IBigQueryColumnMapping = IBigQueryColumnMappingType; + type IWarehouseColumnMapping = IWarehouseColumnMappingType; // Each ChatMessage row stores one Better Agent `ConversationItem` // (message, tool call, or tool result) as JSON. Typed as `unknown[]` // here to avoid pulling `@better-agent/core` into @openpanel/db's diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index 7d55c98d7..d9e1d1c51 100755 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -718,6 +718,7 @@ const zBqIdentifier = z export const zServiceAccountJson = z .string() .min(1) + .max(16384, 'Service account JSON must be under 16 KB') .superRefine((val, ctx) => { let parsed: unknown; try { @@ -757,47 +758,41 @@ export const zServiceAccountJson = z } }); -// Connection creation schema — used in Phase 2 tRPC router -export const zBigQueryConnectionCreate = z.object({ - name: z - .string() - .min(1) - .max(50) - .regex( - /^[a-zA-Z0-9 _\-]+$/, - 'Connection name may only contain letters, digits, spaces, hyphens, and underscores', - ), +// GCP region: regex-based so new Google Cloud regions work without code changes +const zWarehouseRegion = z + .string() + .min(1) + .max(63) + .regex( + /^([A-Z]+|[a-z]+-[a-z0-9]+-[0-9]+|[a-z]+-[a-z0-9]+)$/, + 'Must be a valid cloud region (e.g. US, EU, us-central1, asia-south1)', + ); + +// BigQuery-specific connection config +export const zBigQueryWarehouseConfig = z.object({ + type: z.literal('bigquery'), + gcpProjectId: zGcpProjectId, + gcpRegion: zWarehouseRegion.optional(), serviceAccountJson: zServiceAccountJson, - gcpRegion: z - .enum([ - 'US', - 'EU', - 'us-central1', - 'us-east1', - 'us-east4', - 'us-west1', - 'us-west2', - 'us-west3', - 'us-west4', - 'europe-west1', - 'europe-west2', - 'europe-west3', - 'europe-west4', - 'europe-west6', - 'europe-north1', - 'asia-east1', - 'asia-east2', - 'asia-northeast1', - 'asia-northeast2', - 'asia-northeast3', - 'asia-south1', - 'asia-southeast1', - 'asia-southeast2', - 'australia-southeast1', - 'northamerica-northeast1', - 'southamerica-east1', - ]) - .default('US'), +}); + +// Discriminated union — add zSnowflakeWarehouseConfig, zRedshiftWarehouseConfig etc. here as they land +export const zWarehouseConfig = z.discriminatedUnion('type', [ + zBigQueryWarehouseConfig, +]); + +const zWarehouseConnectionName = z + .string() + .min(1) + .max(50) + .regex( + /^[a-zA-Z0-9 _\-]+$/, + 'Connection name may only contain letters, digits, spaces, hyphens, and underscores', + ); + +export const zWarehouseConnectionCreate = z.object({ + name: zWarehouseConnectionName, + config: zWarehouseConfig, }); // Column reference: simple field name or dot-notation path for STRUCT fields (e.g. user.profile.email) @@ -811,11 +806,34 @@ const zBqColRef = z export const zBigQueryColumnMappingEvents = z.object({ mappingType: z.literal('events'), - eventName: zBqColRef.optional(), + // Event name — two modes (only one should be set): + // eventName: column reference → value of that column becomes the event name per row + // eventNameStatic: fixed string → every row gets this event name (e.g. "Subscription") + // If neither is set, the table name is used as the event name fallback. + eventNameStatic: z.string().min(1).max(100).optional(), + // The authenticated user identifier — maps to profile_id in OpenPanel. + // For warehouse data this is always a known user ID (ClientCode, user_id, etc.). + // When absent, events are ingested without a profile association. profileId: zBqColRef.optional(), + // Anonymous device identifier — maps to device_id in OpenPanel. + // Rare in warehouse tables; exists when the source table tracks both device + // and user separately (e.g. pre-login funnel data). + // If omitted, device_id is set equal to profileId at ingest time. deviceId: zBqColRef.optional(), + // Custom event ID for deduplication — maps to the ClickHouse event id. + // When set, this column's value becomes the event's unique id (prevents + // duplicate imports on re-runs). When omitted, id is computed as + // sha256(syncId + ':' + rowHash). + eventId: zBqColRef.optional(), + // Revenue — first-class numeric field in OpenPanel events. + // Map to any numeric column representing transaction or subscription value. + revenue: zBqColRef.optional(), + eventName: zBqColRef.optional(), timestamp: zBqColRef.optional(), insertTime: zBqColRef.optional(), + // JSON column whose key-value content is flattened into event properties. + // e.g. a `metadata JSON` column → its keys become individual event properties. + jsonProperties: zBqColRef.optional(), }); export const zBigQueryColumnMappingProfiles = z.object({ @@ -825,6 +843,11 @@ export const zBigQueryColumnMappingProfiles = z.object({ lastName: zBqColRef.optional(), email: zBqColRef.optional(), avatar: zBqColRef.optional(), + // When the source table has a profile creation timestamp, map it here. + // Used to preserve original signup/creation date rather than defaulting to sync time. + createdAt: zBqColRef.optional(), + // JSON column whose key-value content is merged into profile properties. + jsonProperties: zBqColRef.optional(), }); export const zBigQuerySyncConfig = z @@ -832,8 +855,9 @@ export const zBigQuerySyncConfig = z displayName: z.string().min(1).max(100), dataset: zBqIdentifier, tableName: zBqIdentifier, - syncMode: z.enum(['append', 'full']), - schedule: z.enum(['hourly', 'daily', 'weekly']), + syncMode: z.enum(['append', 'full', 'onetime']), + // Required for append and full (recurring) modes. Not set for onetime syncs. + schedule: z.enum(['hourly', 'daily', 'weekly']).optional(), partitionFilter: z.string().max(500).optional(), columnMapping: z.discriminatedUnion('mappingType', [ zBigQueryColumnMappingEvents, @@ -841,6 +865,15 @@ export const zBigQuerySyncConfig = z ]), }) .superRefine((data, ctx) => { + // Recurring modes require a schedule + if (data.syncMode !== 'onetime' && !data.schedule) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['schedule'], + message: 'schedule is required for append and full sync modes', + }); + } + // Append mode requires an insertTime cursor column if ( data.syncMode === 'append' && data.columnMapping.mappingType === 'events' && @@ -853,21 +886,35 @@ export const zBigQuerySyncConfig = z 'insertTime column is required for append mode — it must point to a TIMESTAMP column used as the incremental cursor', }); } + // eventName and eventNameStatic are mutually exclusive + if ( + data.columnMapping.mappingType === 'events' && + data.columnMapping.eventName && + data.columnMapping.eventNameStatic + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['columnMapping', 'eventNameStatic'], + message: + 'set either eventName (column reference) or eventNameStatic (fixed value), not both', + }); + } }); -export type IBigQueryColumnMappingEvents = z.infer< +export type IWarehouseColumnMappingEvents = z.infer< typeof zBigQueryColumnMappingEvents >; -export type IBigQueryColumnMappingProfiles = z.infer< +export type IWarehouseColumnMappingProfiles = z.infer< typeof zBigQueryColumnMappingProfiles >; -export type IBigQueryColumnMapping = - | IBigQueryColumnMappingEvents - | IBigQueryColumnMappingProfiles; +export type IWarehouseColumnMapping = + | IWarehouseColumnMappingEvents + | IWarehouseColumnMappingProfiles; export type IBigQuerySyncConfig = z.infer; -export type IBigQueryConnectionCreate = z.infer< - typeof zBigQueryConnectionCreate +export type IWarehouseConnectionCreate = z.infer< + typeof zWarehouseConnectionCreate >; +export type IWarehouseConfig = z.infer; export { zGcpProjectId, zBqIdentifier }; export * from './types.insights'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f5abed03d..9681b5a12 100755 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1215,8 +1215,8 @@ importers: specifier: ^1.18.5 version: 1.18.5 '@google-cloud/bigquery': - specifier: ^7.9.1 - version: 7.9.4 + specifier: ^8.3.1 + version: 8.3.1 '@openpanel/common': specifier: workspace:* version: link:../common @@ -4830,21 +4830,21 @@ packages: '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} - '@google-cloud/bigquery@7.9.4': - resolution: {integrity: sha512-C7jeI+9lnCDYK3cRDujcBsPgiwshWKn/f0BiaJmClplfyosCLfWE83iGQ0eKH113UZzjR9c9q7aZQg0nU388sw==} - engines: {node: '>=14.0.0'} + '@google-cloud/bigquery@8.3.1': + resolution: {integrity: sha512-F4g9oMgI3EB5Uo+6npRHDSWn1HkVko8GebG1tJtzasn/gknAEFeobQzuLeGYC6SJ92RVdaBpGVisw86P70RrHQ==} + engines: {node: '>=18'} - '@google-cloud/common@5.0.2': - resolution: {integrity: sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==} - engines: {node: '>=14.0.0'} + '@google-cloud/common@6.0.1': + resolution: {integrity: sha512-1uvKzbmAWUdchIYRsg0f4rUmezOamWuVBSWAPAhnYoUE5OiPEx6v6JOxFIdr3MsrqV+6fyLV3EI1vMPlHoeRvw==} + engines: {node: '>=18'} - '@google-cloud/paginator@5.0.2': - resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==} - engines: {node: '>=14.0.0'} + '@google-cloud/paginator@6.0.1': + resolution: {integrity: sha512-HtzIe4n9b7It3MjimmFeXwQCuUsPI621e99zBTFqZjbPH7pZpRKDRptlYM0i0+nyot2XXhw0wPfVlTwwR/TyKA==} + engines: {node: '>=18'} - '@google-cloud/precise-date@4.0.0': - resolution: {integrity: sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==} - engines: {node: '>=14.0.0'} + '@google-cloud/precise-date@5.0.1': + resolution: {integrity: sha512-9HlRbOcDb8b2tSsOvljPD/Rm+Jn9KxMVB6sLf85CBnoIYdCFTNO1FIizQ13P75itXpSXsLuMlg1XK5opHKVzjg==} + engines: {node: '>=18'} '@google-cloud/projectify@4.0.0': resolution: {integrity: sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==} @@ -4854,6 +4854,10 @@ packages: resolution: {integrity: sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==} engines: {node: '>=14'} + '@google-cloud/promisify@5.0.1': + resolution: {integrity: sha512-Ste6NGraHq30ge3Sdq7m+pZE6lRUlkt2YgsEdq/vcoEgdbdZ/7h37Z1dMvivjhlbgrcmYRvlLF9FgI7tXm5wjg==} + engines: {node: '>=18'} + '@graphql-typed-document-node/core@3.2.0': resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==} peerDependencies: @@ -9397,10 +9401,6 @@ packages: '@types/react-dom': optional: true - '@tootallnate/once@2.0.1': - resolution: {integrity: sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ==} - engines: {node: '>= 10'} - '@trpc-limiter/core@1.0.0': resolution: {integrity: sha512-Wjq2oTCmCdwNbZKRfKpzpl1Um9QXGE8OHXOab+EUSj5Wk+26I/V6Vs2p0VBFAz6eEsBn36982cCC/vaaznA8+Q==} engines: {node: '>=16'} @@ -9497,9 +9497,6 @@ packages: '@types/bunyan@1.8.11': resolution: {integrity: sha512-758fRH7umIMk5qt5ELmRMff4mLDlN+xyYzC+dkPTdKwbSkJFvz6xwyScrytPU0QIBbRRwbiE8/BIg8bpajerNQ==} - '@types/caseless@0.12.5': - resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==} - '@types/compression@1.7.5': resolution: {integrity: sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==} @@ -9878,9 +9875,6 @@ packages: '@types/request-ip@0.0.41': resolution: {integrity: sha512-Qzz0PM2nSZej4lsLzzNfADIORZhhxO7PED0fXpg4FjXiHuJ/lMyUg+YFF5q8x9HPZH3Gl6N+NOM8QZjItNgGKg==} - '@types/request@2.48.13': - resolution: {integrity: sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==} - '@types/resolve@1.20.2': resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} @@ -9926,9 +9920,6 @@ packages: '@types/through@0.0.33': resolution: {integrity: sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==} - '@types/tough-cookie@4.0.5': - resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} - '@types/triple-beam@1.3.5': resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} @@ -10473,6 +10464,10 @@ packages: resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} engines: {node: '>=8'} + arrify@3.0.0: + resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} + engines: {node: '>=12'} + asap@2.0.6: resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} @@ -10675,8 +10670,8 @@ packages: resolution: {integrity: sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==} engines: {node: '>=0.6'} - big.js@6.2.2: - resolution: {integrity: sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==} + big.js@7.0.1: + resolution: {integrity: sha512-iFgV784tD8kq4ccF1xtNMZnXeZzVuXWWM+ERFzKQjv+A5G9HC8CY3DuV45vgzFFcW+u2tIvmF95+AzWgs6BjCg==} bignumber.js@9.1.2: resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} @@ -11649,6 +11644,10 @@ packages: dag-map@1.0.2: resolution: {integrity: sha512-+LSAiGFwQ9dRnRdOeaj7g47ZFJcOUPukAP8J3A3fuZ1g9Y44BG+P1sgApjLXTQPOzC4+7S9Wr8kXsfpINM4jpw==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-urls@5.0.0: resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} engines: {node: '>=18'} @@ -12601,6 +12600,10 @@ packages: fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + fetch-retry@4.1.1: resolution: {integrity: sha512-e6eB7zN6UBSwGVwrbWVH+gdLnkW9WwHhmq2YDK1Sh30pzx1onRVGBvogTlUeWxwTa+L86NYdo4hFkh7O8ZjSnA==} @@ -12741,10 +12744,6 @@ packages: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} - form-data@2.5.5: - resolution: {integrity: sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==} - engines: {node: '>= 0.12'} - form-data@3.0.4: resolution: {integrity: sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==} engines: {node: '>= 6'} @@ -12765,6 +12764,10 @@ packages: resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} engines: {node: '>= 12.20'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + forwarded-parse@2.1.2: resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} @@ -13036,10 +13039,18 @@ packages: resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} engines: {node: '>=14'} + gaxios@7.1.5: + resolution: {integrity: sha512-5FZy72Rh8LhtjmvDrKkI+lVhrsQrVKVsItxMoDm5mNQE+xR0WVIIs+jzPSJgBvKVsLi24fZhXJIsNI0bihDzFg==} + engines: {node: '>=18'} + gcp-metadata@6.1.0: resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==} engines: {node: '>=14'} + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + geist@1.5.1: resolution: {integrity: sha512-mAHZxIsL2o3ZITFaBVFBnwyDOw+zNLYum6A6nIjpzCGIO8QtC3V76XF2RnZTyLx1wlDTmMDy8jg3Ib52MIjGvQ==} peerDependencies: @@ -13208,8 +13219,12 @@ packages: peerDependencies: csstype: ^3.0.10 - google-auth-library@9.15.1: - resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} + google-auth-library@10.7.0: + resolution: {integrity: sha512-QpTAbNJ36TliZLx3TTtahR8HG0hN9RllL1e3FymOvQSIKK8JmgV58H924ub2wa2DsS3ANjjP1Aw1N+Ramc8hqQ==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} engines: {node: '>=14'} gopd@1.2.0: @@ -13235,10 +13250,6 @@ packages: peerDependencies: ioredis: '>=5' - gtoken@7.1.0: - resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} - engines: {node: '>=14.0.0'} - gzip-size@6.0.0: resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} engines: {node: '>=10'} @@ -13444,10 +13455,6 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} - http-proxy-agent@5.0.0: - resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==} - engines: {node: '>= 6'} - http-proxy-agent@7.0.2: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} @@ -13993,10 +14000,6 @@ packages: resolution: {integrity: sha512-jv+8jaWCl0g2lSBkNSVXdzfBA0npK1HGC2KtWM9FumFRoGS94g3NbCCLVnCYHLjp4GrW2KZeeSTMo5ddtznmGw==} engines: {node: '>=18'} - is@3.3.2: - resolution: {integrity: sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==} - engines: {node: '>= 0.4'} - isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} @@ -15472,6 +15475,10 @@ packages: encoding: optional: true + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-forge@1.3.1: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} engines: {node: '>= 6.13.0'} @@ -17170,9 +17177,9 @@ packages: retext@9.0.0: resolution: {integrity: sha512-sbMDcpHCNjvlheSgMfEcVrZko3cDzdbe1x/e7G66dFp0Ff7Mldvi2uv6JkJQzdRcvLYE8CA8Oe8siQx8ZOgTcA==} - retry-request@7.0.2: - resolution: {integrity: sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==} - engines: {node: '>=14'} + retry-request@8.0.3: + resolution: {integrity: sha512-qqoc4kkGgP9cmQDWELlOpAmfgJOg0Yi7MT82ZjiPWu451ayju4itwomjM4/dBEliify8C1b3tSaeCOldugtwPQ==} + engines: {node: '>=18'} retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} @@ -18060,9 +18067,9 @@ packages: tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} - teeny-request@9.0.0: - resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==} - engines: {node: '>=14'} + teeny-request@10.1.3: + resolution: {integrity: sha512-5yDliI1uWkYPo7W+Zvrxg6YmoWuj5iC5EydewqrRTvc68nyMTZhlPPlLg6cptUGfbQAb+N9XDPDPzF6N081lug==} + engines: {node: '>=18'} temp-dir@1.0.0: resolution: {integrity: sha512-xZFXEGbG7SNC3itwBzI3RYjq/cEhBkx2hJuKGIUOcEULmkQExXiHat2z/qkISYsuR+IKumhEfKKbV5qXmhICFQ==} @@ -19414,6 +19421,10 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + web-streams-polyfill@4.0.0-beta.3: resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} engines: {node: '>= 14'} @@ -24572,49 +24583,47 @@ snapshots: '@gar/promisify@1.1.3': {} - '@google-cloud/bigquery@7.9.4': + '@google-cloud/bigquery@8.3.1': dependencies: - '@google-cloud/common': 5.0.2 - '@google-cloud/paginator': 5.0.2 - '@google-cloud/precise-date': 4.0.0 - '@google-cloud/promisify': 4.0.0 - arrify: 2.0.1 - big.js: 6.2.2 + '@google-cloud/common': 6.0.1 + '@google-cloud/paginator': 6.0.1 + '@google-cloud/precise-date': 5.0.1 + '@google-cloud/promisify': 5.0.1 + arrify: 3.0.0 + big.js: 7.0.1 duplexify: 4.1.3 extend: 3.0.2 - is: 3.3.2 stream-events: 1.0.5 - uuid: 9.0.1 + teeny-request: 10.1.3 transitivePeerDependencies: - - encoding - supports-color - '@google-cloud/common@5.0.2': + '@google-cloud/common@6.0.1': dependencies: '@google-cloud/projectify': 4.0.0 '@google-cloud/promisify': 4.0.0 arrify: 2.0.1 duplexify: 4.1.3 extend: 3.0.2 - google-auth-library: 9.15.1 + google-auth-library: 10.7.0 html-entities: 2.6.0 - retry-request: 7.0.2 - teeny-request: 9.0.0 + retry-request: 8.0.3 + teeny-request: 10.1.3 transitivePeerDependencies: - - encoding - supports-color - '@google-cloud/paginator@5.0.2': + '@google-cloud/paginator@6.0.1': dependencies: - arrify: 2.0.1 extend: 3.0.2 - '@google-cloud/precise-date@4.0.0': {} + '@google-cloud/precise-date@5.0.1': {} '@google-cloud/projectify@4.0.0': {} '@google-cloud/promisify@4.0.0': {} + '@google-cloud/promisify@5.0.1': {} + '@graphql-typed-document-node/core@3.2.0(graphql@15.8.0)': dependencies: graphql: 15.8.0 @@ -30045,8 +30054,6 @@ snapshots: '@types/react': 19.2.7 '@types/react-dom': 19.2.3(@types/react@19.2.7) - '@tootallnate/once@2.0.1': {} - '@trpc-limiter/core@1.0.0(@trpc/client@11.17.0(@trpc/server@11.17.0(typescript@5.9.3))(typescript@5.9.3))(@trpc/server@11.17.0(typescript@5.9.3))': dependencies: '@trpc/client': 11.17.0(@trpc/server@11.17.0(typescript@5.9.3))(typescript@5.9.3) @@ -30150,8 +30157,6 @@ snapshots: dependencies: '@types/node': 20.19.24 - '@types/caseless@0.12.5': {} - '@types/compression@1.7.5': dependencies: '@types/express': 5.0.3 @@ -30596,13 +30601,6 @@ snapshots: dependencies: '@types/node': 20.19.24 - '@types/request@2.48.13': - dependencies: - '@types/caseless': 0.12.5 - '@types/node': 20.19.24 - '@types/tough-cookie': 4.0.5 - form-data: 2.5.5 - '@types/resolve@1.20.2': {} '@types/retry@0.12.0': {} @@ -30653,8 +30651,6 @@ snapshots: dependencies: '@types/node': 20.19.24 - '@types/tough-cookie@4.0.5': {} - '@types/triple-beam@1.3.5': {} '@types/tsscmp@1.0.2': {} @@ -31384,6 +31380,8 @@ snapshots: arrify@2.0.1: {} + arrify@3.0.0: {} + asap@2.0.6: {} assertion-error@1.1.0: {} @@ -31740,7 +31738,7 @@ snapshots: big-integer@1.6.52: {} - big.js@6.2.2: {} + big.js@7.0.1: {} bignumber.js@9.1.2: {} @@ -32895,6 +32893,8 @@ snapshots: dag-map@1.0.2: {} + data-uri-to-buffer@4.0.1: {} + data-urls@5.0.0: dependencies: whatwg-mimetype: 4.0.0 @@ -34309,6 +34309,11 @@ snapshots: fecha@4.2.3: {} + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + fetch-retry@4.1.1: {} fetchdts@0.1.7: {} @@ -34481,15 +34486,6 @@ snapshots: combined-stream: 1.0.8 mime-types: 2.1.35 - form-data@2.5.5: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - es-set-tostringtag: 2.1.0 - hasown: 2.0.2 - mime-types: 2.1.35 - safe-buffer: 5.2.1 - form-data@3.0.4: dependencies: asynckit: 0.4.0 @@ -34519,6 +34515,10 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 4.0.0-beta.3 + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + forwarded-parse@2.1.2: {} forwarded@0.2.0: {} @@ -34800,6 +34800,14 @@ snapshots: - encoding - supports-color + gaxios@7.1.5: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + gcp-metadata@6.1.0: dependencies: gaxios: 6.7.1 @@ -34808,6 +34816,14 @@ snapshots: - encoding - supports-color + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.1.5 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + geist@1.5.1(next@16.0.7(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)): dependencies: next: 16.0.7(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -35018,18 +35034,19 @@ snapshots: dependencies: csstype: 3.2.3 - google-auth-library@9.15.1: + google-auth-library@10.7.0: dependencies: base64-js: 1.5.1 ecdsa-sig-formatter: 1.0.11 - gaxios: 6.7.1 - gcp-metadata: 6.1.0 - gtoken: 7.1.0 + gaxios: 7.1.5 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 jws: 4.0.1 transitivePeerDependencies: - - encoding - supports-color + google-logging-utils@1.1.3: {} + gopd@1.2.0: {} graceful-fs@4.2.11: {} @@ -35046,14 +35063,6 @@ snapshots: cron-parser: 4.9.0 ioredis: 5.8.2 - gtoken@7.1.0: - dependencies: - gaxios: 6.7.1 - jws: 4.0.1 - transitivePeerDependencies: - - encoding - - supports-color - gzip-size@6.0.0: dependencies: duplexer: 0.1.2 @@ -35376,14 +35385,6 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 - http-proxy-agent@5.0.0: - dependencies: - '@tootallnate/once': 2.0.1 - agent-base: 6.0.2 - debug: 4.4.3 - transitivePeerDependencies: - - supports-color - http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 @@ -35902,8 +35903,6 @@ snapshots: dependencies: system-architecture: 0.1.0 - is@3.3.2: {} - isarray@1.0.0: {} isarray@2.0.5: {} @@ -37979,6 +37978,12 @@ snapshots: dependencies: whatwg-url: 5.0.0 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + node-forge@1.3.1: {} node-forge@1.4.0: {} @@ -40140,13 +40145,11 @@ snapshots: retext-stringify: 4.0.0 unified: 11.0.5 - retry-request@7.0.2: + retry-request@8.0.3: dependencies: - '@types/request': 2.48.13 extend: 3.0.2 - teeny-request: 9.0.0 + teeny-request: 10.1.3 transitivePeerDependencies: - - encoding - supports-color retry@0.13.1: {} @@ -41258,15 +41261,13 @@ snapshots: dependencies: bintrees: 1.0.2 - teeny-request@9.0.0: + teeny-request@10.1.3: dependencies: - http-proxy-agent: 5.0.0 - https-proxy-agent: 5.0.1 - node-fetch: 2.7.0 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 stream-events: 1.0.5 - uuid: 9.0.1 transitivePeerDependencies: - - encoding - supports-color temp-dir@1.0.0: {} @@ -42624,6 +42625,8 @@ snapshots: web-namespaces@2.0.1: {} + web-streams-polyfill@3.3.3: {} + web-streams-polyfill@4.0.0-beta.3: {} web-vitals@4.2.4: {} From 3e7dde4216bdde89fab1c2b081f73e50f53556e3 Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Thu, 11 Jun 2026 07:19:09 +0000 Subject: [PATCH 7/9] fix(warehouse): address CodeRabbit review findings on PR #391 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename PrismaJson type alias IWarehouseColumnMapping → IPrismaWarehouseColumnMapping to match the established IPrisma* prefix convention used by all other types in the namespace - Update schema.prisma annotation to match (/// [IPrismaWarehouseColumnMapping]) - Add @@unique([id, projectId]) on WarehouseSync to back a composite FK - Replace single-column syncId FK on WarehouseSyncRun with composite (syncId, projectId) → warehouse_syncs(id, projectId) to enforce that a run's projectId always matches its parent sync's projectId — closes the same cross-tenant gap that the existing composite FK on WarehouseSync already closes one level up --- .../migration.sql | 18 ++++++++++++++++++ packages/db/prisma/schema.prisma | 5 +++-- packages/db/src/types.ts | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql diff --git a/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql b/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql new file mode 100644 index 000000000..da017cffd --- /dev/null +++ b/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql @@ -0,0 +1,18 @@ +-- Tenant isolation: enforce that a sync run's projectId must match +-- its parent sync's projectId. Mirrors the same pattern used on +-- warehouse_syncs → warehouse_connections (composite FK there too). + +-- Step 1: unique index on warehouse_syncs(id, projectId) to back the FK +CREATE UNIQUE INDEX "warehouse_syncs_id_projectId_key" + ON "public"."warehouse_syncs"("id", "projectId"); + +-- Step 2: replace single-column syncId FK with a composite (syncId, projectId) FK +-- so that a run row cannot carry a projectId that differs from its sync +ALTER TABLE "public"."warehouse_sync_runs" + DROP CONSTRAINT "warehouse_sync_runs_syncId_fkey"; + +ALTER TABLE "public"."warehouse_sync_runs" + ADD CONSTRAINT "warehouse_sync_runs_syncId_projectId_fkey" + FOREIGN KEY ("syncId", "projectId") + REFERENCES "public"."warehouse_syncs"("id", "projectId") + ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 3f3a61fc2..fe9cc9171 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -769,7 +769,7 @@ model WarehouseSync { mappingType WarehouseSyncMappingType syncMode WarehouseSyncMode schedule WarehouseSyncSchedule? - /// [IWarehouseColumnMapping] + /// [IPrismaWarehouseColumnMapping] columnMapping Json lastCursor String? lastSyncedAt DateTime? @@ -786,6 +786,7 @@ model WarehouseSync { updatedAt DateTime @default(now()) @updatedAt runs WarehouseSyncRun[] + @@unique([id, projectId]) @@index([projectId]) @@index([connectionId]) @@map("warehouse_syncs") @@ -794,8 +795,8 @@ model WarehouseSync { model WarehouseSyncRun { id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid syncId String @db.Uuid - sync WarehouseSync @relation(fields: [syncId], references: [id], onDelete: Cascade) projectId String + sync WarehouseSync @relation(fields: [syncId, projectId], references: [id, projectId], onDelete: Cascade) project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) status WarehouseSyncRunStatus @default(pending) rowCount BigInt @default(0) diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 42b3a5211..c2f7ceda7 100755 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -28,7 +28,7 @@ declare global { type IPrismaClickhouseProfile = IClickhouseProfile; type IPrismaClickhouseBotEvent = IClickhouseBotEvent; type IPrismaCohortDefinition = CohortDefinition; - type IWarehouseColumnMapping = IWarehouseColumnMappingType; + type IPrismaWarehouseColumnMapping = IWarehouseColumnMappingType; // Each ChatMessage row stores one Better Agent `ConversationItem` // (message, tool call, or tool result) as JSON. Typed as `unknown[]` // here to avoid pulling `@better-agent/core` into @openpanel/db's From fc5dce7ec0e94090462430f75ea53e3cabbaeaf8 Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Thu, 11 Jun 2026 07:37:09 +0000 Subject: [PATCH 8/9] fix(warehouse): add backfill before composite FK on warehouse_sync_runs Before adding the (syncId, projectId) composite FK, normalize any runs whose projectId doesn't match their parent sync's projectId. Without this the ADD CONSTRAINT fails if mismatched rows exist. Tables are dev-only now so this is a no-op, but makes the migration safe to apply to any environment. --- .../migration.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql b/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql index da017cffd..5a9cb17da 100644 --- a/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql +++ b/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql @@ -11,6 +11,15 @@ CREATE UNIQUE INDEX "warehouse_syncs_id_projectId_key" ALTER TABLE "public"."warehouse_sync_runs" DROP CONSTRAINT "warehouse_sync_runs_syncId_fkey"; +-- Backfill: normalize any runs whose projectId doesn't match their parent sync. +-- Defensive — tables are dev-only at this stage, but makes the migration +-- safe to apply to any environment that might have mismatched rows. +UPDATE "public"."warehouse_sync_runs" r +SET "projectId" = s."projectId" +FROM "public"."warehouse_syncs" s +WHERE s."id" = r."syncId" + AND r."projectId" IS DISTINCT FROM s."projectId"; + ALTER TABLE "public"."warehouse_sync_runs" ADD CONSTRAINT "warehouse_sync_runs_syncId_projectId_fkey" FOREIGN KEY ("syncId", "projectId") From acc41072f88955ea351f22b951f4b37c5d1b29da Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Sat, 13 Jun 2026 15:04:21 +0000 Subject: [PATCH 9/9] =?UTF-8?q?fix(warehouse):=20phase=201=20audit=20?= =?UTF-8?q?=E2=80=94=20validation=20gaps,=20schema=20gap,=20type=20export?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validation fixes (packages/validation/src/index.ts): - Gap 1: reject whitespace-only connection names (Zod .refine + btrim DB check) - Gap 2: require createdAt cursor for profiles in append mode (superRefine) - Gap 4: block SQL injection tokens in partitionFilter (-- /* */ ;) - Gap B: export IBigQueryWarehouseConfig named type Schema + migrations: - migration 20260611000001: tighten name_nonempty_check to char_length(btrim(name)) > 0 - migration 20260611000002: add lastTestError String? to warehouse_connections so testConnection and connect can surface a human-readable failure reason (permission denied, project not found, key revoked) alongside the boolean status Regenerated Prisma client to include lastTestError field. --- .../migration.sql | 11 +++++++ .../migration.sql | 5 ++++ packages/db/prisma/schema.prisma | 1 + packages/db/src/generated/empty | 0 packages/validation/src/index.ts | 30 +++++++++++++++++-- 5 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 packages/db/prisma/migrations/20260611000001_warehouse_name_btrim_check/migration.sql create mode 100644 packages/db/prisma/migrations/20260611000002_warehouse_connection_last_test_error/migration.sql mode change 100644 => 100755 packages/db/src/generated/empty diff --git a/packages/db/prisma/migrations/20260611000001_warehouse_name_btrim_check/migration.sql b/packages/db/prisma/migrations/20260611000001_warehouse_name_btrim_check/migration.sql new file mode 100644 index 000000000..d62d3822d --- /dev/null +++ b/packages/db/prisma/migrations/20260611000001_warehouse_name_btrim_check/migration.sql @@ -0,0 +1,11 @@ +-- Tighten the name nonempty check on warehouse_connections. +-- The previous constraint used char_length(name) > 0, which passes +-- whitespace-only strings like ' ' (char_length returns 3). +-- btrim strips leading/trailing whitespace before the length check, +-- so ' ' → '' → char_length 0 → rejected. +ALTER TABLE "public"."warehouse_connections" + DROP CONSTRAINT "warehouse_connections_name_nonempty_check"; + +ALTER TABLE "public"."warehouse_connections" + ADD CONSTRAINT "warehouse_connections_name_nonempty_check" + CHECK (char_length(btrim("name")) > 0); diff --git a/packages/db/prisma/migrations/20260611000002_warehouse_connection_last_test_error/migration.sql b/packages/db/prisma/migrations/20260611000002_warehouse_connection_last_test_error/migration.sql new file mode 100644 index 000000000..25f407cdf --- /dev/null +++ b/packages/db/prisma/migrations/20260611000002_warehouse_connection_last_test_error/migration.sql @@ -0,0 +1,5 @@ +-- Add lastTestError to warehouse_connections so that failed connectivity tests +-- can surface a human-readable reason (permission denied, project not found, etc.) +-- rather than just a boolean false in lastTestStatus. +ALTER TABLE "public"."warehouse_connections" + ADD COLUMN "lastTestError" TEXT; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index fe9cc9171..3a6245a34 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -748,6 +748,7 @@ model WarehouseConnection { displayEmail String? lastTestedAt DateTime? lastTestStatus Boolean? + lastTestError String? createdAt DateTime @default(now()) updatedAt DateTime @default(now()) @updatedAt syncs WarehouseSync[] diff --git a/packages/db/src/generated/empty b/packages/db/src/generated/empty old mode 100644 new mode 100755 diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index d9e1d1c51..f50c5b466 100755 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -788,7 +788,10 @@ const zWarehouseConnectionName = z .regex( /^[a-zA-Z0-9 _\-]+$/, 'Connection name may only contain letters, digits, spaces, hyphens, and underscores', - ); + ) + .refine((s) => s.trim().length > 0, { + message: 'Connection name cannot consist of only whitespace', + }); export const zWarehouseConnectionCreate = z.object({ name: zWarehouseConnectionName, @@ -858,7 +861,14 @@ export const zBigQuerySyncConfig = z syncMode: z.enum(['append', 'full', 'onetime']), // Required for append and full (recurring) modes. Not set for onetime syncs. schedule: z.enum(['hourly', 'daily', 'weekly']).optional(), - partitionFilter: z.string().max(500).optional(), + partitionFilter: z + .string() + .max(500) + .refine( + (v) => !/--|\/\*|\*\/|;/.test(v), + 'Partition filter must not contain SQL comments (-- or /* */) or statement terminators (;)', + ) + .optional(), columnMapping: z.discriminatedUnion('mappingType', [ zBigQueryColumnMappingEvents, zBigQueryColumnMappingProfiles, @@ -873,7 +883,7 @@ export const zBigQuerySyncConfig = z message: 'schedule is required for append and full sync modes', }); } - // Append mode requires an insertTime cursor column + // Append mode requires an insertTime cursor column (events) if ( data.syncMode === 'append' && data.columnMapping.mappingType === 'events' && @@ -886,6 +896,19 @@ export const zBigQuerySyncConfig = z 'insertTime column is required for append mode — it must point to a TIMESTAMP column used as the incremental cursor', }); } + // Append mode requires a createdAt cursor column (profiles) + if ( + data.syncMode === 'append' && + data.columnMapping.mappingType === 'profiles' && + !data.columnMapping.createdAt + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['columnMapping', 'createdAt'], + message: + 'createdAt column is required for append mode on profiles — it must point to a TIMESTAMP column used as the incremental cursor', + }); + } // eventName and eventNameStatic are mutually exclusive if ( data.columnMapping.mappingType === 'events' && @@ -910,6 +933,7 @@ export type IWarehouseColumnMappingProfiles = z.infer< export type IWarehouseColumnMapping = | IWarehouseColumnMappingEvents | IWarehouseColumnMappingProfiles; +export type IBigQueryWarehouseConfig = z.infer; export type IBigQuerySyncConfig = z.infer; export type IWarehouseConnectionCreate = z.infer< typeof zWarehouseConnectionCreate