diff --git a/packages/db/package.json b/packages/db/package.json old mode 100644 new mode 100755 index 634a264b1..85e5be97e --- a/packages/db/package.json +++ b/packages/db/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "@clickhouse/client": "^1.18.5", + "@google-cloud/bigquery": "^8.3.1", "@openpanel/common": "workspace:*", "@openpanel/constants": "workspace:*", "@openpanel/json": "workspace:*", diff --git a/packages/db/prisma/migrations/20260607120000_add_event_meta_description/migration.sql b/packages/db/prisma/migrations/20260607120000_add_event_meta_description/migration.sql new file mode 100644 index 000000000..7ab277bb8 --- /dev/null +++ b/packages/db/prisma/migrations/20260607120000_add_event_meta_description/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."event_meta" ADD COLUMN "description" TEXT; diff --git a/packages/db/prisma/migrations/20260608090217_add_bigquery_connector/migration.sql b/packages/db/prisma/migrations/20260608090217_add_bigquery_connector/migration.sql new file mode 100644 index 000000000..90443c2c0 --- /dev/null +++ b/packages/db/prisma/migrations/20260608090217_add_bigquery_connector/migration.sql @@ -0,0 +1,76 @@ +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncMappingType" AS ENUM ('events', 'profiles'); + +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncMode" AS ENUM ('append', 'full'); + +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncSchedule" AS ENUM ('hourly', 'daily', 'weekly'); + +-- CreateEnum +CREATE TYPE "public"."BigQuerySyncRunStatus" AS ENUM ('pending', 'running', 'completed', 'failed'); + +-- CreateTable +CREATE TABLE "public"."bigquery_connections" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "projectId" TEXT NOT NULL, + "gcpProjectId" TEXT NOT NULL, + "serviceAccountEmail" TEXT NOT NULL, + "serviceAccountJsonEncrypted" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "bigquery_connections_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."bigquery_syncs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "connectionId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "displayName" TEXT NOT NULL, + "dataset" TEXT NOT NULL, + "tableName" TEXT NOT NULL, + "mappingType" "public"."BigQuerySyncMappingType" NOT NULL, + "syncMode" "public"."BigQuerySyncMode" NOT NULL, + "schedule" "public"."BigQuerySyncSchedule" NOT NULL, + "columnMapping" JSONB NOT NULL, + "lastCursor" TEXT, + "lastSyncedAt" TIMESTAMP(3), + "lastSyncStatus" TEXT, + "lastSyncError" TEXT, + "isEnabled" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "bigquery_syncs_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."bigquery_sync_runs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "syncId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "status" "public"."BigQuerySyncRunStatus" NOT NULL DEFAULT 'pending', + "rowCount" INTEGER NOT NULL DEFAULT 0, + "errorMessage" TEXT, + "startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TIMESTAMP(3), + + CONSTRAINT "bigquery_sync_runs_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "bigquery_connections_projectId_key" ON "public"."bigquery_connections"("projectId"); + +-- AddForeignKey +ALTER TABLE "public"."bigquery_connections" ADD CONSTRAINT "bigquery_connections_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."bigquery_syncs" ADD CONSTRAINT "bigquery_syncs_connectionId_fkey" FOREIGN KEY ("connectionId") REFERENCES "public"."bigquery_connections"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."bigquery_syncs" ADD CONSTRAINT "bigquery_syncs_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."bigquery_sync_runs" ADD CONSTRAINT "bigquery_sync_runs_syncId_fkey" FOREIGN KEY ("syncId") REFERENCES "public"."bigquery_syncs"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20260608091000_bigquery_multi_connection/migration.sql b/packages/db/prisma/migrations/20260608091000_bigquery_multi_connection/migration.sql new file mode 100644 index 000000000..003bde308 --- /dev/null +++ b/packages/db/prisma/migrations/20260608091000_bigquery_multi_connection/migration.sql @@ -0,0 +1,12 @@ +-- Allow multiple BigQuery connections per project (named connections) +-- Drop single-column unique index on projectId +DROP INDEX IF EXISTS "public"."bigquery_connections_projectId_key"; + +-- Add name column (default '' for any pre-existing rows during dev) +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "name" TEXT NOT NULL DEFAULT ''; + +-- Remove the default now that column exists +ALTER TABLE "public"."bigquery_connections" ALTER COLUMN "name" DROP DEFAULT; + +-- Add composite unique constraint: name must be unique within a project +CREATE UNIQUE INDEX "bigquery_connections_projectId_name_key" ON "public"."bigquery_connections"("projectId", "name"); diff --git a/packages/db/prisma/migrations/20260608120000_bigquery_harden/migration.sql b/packages/db/prisma/migrations/20260608120000_bigquery_harden/migration.sql new file mode 100644 index 000000000..ae3eaac88 --- /dev/null +++ b/packages/db/prisma/migrations/20260608120000_bigquery_harden/migration.sql @@ -0,0 +1,16 @@ +-- Phase 1 hardening: production-grade fields for BigQuery connector + +-- BigQueryConnection: region (GDPR compliance) + connection health tracking +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "gcpRegion" TEXT NOT NULL DEFAULT 'US'; +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "lastTestedAt" TIMESTAMP(3); +ALTER TABLE "public"."bigquery_connections" ADD COLUMN "lastTestStatus" BOOLEAN; + +-- BigQuerySync: typed status enum, circuit-breaker fields, partition filter +ALTER TABLE "public"."bigquery_syncs" ALTER COLUMN "lastSyncStatus" TYPE "public"."BigQuerySyncRunStatus" USING "lastSyncStatus"::"public"."BigQuerySyncRunStatus"; +ALTER TABLE "public"."bigquery_syncs" ADD COLUMN "errorRetryCount" INTEGER NOT NULL DEFAULT 0; +ALTER TABLE "public"."bigquery_syncs" ADD COLUMN "isErrorPaused" BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE "public"."bigquery_syncs" ADD COLUMN "partitionFilter" TEXT; + +-- BigQuerySyncRun: BigInt rowCount (INT max ~2.1B insufficient for large tables), bytes for cost tracking +ALTER TABLE "public"."bigquery_sync_runs" ALTER COLUMN "rowCount" TYPE BIGINT USING "rowCount"::BIGINT; +ALTER TABLE "public"."bigquery_sync_runs" ADD COLUMN "bytesProcessed" BIGINT; diff --git a/packages/db/prisma/migrations/20260608140000_bigquery_referential_integrity/migration.sql b/packages/db/prisma/migrations/20260608140000_bigquery_referential_integrity/migration.sql new file mode 100644 index 000000000..3a661fcf9 --- /dev/null +++ b/packages/db/prisma/migrations/20260608140000_bigquery_referential_integrity/migration.sql @@ -0,0 +1,29 @@ +-- Fix 1: Add FK on bigquery_sync_runs.projectId (was missing, allowing orphan runs) +ALTER TABLE "public"."bigquery_sync_runs" + ADD CONSTRAINT "bigquery_sync_runs_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Fix 2: Cross-tenant vulnerability — enforce connection belongs to same project as sync +-- Requires a unique index on (projectId, id) in bigquery_connections to back the composite FK +CREATE UNIQUE INDEX "bigquery_connections_projectId_id_key" + ON "public"."bigquery_connections"("projectId", "id"); + +-- Add composite FK on bigquery_syncs: (projectId, connectionId) must match a connection +-- that belongs to the same project. This is an additional constraint on top of Prisma's +-- single-column connectionId FK — both coexist for defence in depth. +ALTER TABLE "public"."bigquery_syncs" + ADD CONSTRAINT "bigquery_syncs_projectId_connectionId_fkey" + FOREIGN KEY ("projectId", "connectionId") + REFERENCES "public"."bigquery_connections"("projectId", "id") + ON DELETE CASCADE ON UPDATE CASCADE; + +-- Fix 3: Prevent empty-string connection names at the DB level +-- (Zod already blocks this at the application layer; this is belt-and-suspenders) +-- Backfill any dev rows that got the empty-string default from the prior migration +UPDATE "public"."bigquery_connections" + SET "name" = concat('connection_', "id") + WHERE char_length("name") = 0; + +ALTER TABLE "public"."bigquery_connections" + ADD CONSTRAINT "bigquery_connections_name_nonempty_check" + CHECK (char_length("name") > 0); diff --git a/packages/db/prisma/migrations/20260610115042_warehouse_restructure/migration.sql b/packages/db/prisma/migrations/20260610115042_warehouse_restructure/migration.sql new file mode 100644 index 000000000..23a9e367b --- /dev/null +++ b/packages/db/prisma/migrations/20260610115042_warehouse_restructure/migration.sql @@ -0,0 +1,124 @@ +-- Restructure: BigQuery-specific tables → generic Warehouse tables +-- Phase 1 tables are dev-only (no production data) — drop and recreate cleanly. +-- This makes the schema ready for BigQuery, Snowflake, Redshift, Databricks, Postgres. + +-- Drop old tables (CASCADE removes all FKs and indexes automatically) +DROP TABLE IF EXISTS "public"."bigquery_sync_runs"; +DROP TABLE IF EXISTS "public"."bigquery_syncs"; +DROP TABLE IF EXISTS "public"."bigquery_connections"; + +-- Drop old BigQuery-specific enums +DROP TYPE IF EXISTS "public"."BigQuerySyncMappingType"; +DROP TYPE IF EXISTS "public"."BigQuerySyncMode"; +DROP TYPE IF EXISTS "public"."BigQuerySyncSchedule"; +DROP TYPE IF EXISTS "public"."BigQuerySyncRunStatus"; + +-- WarehouseType: all supported warehouse providers +CREATE TYPE "public"."WarehouseType" AS ENUM ('bigquery', 'snowflake', 'redshift', 'databricks', 'postgres'); + +-- Shared sync enums (provider-agnostic) +CREATE TYPE "public"."WarehouseSyncMappingType" AS ENUM ('events', 'profiles'); +CREATE TYPE "public"."WarehouseSyncMode" AS ENUM ('append', 'full'); +CREATE TYPE "public"."WarehouseSyncSchedule" AS ENUM ('hourly', 'daily', 'weekly'); +CREATE TYPE "public"."WarehouseSyncRunStatus" AS ENUM ('pending', 'running', 'completed', 'failed'); + +-- warehouse_connections: one row per named connection, any provider +-- configEncrypted: AES-256-GCM encrypted JSON (shape validated by zWarehouseConfig discriminated union) +-- displayIdentifier: plain-text for UI display without decryption (GCP project ID / Snowflake account / etc.) +-- displayEmail: plain-text for UI display (SA email / username) +CREATE TABLE "public"."warehouse_connections" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "projectId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "type" "public"."WarehouseType" NOT NULL, + "configEncrypted" TEXT NOT NULL, + "displayIdentifier" TEXT, + "displayEmail" TEXT, + "lastTestedAt" TIMESTAMP(3), + "lastTestStatus" BOOLEAN, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "warehouse_connections_pkey" PRIMARY KEY ("id") +); + +CREATE TABLE "public"."warehouse_syncs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "connectionId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "displayName" TEXT NOT NULL, + "dataset" TEXT NOT NULL, + "tableName" TEXT NOT NULL, + "mappingType" "public"."WarehouseSyncMappingType" NOT NULL, + "syncMode" "public"."WarehouseSyncMode" NOT NULL, + "schedule" "public"."WarehouseSyncSchedule" NOT NULL, + "columnMapping" JSONB NOT NULL, + "lastCursor" TEXT, + "lastSyncedAt" TIMESTAMP(3), + "lastSyncStatus" "public"."WarehouseSyncRunStatus", + "lastSyncError" TEXT, + "isEnabled" BOOLEAN NOT NULL DEFAULT true, + "errorRetryCount" INTEGER NOT NULL DEFAULT 0, + "isErrorPaused" BOOLEAN NOT NULL DEFAULT false, + "partitionFilter" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "warehouse_syncs_pkey" PRIMARY KEY ("id") +); + +CREATE TABLE "public"."warehouse_sync_runs" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "syncId" UUID NOT NULL, + "projectId" TEXT NOT NULL, + "status" "public"."WarehouseSyncRunStatus" NOT NULL DEFAULT 'pending', + "rowCount" BIGINT NOT NULL DEFAULT 0, + "bytesProcessed" BIGINT, + "errorMessage" TEXT, + "startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TIMESTAMP(3), + CONSTRAINT "warehouse_sync_runs_pkey" PRIMARY KEY ("id") +); + +-- Unique indexes +-- (projectId, name): connection name must be unique per project, across ALL warehouse types +CREATE UNIQUE INDEX "warehouse_connections_projectId_name_key" + ON "public"."warehouse_connections"("projectId", "name"); + +-- (projectId, id): backs the composite FK on warehouse_syncs for cross-tenant protection +CREATE UNIQUE INDEX "warehouse_connections_projectId_id_key" + ON "public"."warehouse_connections"("projectId", "id"); + +-- DB-level name nonempty (belt-and-suspenders; Zod blocks empty strings at app layer) +ALTER TABLE "public"."warehouse_connections" + ADD CONSTRAINT "warehouse_connections_name_nonempty_check" + CHECK (char_length("name") > 0); + +-- FKs on warehouse_connections +ALTER TABLE "public"."warehouse_connections" + ADD CONSTRAINT "warehouse_connections_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- FKs on warehouse_syncs +ALTER TABLE "public"."warehouse_syncs" + ADD CONSTRAINT "warehouse_syncs_connectionId_fkey" + FOREIGN KEY ("connectionId") REFERENCES "public"."warehouse_connections"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +ALTER TABLE "public"."warehouse_syncs" + ADD CONSTRAINT "warehouse_syncs_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Composite FK: prevents cross-tenant exploit where a sync in project A +-- could reference a connection belonging to project B +ALTER TABLE "public"."warehouse_syncs" + ADD CONSTRAINT "warehouse_syncs_projectId_connectionId_fkey" + FOREIGN KEY ("projectId", "connectionId") + REFERENCES "public"."warehouse_connections"("projectId", "id") + ON DELETE CASCADE ON UPDATE CASCADE; + +-- FKs on warehouse_sync_runs +ALTER TABLE "public"."warehouse_sync_runs" + ADD CONSTRAINT "warehouse_sync_runs_syncId_fkey" + FOREIGN KEY ("syncId") REFERENCES "public"."warehouse_syncs"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +ALTER TABLE "public"."warehouse_sync_runs" + ADD CONSTRAINT "warehouse_sync_runs_projectId_fkey" + FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20260610115043_warehouse_security_fks/migration.sql b/packages/db/prisma/migrations/20260610115043_warehouse_security_fks/migration.sql new file mode 100644 index 000000000..1a751c228 --- /dev/null +++ b/packages/db/prisma/migrations/20260610115043_warehouse_security_fks/migration.sql @@ -0,0 +1,6 @@ +-- Drop the single-field connectionId FK on warehouse_syncs. +-- The composite FK (projectId, connectionId) → warehouse_connections(projectId, id) +-- from 20260610115042_warehouse_restructure already enforces the same referential +-- integrity PLUS prevents cross-tenant access — keeping both would be redundant. +ALTER TABLE "public"."warehouse_syncs" + DROP CONSTRAINT "warehouse_syncs_connectionId_fkey"; diff --git a/packages/db/prisma/migrations/20260610115044_warehouse_phase1_finalize/migration.sql b/packages/db/prisma/migrations/20260610115044_warehouse_phase1_finalize/migration.sql new file mode 100644 index 000000000..566a54f6d --- /dev/null +++ b/packages/db/prisma/migrations/20260610115044_warehouse_phase1_finalize/migration.sql @@ -0,0 +1,24 @@ +-- Phase 1 finalizing migration: performance indexes + column additions +-- These were identified as missing after the initial restructure. + +-- Performance indexes on FK columns (PostgreSQL does NOT auto-create these). +-- Required for list queries: syncs-by-project, syncs-by-connection, runs-by-sync. +CREATE INDEX "warehouse_syncs_projectId_idx" + ON "public"."warehouse_syncs"("projectId"); + +CREATE INDEX "warehouse_syncs_connectionId_idx" + ON "public"."warehouse_syncs"("connectionId"); + +CREATE INDEX "warehouse_sync_runs_syncId_idx" + ON "public"."warehouse_sync_runs"("syncId"); + +-- failureCount: tracks rows that were fetched but failed to import individually +-- (bad data, type mismatch, etc.) — separate from rowCount (successfully written rows). +-- Matches Mixpanel's Sync History "Failures" column in the UI. +ALTER TABLE "public"."warehouse_sync_runs" + ADD COLUMN "failureCount" BIGINT NOT NULL DEFAULT 0; + +-- createdBy: userId of the person who created this sync. +-- Used for displaying creator avatar + name in the sync list (matching Mixpanel UI). +ALTER TABLE "public"."warehouse_syncs" + ADD COLUMN "createdBy" TEXT; diff --git a/packages/db/prisma/migrations/20260610115045_warehouse_onetime_mode/migration.sql b/packages/db/prisma/migrations/20260610115045_warehouse_onetime_mode/migration.sql new file mode 100644 index 000000000..cf63e5bc2 --- /dev/null +++ b/packages/db/prisma/migrations/20260610115045_warehouse_onetime_mode/migration.sql @@ -0,0 +1,8 @@ +-- Add 'onetime' sync mode: run once immediately, then disable automatically. +-- Used for historical backfills — import all historical data once, then set up +-- an Append sync for ongoing data. +ALTER TYPE "public"."WarehouseSyncMode" ADD VALUE 'onetime'; + +-- Make schedule nullable: onetime syncs have no recurring schedule. +ALTER TABLE "public"."warehouse_syncs" + ALTER COLUMN "schedule" DROP NOT NULL; diff --git a/packages/db/prisma/migrations/20260610115046_warehouse_sync_overlap_delay/migration.sql b/packages/db/prisma/migrations/20260610115046_warehouse_sync_overlap_delay/migration.sql new file mode 100644 index 000000000..96ae7e6ef --- /dev/null +++ b/packages/db/prisma/migrations/20260610115046_warehouse_sync_overlap_delay/migration.sql @@ -0,0 +1,12 @@ +-- cursorOverlapMinutes: overlap window for append-mode cursor. +-- Rewinds the cursor by this many minutes to catch late-arriving rows that share +-- the same insertTime as the previous high-water mark. Worker handles dedup via +-- eventId (sha256 hash or user-mapped column) so no duplicate events are created. +ALTER TABLE "public"."warehouse_syncs" + ADD COLUMN "cursorOverlapMinutes" INTEGER NOT NULL DEFAULT 10; + +-- syncDelayMinutes: delay after scheduled fire time before running. +-- Allows pipeline data to fully land in BigQuery before the sync executes. +-- e.g. a daily sync firing at midnight waits N minutes for the ETL job to finish. +ALTER TABLE "public"."warehouse_syncs" + ADD COLUMN "syncDelayMinutes" INTEGER NOT NULL DEFAULT 0; diff --git a/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql b/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql new file mode 100644 index 000000000..5a9cb17da --- /dev/null +++ b/packages/db/prisma/migrations/20260611000000_warehouse_run_composite_fk/migration.sql @@ -0,0 +1,27 @@ +-- Tenant isolation: enforce that a sync run's projectId must match +-- its parent sync's projectId. Mirrors the same pattern used on +-- warehouse_syncs → warehouse_connections (composite FK there too). + +-- Step 1: unique index on warehouse_syncs(id, projectId) to back the FK +CREATE UNIQUE INDEX "warehouse_syncs_id_projectId_key" + ON "public"."warehouse_syncs"("id", "projectId"); + +-- Step 2: replace single-column syncId FK with a composite (syncId, projectId) FK +-- so that a run row cannot carry a projectId that differs from its sync +ALTER TABLE "public"."warehouse_sync_runs" + DROP CONSTRAINT "warehouse_sync_runs_syncId_fkey"; + +-- Backfill: normalize any runs whose projectId doesn't match their parent sync. +-- Defensive — tables are dev-only at this stage, but makes the migration +-- safe to apply to any environment that might have mismatched rows. +UPDATE "public"."warehouse_sync_runs" r +SET "projectId" = s."projectId" +FROM "public"."warehouse_syncs" s +WHERE s."id" = r."syncId" + AND r."projectId" IS DISTINCT FROM s."projectId"; + +ALTER TABLE "public"."warehouse_sync_runs" + ADD CONSTRAINT "warehouse_sync_runs_syncId_projectId_fkey" + FOREIGN KEY ("syncId", "projectId") + REFERENCES "public"."warehouse_syncs"("id", "projectId") + ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20260611000001_warehouse_name_btrim_check/migration.sql b/packages/db/prisma/migrations/20260611000001_warehouse_name_btrim_check/migration.sql new file mode 100644 index 000000000..d62d3822d --- /dev/null +++ b/packages/db/prisma/migrations/20260611000001_warehouse_name_btrim_check/migration.sql @@ -0,0 +1,11 @@ +-- Tighten the name nonempty check on warehouse_connections. +-- The previous constraint used char_length(name) > 0, which passes +-- whitespace-only strings like ' ' (char_length returns 3). +-- btrim strips leading/trailing whitespace before the length check, +-- so ' ' → '' → char_length 0 → rejected. +ALTER TABLE "public"."warehouse_connections" + DROP CONSTRAINT "warehouse_connections_name_nonempty_check"; + +ALTER TABLE "public"."warehouse_connections" + ADD CONSTRAINT "warehouse_connections_name_nonempty_check" + CHECK (char_length(btrim("name")) > 0); diff --git a/packages/db/prisma/migrations/20260611000002_warehouse_connection_last_test_error/migration.sql b/packages/db/prisma/migrations/20260611000002_warehouse_connection_last_test_error/migration.sql new file mode 100644 index 000000000..25f407cdf --- /dev/null +++ b/packages/db/prisma/migrations/20260611000002_warehouse_connection_last_test_error/migration.sql @@ -0,0 +1,5 @@ +-- Add lastTestError to warehouse_connections so that failed connectivity tests +-- can surface a human-readable reason (permission denied, project not found, etc.) +-- rather than just a boolean false in lastTestStatus. +ALTER TABLE "public"."warehouse_connections" + ADD COLUMN "lastTestError" TEXT; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index a171dc654..3a6245a34 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -251,8 +251,11 @@ model Project { notificationRules NotificationRule[] notifications Notification[] imports Import[] - gscConnection GscConnection? - cohorts Cohort[] + gscConnection GscConnection? + warehouseConnections WarehouseConnection[] + warehouseSyncs WarehouseSync[] + warehouseSyncRuns WarehouseSyncRun[] + cohorts Cohort[] // When deleteAt > now(), the project will be deleted deleteAt DateTime? @@ -500,12 +503,13 @@ model ShareWidget { } model EventMeta { - id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid - name String - conversion Boolean? - color String? - icon String? - projectId String + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + name String + conversion Boolean? + color String? + icon String? + description String? + projectId String project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) createdAt DateTime @default(now()) @@ -701,6 +705,112 @@ model GscConnection { @@map("gsc_connections") } +enum WarehouseType { + bigquery + snowflake + redshift + databricks + postgres +} + +enum WarehouseSyncMappingType { + events + profiles +} + +enum WarehouseSyncMode { + append + full + onetime +} + +enum WarehouseSyncSchedule { + hourly + daily + weekly +} + +enum WarehouseSyncRunStatus { + pending + running + completed + failed +} + +model WarehouseConnection { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + projectId String + name String + type WarehouseType + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + configEncrypted String + displayIdentifier String? + displayEmail String? + lastTestedAt DateTime? + lastTestStatus Boolean? + lastTestError String? + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + syncs WarehouseSync[] + + @@unique([projectId, name]) + @@unique([projectId, id]) + @@map("warehouse_connections") +} + +model WarehouseSync { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + connectionId String @db.Uuid + projectId String + connection WarehouseConnection @relation(fields: [projectId, connectionId], references: [projectId, id], onDelete: Cascade) + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + displayName String + dataset String + tableName String + mappingType WarehouseSyncMappingType + syncMode WarehouseSyncMode + schedule WarehouseSyncSchedule? + /// [IPrismaWarehouseColumnMapping] + columnMapping Json + lastCursor String? + lastSyncedAt DateTime? + lastSyncStatus WarehouseSyncRunStatus? + lastSyncError String? + isEnabled Boolean @default(true) + errorRetryCount Int @default(0) + isErrorPaused Boolean @default(false) + partitionFilter String? + cursorOverlapMinutes Int @default(10) + syncDelayMinutes Int @default(0) + createdBy String? + createdAt DateTime @default(now()) + updatedAt DateTime @default(now()) @updatedAt + runs WarehouseSyncRun[] + + @@unique([id, projectId]) + @@index([projectId]) + @@index([connectionId]) + @@map("warehouse_syncs") +} + +model WarehouseSyncRun { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + syncId String @db.Uuid + projectId String + sync WarehouseSync @relation(fields: [syncId, projectId], references: [id, projectId], onDelete: Cascade) + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + status WarehouseSyncRunStatus @default(pending) + rowCount BigInt @default(0) + failureCount BigInt @default(0) + bytesProcessed BigInt? + errorMessage String? + startedAt DateTime @default(now()) + completedAt DateTime? + + @@index([syncId]) + @@map("warehouse_sync_runs") +} + model EmailUnsubscribe { id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid email String diff --git a/packages/db/src/generated/empty b/packages/db/src/generated/empty old mode 100644 new mode 100755 diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts old mode 100644 new mode 100755 index aacd6d1e7..c2f7ceda7 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -1,11 +1,12 @@ import type { CohortDefinition, + IWarehouseColumnMapping as IWarehouseColumnMappingType, IImportConfig, IIntegrationConfig, INotificationRuleConfig, + InsightPayload, IProjectFilters, IWidgetOptions, - InsightPayload, } from '@openpanel/validation'; import type { IClickhouseBotEvent, @@ -27,6 +28,7 @@ declare global { type IPrismaClickhouseProfile = IClickhouseProfile; type IPrismaClickhouseBotEvent = IClickhouseBotEvent; type IPrismaCohortDefinition = CohortDefinition; + type IPrismaWarehouseColumnMapping = IWarehouseColumnMappingType; // Each ChatMessage row stores one Better Agent `ConversationItem` // (message, tool call, or tool result) as JSON. Typed as `unknown[]` // here to avoid pulling `@better-agent/core` into @openpanel/db's diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts old mode 100644 new mode 100755 index 4b2fd87cb..f50c5b466 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -692,6 +692,255 @@ export const zCreateImport = z.object({ export type ICreateImport = z.infer; +// BigQuery identifier validators — enforce BQ naming rules at form-save time, not query time + +// GCP project ID: lowercase letters, digits, hyphens; 6-30 chars; must start/end with letter or digit +const zGcpProjectId = z + .string() + .min(6, 'GCP project ID must be at least 6 characters') + .max(30, 'GCP project ID must be at most 30 characters') + .regex( + /^[a-z][a-z0-9\-]{4,28}[a-z0-9]$/, + 'GCP project ID must start with a lowercase letter, contain only lowercase letters, digits, and hyphens, and end with a letter or digit', + ); + +// BigQuery dataset/table names: letters, digits, underscores only (no hyphens without backtick quoting) +const zBqIdentifier = z + .string() + .min(1) + .max(1024) + .regex( + /^[a-zA-Z0-9_]+$/, + 'BigQuery identifiers may only contain letters, digits, and underscores (no hyphens or spaces)', + ); + +// Service account JSON structure check — catches authorized_user creds pasted by mistake +export const zServiceAccountJson = z + .string() + .min(1) + .max(16384, 'Service account JSON must be under 16 KB') + .superRefine((val, ctx) => { + let parsed: unknown; + try { + parsed = JSON.parse(val); + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Must be valid JSON', + }); + return; + } + const sa = parsed as Record; + if (sa.type !== 'service_account') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + 'Must be a service account key (type: "service_account"). Application Default Credentials are not supported.', + }); + } + if (!sa.private_key || typeof sa.private_key !== 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Service account key is missing the private_key field', + }); + } + if (!sa.client_email || typeof sa.client_email !== 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Service account key is missing the client_email field', + }); + } + if (!sa.project_id || typeof sa.project_id !== 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Service account key is missing the project_id field', + }); + } + }); + +// GCP region: regex-based so new Google Cloud regions work without code changes +const zWarehouseRegion = z + .string() + .min(1) + .max(63) + .regex( + /^([A-Z]+|[a-z]+-[a-z0-9]+-[0-9]+|[a-z]+-[a-z0-9]+)$/, + 'Must be a valid cloud region (e.g. US, EU, us-central1, asia-south1)', + ); + +// BigQuery-specific connection config +export const zBigQueryWarehouseConfig = z.object({ + type: z.literal('bigquery'), + gcpProjectId: zGcpProjectId, + gcpRegion: zWarehouseRegion.optional(), + serviceAccountJson: zServiceAccountJson, +}); + +// Discriminated union — add zSnowflakeWarehouseConfig, zRedshiftWarehouseConfig etc. here as they land +export const zWarehouseConfig = z.discriminatedUnion('type', [ + zBigQueryWarehouseConfig, +]); + +const zWarehouseConnectionName = z + .string() + .min(1) + .max(50) + .regex( + /^[a-zA-Z0-9 _\-]+$/, + 'Connection name may only contain letters, digits, spaces, hyphens, and underscores', + ) + .refine((s) => s.trim().length > 0, { + message: 'Connection name cannot consist of only whitespace', + }); + +export const zWarehouseConnectionCreate = z.object({ + name: zWarehouseConnectionName, + config: zWarehouseConfig, +}); + +// Column reference: simple field name or dot-notation path for STRUCT fields (e.g. user.profile.email) +const zBqColRef = z + .string() + .min(1) + .regex( + /^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*$/, + 'Column reference must be a valid BigQuery field name or dot-notation path (e.g. user.email)', + ); + +export const zBigQueryColumnMappingEvents = z.object({ + mappingType: z.literal('events'), + // Event name — two modes (only one should be set): + // eventName: column reference → value of that column becomes the event name per row + // eventNameStatic: fixed string → every row gets this event name (e.g. "Subscription") + // If neither is set, the table name is used as the event name fallback. + eventNameStatic: z.string().min(1).max(100).optional(), + // The authenticated user identifier — maps to profile_id in OpenPanel. + // For warehouse data this is always a known user ID (ClientCode, user_id, etc.). + // When absent, events are ingested without a profile association. + profileId: zBqColRef.optional(), + // Anonymous device identifier — maps to device_id in OpenPanel. + // Rare in warehouse tables; exists when the source table tracks both device + // and user separately (e.g. pre-login funnel data). + // If omitted, device_id is set equal to profileId at ingest time. + deviceId: zBqColRef.optional(), + // Custom event ID for deduplication — maps to the ClickHouse event id. + // When set, this column's value becomes the event's unique id (prevents + // duplicate imports on re-runs). When omitted, id is computed as + // sha256(syncId + ':' + rowHash). + eventId: zBqColRef.optional(), + // Revenue — first-class numeric field in OpenPanel events. + // Map to any numeric column representing transaction or subscription value. + revenue: zBqColRef.optional(), + eventName: zBqColRef.optional(), + timestamp: zBqColRef.optional(), + insertTime: zBqColRef.optional(), + // JSON column whose key-value content is flattened into event properties. + // e.g. a `metadata JSON` column → its keys become individual event properties. + jsonProperties: zBqColRef.optional(), +}); + +export const zBigQueryColumnMappingProfiles = z.object({ + mappingType: z.literal('profiles'), + profileIdColumn: zBqColRef, + firstName: zBqColRef.optional(), + lastName: zBqColRef.optional(), + email: zBqColRef.optional(), + avatar: zBqColRef.optional(), + // When the source table has a profile creation timestamp, map it here. + // Used to preserve original signup/creation date rather than defaulting to sync time. + createdAt: zBqColRef.optional(), + // JSON column whose key-value content is merged into profile properties. + jsonProperties: zBqColRef.optional(), +}); + +export const zBigQuerySyncConfig = z + .object({ + displayName: z.string().min(1).max(100), + dataset: zBqIdentifier, + tableName: zBqIdentifier, + syncMode: z.enum(['append', 'full', 'onetime']), + // Required for append and full (recurring) modes. Not set for onetime syncs. + schedule: z.enum(['hourly', 'daily', 'weekly']).optional(), + partitionFilter: z + .string() + .max(500) + .refine( + (v) => !/--|\/\*|\*\/|;/.test(v), + 'Partition filter must not contain SQL comments (-- or /* */) or statement terminators (;)', + ) + .optional(), + columnMapping: z.discriminatedUnion('mappingType', [ + zBigQueryColumnMappingEvents, + zBigQueryColumnMappingProfiles, + ]), + }) + .superRefine((data, ctx) => { + // Recurring modes require a schedule + if (data.syncMode !== 'onetime' && !data.schedule) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['schedule'], + message: 'schedule is required for append and full sync modes', + }); + } + // Append mode requires an insertTime cursor column (events) + if ( + data.syncMode === 'append' && + data.columnMapping.mappingType === 'events' && + !data.columnMapping.insertTime + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['columnMapping', 'insertTime'], + message: + 'insertTime column is required for append mode — it must point to a TIMESTAMP column used as the incremental cursor', + }); + } + // Append mode requires a createdAt cursor column (profiles) + if ( + data.syncMode === 'append' && + data.columnMapping.mappingType === 'profiles' && + !data.columnMapping.createdAt + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['columnMapping', 'createdAt'], + message: + 'createdAt column is required for append mode on profiles — it must point to a TIMESTAMP column used as the incremental cursor', + }); + } + // eventName and eventNameStatic are mutually exclusive + if ( + data.columnMapping.mappingType === 'events' && + data.columnMapping.eventName && + data.columnMapping.eventNameStatic + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['columnMapping', 'eventNameStatic'], + message: + 'set either eventName (column reference) or eventNameStatic (fixed value), not both', + }); + } + }); + +export type IWarehouseColumnMappingEvents = z.infer< + typeof zBigQueryColumnMappingEvents +>; +export type IWarehouseColumnMappingProfiles = z.infer< + typeof zBigQueryColumnMappingProfiles +>; +export type IWarehouseColumnMapping = + | IWarehouseColumnMappingEvents + | IWarehouseColumnMappingProfiles; +export type IBigQueryWarehouseConfig = z.infer; +export type IBigQuerySyncConfig = z.infer; +export type IWarehouseConnectionCreate = z.infer< + typeof zWarehouseConnectionCreate +>; +export type IWarehouseConfig = z.infer; +export { zGcpProjectId, zBqIdentifier }; + export * from './types.insights'; export * from './types.validation'; export * from './track.validation'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml old mode 100644 new mode 100755 index a7e168cf0..9681b5a12 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1214,6 +1214,9 @@ importers: '@clickhouse/client': specifier: ^1.18.5 version: 1.18.5 + '@google-cloud/bigquery': + specifier: ^8.3.1 + version: 8.3.1 '@openpanel/common': specifier: workspace:* version: link:../common @@ -4827,6 +4830,34 @@ packages: '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} + '@google-cloud/bigquery@8.3.1': + resolution: {integrity: sha512-F4g9oMgI3EB5Uo+6npRHDSWn1HkVko8GebG1tJtzasn/gknAEFeobQzuLeGYC6SJ92RVdaBpGVisw86P70RrHQ==} + engines: {node: '>=18'} + + '@google-cloud/common@6.0.1': + resolution: {integrity: sha512-1uvKzbmAWUdchIYRsg0f4rUmezOamWuVBSWAPAhnYoUE5OiPEx6v6JOxFIdr3MsrqV+6fyLV3EI1vMPlHoeRvw==} + engines: {node: '>=18'} + + '@google-cloud/paginator@6.0.1': + resolution: {integrity: sha512-HtzIe4n9b7It3MjimmFeXwQCuUsPI621e99zBTFqZjbPH7pZpRKDRptlYM0i0+nyot2XXhw0wPfVlTwwR/TyKA==} + engines: {node: '>=18'} + + '@google-cloud/precise-date@5.0.1': + resolution: {integrity: sha512-9HlRbOcDb8b2tSsOvljPD/Rm+Jn9KxMVB6sLf85CBnoIYdCFTNO1FIizQ13P75itXpSXsLuMlg1XK5opHKVzjg==} + engines: {node: '>=18'} + + '@google-cloud/projectify@4.0.0': + resolution: {integrity: sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==} + engines: {node: '>=14.0.0'} + + '@google-cloud/promisify@4.0.0': + resolution: {integrity: sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==} + engines: {node: '>=14'} + + '@google-cloud/promisify@5.0.1': + resolution: {integrity: sha512-Ste6NGraHq30ge3Sdq7m+pZE6lRUlkt2YgsEdq/vcoEgdbdZ/7h37Z1dMvivjhlbgrcmYRvlLF9FgI7tXm5wjg==} + engines: {node: '>=18'} + '@graphql-typed-document-node/core@3.2.0': resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==} peerDependencies: @@ -9949,6 +9980,7 @@ packages: '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} + deprecated: Potential CWE-502 - Update to 1.3.1 or higher '@unhead/vue@2.0.19': resolution: {integrity: sha512-7BYjHfOaoZ9+ARJkT10Q2TjnTUqDXmMpfakIAsD/hXiuff1oqWg1xeXT5+MomhNcC15HbiABpbbBmITLSHxdKg==} @@ -10432,6 +10464,10 @@ packages: resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} engines: {node: '>=8'} + arrify@3.0.0: + resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} + engines: {node: '>=12'} + asap@2.0.6: resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} @@ -10634,6 +10670,9 @@ packages: resolution: {integrity: sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==} engines: {node: '>=0.6'} + big.js@7.0.1: + resolution: {integrity: sha512-iFgV784tD8kq4ccF1xtNMZnXeZzVuXWWM+ERFzKQjv+A5G9HC8CY3DuV45vgzFFcW+u2tIvmF95+AzWgs6BjCg==} + bignumber.js@9.1.2: resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} @@ -11605,6 +11644,10 @@ packages: dag-map@1.0.2: resolution: {integrity: sha512-+LSAiGFwQ9dRnRdOeaj7g47ZFJcOUPukAP8J3A3fuZ1g9Y44BG+P1sgApjLXTQPOzC4+7S9Wr8kXsfpINM4jpw==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-urls@5.0.0: resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} engines: {node: '>=18'} @@ -12557,6 +12600,10 @@ packages: fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + fetch-retry@4.1.1: resolution: {integrity: sha512-e6eB7zN6UBSwGVwrbWVH+gdLnkW9WwHhmq2YDK1Sh30pzx1onRVGBvogTlUeWxwTa+L86NYdo4hFkh7O8ZjSnA==} @@ -12717,6 +12764,10 @@ packages: resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} engines: {node: '>= 12.20'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + forwarded-parse@2.1.2: resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} @@ -12988,10 +13039,18 @@ packages: resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} engines: {node: '>=14'} + gaxios@7.1.5: + resolution: {integrity: sha512-5FZy72Rh8LhtjmvDrKkI+lVhrsQrVKVsItxMoDm5mNQE+xR0WVIIs+jzPSJgBvKVsLi24fZhXJIsNI0bihDzFg==} + engines: {node: '>=18'} + gcp-metadata@6.1.0: resolution: {integrity: sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==} engines: {node: '>=14'} + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + geist@1.5.1: resolution: {integrity: sha512-mAHZxIsL2o3ZITFaBVFBnwyDOw+zNLYum6A6nIjpzCGIO8QtC3V76XF2RnZTyLx1wlDTmMDy8jg3Ib52MIjGvQ==} peerDependencies: @@ -13160,6 +13219,14 @@ packages: peerDependencies: csstype: ^3.0.10 + google-auth-library@10.7.0: + resolution: {integrity: sha512-QpTAbNJ36TliZLx3TTtahR8HG0hN9RllL1e3FymOvQSIKK8JmgV58H924ub2wa2DsS3ANjjP1Aw1N+Ramc8hqQ==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -13355,6 +13422,9 @@ packages: resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} engines: {node: '>=18'} + html-entities@2.6.0: + resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==} + html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} @@ -14158,9 +14228,15 @@ packages: jwa@1.4.1: resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + jws@3.2.2: resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + kafkajs@2.2.4: resolution: {integrity: sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==} engines: {node: '>=14.0.0'} @@ -15399,6 +15475,10 @@ packages: encoding: optional: true + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-forge@1.3.1: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} engines: {node: '>= 6.13.0'} @@ -16822,6 +16902,7 @@ packages: recharts@2.15.4: resolution: {integrity: sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==} engines: {node: '>=14'} + deprecated: 1.x and 2.x branches are no longer active. Bump to Recharts v3 to receive latest features and bugfixes. See https://github.com/recharts/recharts/wiki/3.0-migration-guide peerDependencies: react: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 @@ -17096,6 +17177,10 @@ packages: retext@9.0.0: resolution: {integrity: sha512-sbMDcpHCNjvlheSgMfEcVrZko3cDzdbe1x/e7G66dFp0Ff7Mldvi2uv6JkJQzdRcvLYE8CA8Oe8siQx8ZOgTcA==} + retry-request@8.0.3: + resolution: {integrity: sha512-qqoc4kkGgP9cmQDWELlOpAmfgJOg0Yi7MT82ZjiPWu451ayju4itwomjM4/dBEliify8C1b3tSaeCOldugtwPQ==} + engines: {node: '>=18'} + retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} @@ -17714,6 +17799,9 @@ packages: resolution: {integrity: sha512-uyQK/mx5QjHun80FLJTfaWE7JtwfRMKBLkMne6udYOmvH0CawotVa7TfgYHzAnpphn4+TweIx1QKMnRIbipmUg==} engines: {node: '>= 0.10.0'} + stream-events@1.0.5: + resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==} + stream-shift@1.0.3: resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==} @@ -17827,6 +17915,9 @@ packages: structured-headers@0.4.1: resolution: {integrity: sha512-0MP/Cxx5SzeeZ10p/bZI0S6MpgD+yxAhi1BOQ34jgnMXsCq3j1t6tQnZu+KdlL7dvJTLT3g9xN8tl10TqgFMcg==} + stubs@3.0.0: + resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==} + style-mod@4.1.3: resolution: {integrity: sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==} @@ -17976,6 +18067,10 @@ packages: tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + teeny-request@10.1.3: + resolution: {integrity: sha512-5yDliI1uWkYPo7W+Zvrxg6YmoWuj5iC5EydewqrRTvc68nyMTZhlPPlLg6cptUGfbQAb+N9XDPDPzF6N081lug==} + engines: {node: '>=18'} + temp-dir@1.0.0: resolution: {integrity: sha512-xZFXEGbG7SNC3itwBzI3RYjq/cEhBkx2hJuKGIUOcEULmkQExXiHat2z/qkISYsuR+IKumhEfKKbV5qXmhICFQ==} engines: {node: '>=4'} @@ -19326,6 +19421,10 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + web-streams-polyfill@4.0.0-beta.3: resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} engines: {node: '>= 14'} @@ -21102,7 +21201,7 @@ snapshots: dependencies: '@babel/template': 7.28.6 '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -22203,7 +22302,7 @@ snapshots: '@babel/core': 7.28.3 '@babel/helper-module-transforms': 7.28.6(@babel/core@7.28.3) '@babel/helper-plugin-utils': 7.28.6 - '@babel/helper-validator-identifier': 7.28.5 + '@babel/helper-validator-identifier': 7.29.7 '@babel/traverse': 7.29.0 transitivePeerDependencies: - supports-color @@ -22214,7 +22313,7 @@ snapshots: '@babel/core': 7.28.5 '@babel/helper-module-transforms': 7.28.6(@babel/core@7.28.5) '@babel/helper-plugin-utils': 7.28.6 - '@babel/helper-validator-identifier': 7.28.5 + '@babel/helper-validator-identifier': 7.29.7 '@babel/traverse': 7.29.0 transitivePeerDependencies: - supports-color @@ -22583,7 +22682,7 @@ snapshots: '@babel/helper-module-imports': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.28.5) - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -22594,7 +22693,7 @@ snapshots: '@babel/helper-module-imports': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0) - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -23061,7 +23160,7 @@ snapshots: dependencies: '@babel/core': 7.28.3 '@babel/helper-plugin-utils': 7.28.6 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 esutils: 2.0.3 optional: true @@ -23069,7 +23168,7 @@ snapshots: dependencies: '@babel/core': 7.28.5 '@babel/helper-plugin-utils': 7.28.6 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 esutils: 2.0.3 '@babel/preset-react@7.28.5(@babel/core@7.28.5)': @@ -24238,8 +24337,8 @@ snapshots: dependencies: '@babel/core': 7.29.0 '@babel/generator': 7.29.1 - '@babel/parser': 7.29.2 - '@babel/types': 7.29.0 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 '@expo/config': 8.5.4 '@expo/env': 0.2.3 '@expo/json-file': 8.3.3 @@ -24484,6 +24583,47 @@ snapshots: '@gar/promisify@1.1.3': {} + '@google-cloud/bigquery@8.3.1': + dependencies: + '@google-cloud/common': 6.0.1 + '@google-cloud/paginator': 6.0.1 + '@google-cloud/precise-date': 5.0.1 + '@google-cloud/promisify': 5.0.1 + arrify: 3.0.0 + big.js: 7.0.1 + duplexify: 4.1.3 + extend: 3.0.2 + stream-events: 1.0.5 + teeny-request: 10.1.3 + transitivePeerDependencies: + - supports-color + + '@google-cloud/common@6.0.1': + dependencies: + '@google-cloud/projectify': 4.0.0 + '@google-cloud/promisify': 4.0.0 + arrify: 2.0.1 + duplexify: 4.1.3 + extend: 3.0.2 + google-auth-library: 10.7.0 + html-entities: 2.6.0 + retry-request: 8.0.3 + teeny-request: 10.1.3 + transitivePeerDependencies: + - supports-color + + '@google-cloud/paginator@6.0.1': + dependencies: + extend: 3.0.2 + + '@google-cloud/precise-date@5.0.1': {} + + '@google-cloud/projectify@4.0.0': {} + + '@google-cloud/promisify@4.0.0': {} + + '@google-cloud/promisify@5.0.1': {} + '@graphql-typed-document-node/core@3.2.0(graphql@15.8.0)': dependencies: graphql: 15.8.0 @@ -31240,6 +31380,8 @@ snapshots: arrify@2.0.1: {} + arrify@3.0.0: {} + asap@2.0.6: {} assertion-error@1.1.0: {} @@ -31596,6 +31738,8 @@ snapshots: big-integer@1.6.52: {} + big.js@7.0.1: {} + bignumber.js@9.1.2: {} binary-extensions@2.2.0: {} @@ -32749,6 +32893,8 @@ snapshots: dag-map@1.0.2: {} + data-uri-to-buffer@4.0.1: {} + data-urls@5.0.0: dependencies: whatwg-mimetype: 4.0.0 @@ -33058,14 +33204,14 @@ snapshots: duplexify@3.7.1: dependencies: - end-of-stream: 1.4.4 + end-of-stream: 1.4.5 inherits: 2.0.4 readable-stream: 2.3.8 stream-shift: 1.0.3 duplexify@4.1.3: dependencies: - end-of-stream: 1.4.4 + end-of-stream: 1.4.5 inherits: 2.0.4 readable-stream: 3.6.2 stream-shift: 1.0.3 @@ -34163,6 +34309,11 @@ snapshots: fecha@4.2.3: {} + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + fetch-retry@4.1.1: {} fetchdts@0.1.7: {} @@ -34364,6 +34515,10 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 4.0.0-beta.3 + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + forwarded-parse@2.1.2: {} forwarded@0.2.0: {} @@ -34645,6 +34800,14 @@ snapshots: - encoding - supports-color + gaxios@7.1.5: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + gcp-metadata@6.1.0: dependencies: gaxios: 6.7.1 @@ -34653,6 +34816,14 @@ snapshots: - encoding - supports-color + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.1.5 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + geist@1.5.1(next@16.0.7(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)): dependencies: next: 16.0.7(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -34863,6 +35034,19 @@ snapshots: dependencies: csstype: 3.2.3 + google-auth-library@10.7.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.1.5 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.3: {} + gopd@1.2.0: {} graceful-fs@4.2.11: {} @@ -35153,6 +35337,8 @@ snapshots: dependencies: whatwg-encoding: 3.1.1 + html-entities@2.6.0: {} + html-escaper@3.0.3: {} html-to-text@9.0.5: @@ -36040,11 +36226,22 @@ snapshots: ecdsa-sig-formatter: 1.0.11 safe-buffer: 5.2.1 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + jws@3.2.2: dependencies: jwa: 1.4.1 safe-buffer: 5.2.1 + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + kafkajs@2.2.4: {} katex@0.16.21: @@ -37781,6 +37978,12 @@ snapshots: dependencies: whatwg-url: 5.0.0 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + node-forge@1.3.1: {} node-forge@1.4.0: {} @@ -39942,6 +40145,13 @@ snapshots: retext-stringify: 4.0.0 unified: 11.0.5 + retry-request@8.0.3: + dependencies: + extend: 3.0.2 + teeny-request: 10.1.3 + transitivePeerDependencies: + - supports-color + retry@0.13.1: {} reusify@1.0.4: {} @@ -40699,6 +40909,10 @@ snapshots: stream-buffers@2.2.0: {} + stream-events@1.0.5: + dependencies: + stubs: 3.0.0 + stream-shift@1.0.3: {} streamsearch@1.1.0: {} @@ -40837,6 +41051,8 @@ snapshots: structured-headers@0.4.1: {} + stubs@3.0.0: {} + style-mod@4.1.3: {} style-to-js@1.1.21: @@ -41045,6 +41261,15 @@ snapshots: dependencies: bintrees: 1.0.2 + teeny-request@10.1.3: + dependencies: + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + stream-events: 1.0.5 + transitivePeerDependencies: + - supports-color + temp-dir@1.0.0: {} temp-dir@2.0.0: {} @@ -42400,6 +42625,8 @@ snapshots: web-namespaces@2.0.1: {} + web-streams-polyfill@3.3.3: {} + web-streams-polyfill@4.0.0-beta.3: {} web-vitals@4.2.4: {}