From 95808e1797e941fd0f82f0e23f5ab279e1240e47 Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Sun, 7 Jun 2026 09:39:53 +0000 Subject: [PATCH 1/4] fix(export): include properties in /export/events response The export controller's `getEventList` call never enabled `properties` in its select object, so the column was silently omitted from SQL and every exported event had an empty properties field. Fixes #281 Co-Authored-By: Claude Sonnet 4.6 --- apps/api/src/controllers/export.controller.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/controllers/export.controller.ts b/apps/api/src/controllers/export.controller.ts index 15b045c03..4a8b25fb0 100644 --- a/apps/api/src/controllers/export.controller.ts +++ b/apps/api/src/controllers/export.controller.ts @@ -125,6 +125,7 @@ export async function events( select: { profile: false, meta: false, + properties: true, ...includes?.reduce((acc, key) => ({ ...acc, [key]: true }), {}), }, }; From 598b67388456d30ec441f06d3eb9f432176ab9b5 Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Sun, 7 Jun 2026 10:36:03 +0000 Subject: [PATCH 2/4] feat(export): add property_keys filter to reduce payload size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When exporting events with large property sets, callers can now pass `?property_keys=key1,key2` to receive only the specified keys in each event's properties map instead of the full payload. Uses ClickHouse's mapFilter function at query time so only the requested keys are transferred — no post-processing overhead. Co-Authored-By: Claude Sonnet 4.6 --- apps/api/src/controllers/export.controller.ts | 31 ++++++++++++------- packages/db/src/services/event.service.ts | 9 +++++- 2 files changed, 27 insertions(+), 13 deletions(-) mode change 100644 => 100755 apps/api/src/controllers/export.controller.ts mode change 100644 => 100755 packages/db/src/services/event.service.ts diff --git a/apps/api/src/controllers/export.controller.ts b/apps/api/src/controllers/export.controller.ts old mode 100644 new mode 100755 index 4a8b25fb0..7a418ae8d --- a/apps/api/src/controllers/export.controller.ts +++ b/apps/api/src/controllers/export.controller.ts @@ -86,19 +86,25 @@ export const eventsScheme = z.object({ includes: z .preprocess( (arg) => { - if (arg == null) { - return undefined; - } - if (Array.isArray(arg)) { - return arg; - } - if (typeof arg === 'string') { - const parts = arg.split(',').map((s) => s.trim()).filter(Boolean); - return parts; - } + if (arg == null) return undefined; + if (Array.isArray(arg)) return arg; + if (typeof arg === 'string') + return arg.split(',').map((s) => s.trim()).filter(Boolean); + return arg; + }, + z.array(z.string()), + ) + .optional(), + property_keys: z + .preprocess( + (arg) => { + if (arg == null) return undefined; + if (Array.isArray(arg)) return arg; + if (typeof arg === 'string') + return arg.split(',').map((s) => s.trim()).filter(Boolean); return arg; }, - z.array(z.string()) + z.array(z.string()), ) .optional(), }); @@ -110,7 +116,7 @@ export async function events( reply: FastifyReply ) { const projectId = await getProjectId(request); - const { limit, page: rawPage, event, start, end, profileId, includes, filters } = request.query; + const { limit, page: rawPage, event, start, end, profileId, includes, filters, property_keys } = request.query; const take = Math.max(Math.min(limit, 1000), 1); const cursor = Math.max(rawPage, 1) - 1; const options: GetEventListOptions = { @@ -122,6 +128,7 @@ export async function events( take, profileId, filters, + propertyKeys: property_keys, select: { profile: false, meta: false, diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts old mode 100644 new mode 100755 index 4a393c4d7..1d08d057b --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -466,6 +466,7 @@ export interface GetEventListOptions { select?: SelectHelper; custom?: (sb: SqlBuilderObject) => void; dateIntervalInDays?: number; + propertyKeys?: string[]; } export async function getEventList(options: GetEventListOptions) { @@ -484,6 +485,7 @@ export async function getEventList(options: GetEventListOptions) { custom, select: incomingSelect, dateIntervalInDays = 0.5, + propertyKeys, } = options; const { sb, getSql, join } = createSqlBuilder(); @@ -548,7 +550,12 @@ export async function getEventList(options: GetEventListOptions) { sb.select.sessionId = 'session_id'; } if (select.properties) { - sb.select.properties = 'properties'; + if (propertyKeys && propertyKeys.length > 0) { + const keys = propertyKeys.map((k) => sqlstring.escape(k)).join(', '); + sb.select.properties = `mapFilter((k, v) -> k IN (${keys}), properties)`; + } else { + sb.select.properties = 'properties'; + } } if (select.country) { sb.select.country = 'country'; From 91762e54b949d24248f3c650f2ffadc574be3a2a Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Sun, 7 Jun 2026 11:06:27 +0000 Subject: [PATCH 3/4] fix(export): add AS alias to mapFilter so properties key is preserved mapFilter(...) in a bare SELECT returns a column named after the full expression. transformEvent reads event.properties so the field was silently dropped. Adding 'AS properties' restores the expected column name. Co-Authored-By: Claude Sonnet 4.6 --- packages/db/src/services/event.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index 1d08d057b..8bda9a2cc 100755 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -552,7 +552,7 @@ export async function getEventList(options: GetEventListOptions) { if (select.properties) { if (propertyKeys && propertyKeys.length > 0) { const keys = propertyKeys.map((k) => sqlstring.escape(k)).join(', '); - sb.select.properties = `mapFilter((k, v) -> k IN (${keys}), properties)`; + sb.select.properties = `mapFilter((k, v) -> k IN (${keys}), properties) AS properties`; } else { sb.select.properties = 'properties'; } From 61c2df649b1a5526a4dd98c01697eb7f4e2247dc Mon Sep 17 00:00:00 2001 From: Lalit Shrotriya Date: Mon, 8 Jun 2026 05:33:40 +0000 Subject: [PATCH 4/4] refactor(export): deduplicate propertyKeys and extract shared preprocess helper - Deduplicate propertyKeys before building ClickHouse IN clause to avoid redundant keys in the SQL query - Extract repeated comma-separated array preprocessing logic into a shared helper to eliminate duplication between includes and property_keys fields Co-Authored-By: Claude Sonnet 4.6 --- apps/api/src/controllers/export.controller.ts | 34 ++++++------------- packages/db/src/services/event.service.ts | 2 +- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/apps/api/src/controllers/export.controller.ts b/apps/api/src/controllers/export.controller.ts index 7a418ae8d..86f279a71 100755 --- a/apps/api/src/controllers/export.controller.ts +++ b/apps/api/src/controllers/export.controller.ts @@ -60,6 +60,14 @@ async function getProjectId( return projectId; } +const preprocessCommaSeparatedArray = (arg: unknown) => { + if (arg == null) return undefined; + if (Array.isArray(arg)) return arg; + if (typeof arg === 'string') + return arg.split(',').map((s) => s.trim()).filter(Boolean); + return arg; +}; + export const eventsScheme = z.object({ project_id: z.string().optional(), projectId: z.string().optional(), @@ -83,30 +91,8 @@ export const eventsScheme = z.object({ return value; }, z.array(zChartEventFilter)) .optional(), - includes: z - .preprocess( - (arg) => { - if (arg == null) return undefined; - if (Array.isArray(arg)) return arg; - if (typeof arg === 'string') - return arg.split(',').map((s) => s.trim()).filter(Boolean); - return arg; - }, - z.array(z.string()), - ) - .optional(), - property_keys: z - .preprocess( - (arg) => { - if (arg == null) return undefined; - if (Array.isArray(arg)) return arg; - if (typeof arg === 'string') - return arg.split(',').map((s) => s.trim()).filter(Boolean); - return arg; - }, - z.array(z.string()), - ) - .optional(), + includes: z.preprocess(preprocessCommaSeparatedArray, z.array(z.string())).optional(), + property_keys: z.preprocess(preprocessCommaSeparatedArray, z.array(z.string())).optional(), }); export async function events( diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index 8bda9a2cc..788905372 100755 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -551,7 +551,7 @@ export async function getEventList(options: GetEventListOptions) { } if (select.properties) { if (propertyKeys && propertyKeys.length > 0) { - const keys = propertyKeys.map((k) => sqlstring.escape(k)).join(', '); + const keys = [...new Set(propertyKeys)].map((k) => sqlstring.escape(k)).join(', '); sb.select.properties = `mapFilter((k, v) -> k IN (${keys}), properties) AS properties`; } else { sb.select.properties = 'properties';