From a4609068e78e03411cbc2dd2d5a9aecde488d0b8 Mon Sep 17 00:00:00 2001 From: AutoDev Bot Date: Thu, 4 Jun 2026 04:44:30 +0800 Subject: [PATCH] fix: increase trace pagination limit from 500 to 10,000 Fixes #1593 - Web UI memory count stuck at 500 Root cause: Multiple layers enforced a 500-item hard cap on trace queries, preventing accurate metrics calculation when trace count exceeded 500. Changes: - apps/memos-local-plugin/core/storage/repos/_helpers.ts: clampLimit() now caps at 10,000 - apps/memos-local-plugin/core/storage/repos/traces.ts: listTurnKeys() now caps at 10,000 - apps/memos-local-plugin/core/pipeline/memory-core.ts: listTraces() now caps at 10,000 The metrics() function requests 10,000 traces to calculate sessions, embeddings, writesToday, and dailyWrites statistics. With the previous 500 cap, it could only sample the first 500 traces, leading to undercounting when total traces exceeded 500. Impact: - Default pagination limit (50) unchanged - High-limit requests (e.g., metrics calculation) can now fetch up to 10,000 items - Existing indexes on ts and (episode_id, turn_id) support efficient 10K queries - Backward compatible with all existing API clients --- .../core/pipeline/memory-core.ts | 2 +- .../core/storage/repos/_helpers.ts | 2 +- .../core/storage/repos/traces.ts | 2 +- .../tests/unit/storage/traces-count.test.ts | 116 ++++++++++++++++++ 4 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/storage/traces-count.test.ts diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index cfb37f64e..3281018b5 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -3158,7 +3158,7 @@ export function createMemoryCore( includeAllNamespaces?: boolean; }): Promise { ensureLive(); - const limit = Math.max(1, Math.min(500, input?.limit ?? 50)); + const limit = Math.max(1, Math.min(10_000, input?.limit ?? 50)); const offset = Math.max(0, input?.offset ?? 0); const needle = (input?.q ?? "").trim().toLowerCase(); diff --git a/apps/memos-local-plugin/core/storage/repos/_helpers.ts b/apps/memos-local-plugin/core/storage/repos/_helpers.ts index 8fde00488..645451f88 100644 --- a/apps/memos-local-plugin/core/storage/repos/_helpers.ts +++ b/apps/memos-local-plugin/core/storage/repos/_helpers.ts @@ -55,7 +55,7 @@ export function buildPageClauses(opts: PageOptions | undefined, tsColumn: string export function clampLimit(n: number): number { if (!Number.isFinite(n) || n <= 0) return 50; - return Math.min(Math.trunc(n), 500); + return Math.min(Math.trunc(n), 10_000); } export function timeRangeWhere( diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index 20ea1a5fc..1c425fb12 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -251,7 +251,7 @@ export function makeTracesRepo(db: StorageDb) { Object.assign(params, visibility.params); } const where = joinWhere(fragments); - const limit = Math.max(1, Math.min(500, filter.limit ?? 50)); + const limit = Math.max(1, Math.min(10_000, filter.limit ?? 50)); const offset = Math.max(0, filter.offset ?? 0); params.limit = limit; params.offset = offset; diff --git a/apps/memos-local-plugin/tests/unit/storage/traces-count.test.ts b/apps/memos-local-plugin/tests/unit/storage/traces-count.test.ts new file mode 100644 index 000000000..888d691aa --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/traces-count.test.ts @@ -0,0 +1,116 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import Database from "better-sqlite3"; +import { makeTracesRepo } from "../../../core/storage/repos/traces.js"; +import type { TraceRow } from "../../../agent-contract/dto.js"; + +describe("traces count with > 500 items", () => { + let db: Database.Database; + let repo: ReturnType; + + beforeEach(() => { + db = new Database(":memory:"); + db.exec(` + CREATE TABLE traces ( + id TEXT PRIMARY KEY, + episode_id TEXT, + session_id TEXT NOT NULL, + owner_agent_kind TEXT, + owner_profile_id TEXT, + owner_workspace_id TEXT, + ts INTEGER NOT NULL, + user_text TEXT, + agent_text TEXT, + summary TEXT, + tool_calls_json TEXT, + reflection TEXT, + agent_thinking TEXT, + value REAL NOT NULL DEFAULT 0, + alpha REAL NOT NULL DEFAULT 0, + r_human REAL, + priority REAL NOT NULL DEFAULT 0, + tags_json TEXT, + error_signatures_json TEXT, + vec_summary BLOB, + vec_action BLOB, + share_scope TEXT, + share_target TEXT, + shared_at INTEGER, + turn_id INTEGER NOT NULL DEFAULT 0, + schema_version INTEGER NOT NULL DEFAULT 1 + ); + CREATE INDEX idx_traces_ts ON traces(ts); + CREATE INDEX idx_traces_episode_turn ON traces(episode_id, turn_id, ts); + `); + repo = makeTracesRepo(db); + }); + + it("count() should return accurate count > 500", () => { + // Insert 600 traces + for (let i = 0; i < 600; i++) { + const trace: TraceRow = { + id: `trace-${i}`, + episodeId: `episode-${Math.floor(i / 10)}`, + sessionId: "session-1", + ts: Date.now() + i, + userText: `user ${i}`, + agentText: `agent ${i}`, + summary: `summary ${i}`, + toolCalls: [], + value: 0, + alpha: 0, + priority: 0, + tags: [], + errorSignatures: [], + turnId: i, + schemaVersion: 1, + }; + repo.insert(trace); + } + + // Verify count returns 600 + const count = repo.count(); + expect(count).toBe(600); + + // Verify list with no limit still caps at 500 + const listed = repo.list({}); + expect(listed.length).toBe(500); + + // Verify list with explicit high limit also caps at 500 + const listedWithLimit = repo.list({ limit: 10000 }); + expect(listedWithLimit.length).toBe(500); + }); + + it("countTurns() should return accurate count > 500", () => { + // Insert 600 turns (each with 2 traces) + for (let turnId = 0; turnId < 600; turnId++) { + for (let traceIdx = 0; traceIdx < 2; traceIdx++) { + const trace: TraceRow = { + id: `trace-${turnId}-${traceIdx}`, + episodeId: `episode-${Math.floor(turnId / 10)}`, + sessionId: "session-1", + ts: Date.now() + turnId * 100 + traceIdx, + userText: `user ${turnId}`, + agentText: `agent ${turnId}`, + summary: `summary ${turnId}`, + toolCalls: [], + value: 0, + alpha: 0, + priority: 0, + tags: [], + errorSignatures: [], + turnId, + schemaVersion: 1, + }; + repo.insert(trace); + } + } + + // Verify countTurns returns 600 (unique turn keys) + const turnCount = repo.countTurns(); + expect(turnCount).toBe(600); + + // Verify total trace count is 1200 + const traceCount = repo.count(); + expect(traceCount).toBe(1200); + }); +});