From b58d3720c3b0bec6bcbdc1bbb8b0159123e3298f Mon Sep 17 00:00:00 2001 From: Richardson Gunde Date: Mon, 6 Jul 2026 15:32:03 +0530 Subject: [PATCH] fix(harness): goal gate fails closed on an unreadable events.jsonl (#200) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Goal-activity detection read events.jsonl through a tolerant reader that returns [] on failure, so a corrupt/lost event log on a recipe-driven run made the stage-11 gate report 'no active goal' and PASS without requiring goal_evaluation — fail-open. activeGoalContext now distinguishes an absent log (genuine non-goal run) from one that exists but yields zero parseable events or can't be read (indeterminate), and the gate returns a goal_activity_indeterminate FAIL. Documents the permanent-activity + fail- closed behavior in HARNESS.md. Co-Authored-By: Claude Opus 4.8 --- docs/HARNESS.md | 8 ++++++- src/core/harness/goal-check.js | 41 +++++++++++++++++++++++++++++--- tests/harness-goal-check.test.js | 21 ++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/HARNESS.md b/docs/HARNESS.md index ac17328..d1a56b0 100644 --- a/docs/HARNESS.md +++ b/docs/HARNESS.md @@ -345,7 +345,13 @@ proving a `pipeline loop --goal ` context — a task targeting `11-feedb validation when feedback.json is missing or its `goal_evaluation` section is malformed, with the named checks recorded in validation.json instead of a silent ASK_USER later. Runs with no active goal keep the section optional (a single informational `goal_evaluation_not_required` PASS), and -tasks that never target stage 11 see no goal checks at all. +tasks that never target stage 11 see no goal checks at all. Goal-activity is **permanent** — any +historical `loop_iteration_started`/`goal_evaluated` event marks the run goal-driven for the rest +of its life, so a later unrelated stage-11 revalidation still demands `goal_evaluation` +(conservative by design). And it is **fail-closed on unreadable state (#200)**: if `events.jsonl` +exists but yields zero parseable events, or can't be read at all, the gate returns a +`goal_activity_indeterminate` FAIL rather than assuming "no goal" — a corrupt event log at stage 11 +stops for human eyes instead of silently passing. **Evaluator.** `evaluateGoal(projectRoot, runId, options)` builds the rollup in memory (persists nothing), reads only structured JSON (never prose), always layers harness checks over the criteria diff --git a/src/core/harness/goal-check.js b/src/core/harness/goal-check.js index 8f1e176..b9218bf 100644 --- a/src/core/harness/goal-check.js +++ b/src/core/harness/goal-check.js @@ -520,9 +520,33 @@ async function activeGoalContext(runDir) { if (existsSync(join(runDir, GOAL_DEFINITION_FILE))) { return { active: true, source: GOAL_DEFINITION_FILE }; } - const events = await readJsonlIfPresent(join(runDir, 'events.jsonl')); - const loopEvent = events.find((event) => GOAL_ACTIVE_LOOP_EVENT_TYPES.has(event?.type)); - if (loopEvent) return { active: true, source: `loop event ${loopEvent.type}` }; + // Distinguish "no events file" (a genuine non-goal run) from "events file + // present but unreadable/corrupt". In the latter case we CANNOT tell whether + // a `pipeline loop --goal ` drove this run — the proof lives only in + // events.jsonl — so the tolerant `[]` fallback would silently disarm the gate + // (fail-open). Report indeterminate instead and let the gate fail closed + // (#200; matches goal 4 "never silently degraded" and the #82 data-integrity + // philosophy). A legitimately empty events.jsonl (brand-new run) is NOT + // corrupt — only non-empty content that yields zero parseable events, or a + // read failure despite the file existing, is. + const eventsPath = join(runDir, 'events.jsonl'); + if (existsSync(eventsPath)) { + let raw; + try { + raw = await readFile(eventsPath, 'utf8'); + } catch { + return { active: false, indeterminate: true, reason: 'events.jsonl exists but could not be read' }; + } + const nonEmptyLines = raw.split('\n').filter((line) => line.trim()); + const events = nonEmptyLines.flatMap((line) => { + try { return [JSON.parse(line)]; } catch { return []; } + }); + if (nonEmptyLines.length > 0 && events.length === 0) { + return { active: false, indeterminate: true, reason: `events.jsonl has ${nonEmptyLines.length} line(s) but none are parseable JSON` }; + } + const loopEvent = events.find((event) => GOAL_ACTIVE_LOOP_EVENT_TYPES.has(event?.type)); + if (loopEvent) return { active: true, source: `loop event ${loopEvent.type}` }; + } return { active: false, source: null }; } @@ -538,6 +562,17 @@ export async function validateStageGoalEvaluation({ runDir, stageIds = [] } = {} return summarize([], { goal_active: false, goal_source: null }); } const goalContext = await activeGoalContext(runDir); + if (goalContext.indeterminate) { + // Fail closed: goal activity can't be determined because the run's event + // log is unreadable, so we refuse to pass the gate on the assumption there + // is no goal (#200). A corrupt events.jsonl at stage 11 is a real + // data-integrity problem worth stopping for, not a silent PASS. + return summarize([{ + name: 'goal_activity_indeterminate', + status: 'FAIL', + evidence: `cannot determine goal activity: ${goalContext.reason} — refusing to pass stage-11 validation on unreadable run state (fail-closed)`, + }], { goal_active: false, goal_source: null, goal_activity_indeterminate: true }); + } if (!goalContext.active) { return summarize([{ name: 'goal_evaluation_not_required', diff --git a/tests/harness-goal-check.test.js b/tests/harness-goal-check.test.js index 7d2a317..4d5fff8 100644 --- a/tests/harness-goal-check.test.js +++ b/tests/harness-goal-check.test.js @@ -737,6 +737,27 @@ test('validateStageGoalEvaluation: non-stage-11 tasks and goal-less runs stay un assert.equal(junk.checks.length, 0); }); +test('validateStageGoalEvaluation: a corrupt events.jsonl fails closed instead of disarming the gate (#200)', async () => { + const projectRoot = mkdtempSync(path.join(os.tmpdir(), 'rstack-goal-')); + + // No goal.json — goal activity can only be proven from events.jsonl. A run + // whose event log exists but is unparseable must NOT silently read as "no + // active goal": we can't tell, so fail closed. + const corruptDir = seedRun(projectRoot, 'run-corrupt', { tasks: [task('001', 'PASS')] }); + writeFileSync(path.join(corruptDir, 'events.jsonl'), 'not json\n{ truncated at the sta\n'); + const corrupt = await validateStageGoalEvaluation({ runDir: corruptDir, stageIds: ['11-feedback-loop'] }); + assert.equal(corrupt.ok, false, 'corrupt event log fails the gate'); + assert.equal(corrupt.goal_activity_indeterminate, true); + assert.ok(corrupt.issues.some((check) => check.name === 'goal_activity_indeterminate')); + + // A legitimately empty events.jsonl (brand-new run) is NOT corrupt — the + // section stays optional, a single informational PASS. + const emptyDir = seedRun(projectRoot, 'run-empty', { tasks: [task('001', 'PASS')], events: [] }); + const empty = await validateStageGoalEvaluation({ runDir: emptyDir, stageIds: ['11-feedback-loop'] }); + assert.equal(empty.ok, true, 'an empty event log is a clean non-goal run, not a failure'); + assert.equal(empty.checks[0].name, 'goal_evaluation_not_required'); +}); + test('validateStageGoalEvaluation: a goal-driven run FAILs a missing or malformed goal_evaluation (#196)', async () => { const projectRoot = mkdtempSync(path.join(os.tmpdir(), 'rstack-goal-')); const goal = { goal_id: 'fixture', criteria: [{ id: 'c1', kind: 'judge', question: 'Is it done?' }] };