Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion docs/HARNESS.md
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,13 @@ proving a `pipeline loop --goal <recipe>` context — a task targeting `11-feedb
validation when feedback.json is missing or its `goal_evaluation` section is malformed, with the
named checks recorded in validation.json instead of a silent ASK_USER later. Runs with no active
goal keep the section optional (a single informational `goal_evaluation_not_required` PASS), and
tasks that never target stage 11 see no goal checks at all.
tasks that never target stage 11 see no goal checks at all. Goal-activity is **permanent** — any
historical `loop_iteration_started`/`goal_evaluated` event marks the run goal-driven for the rest
of its life, so a later unrelated stage-11 revalidation still demands `goal_evaluation`
(conservative by design). And it is **fail-closed on unreadable state (#200)**: if `events.jsonl`
exists but yields zero parseable events, or can't be read at all, the gate returns a
`goal_activity_indeterminate` FAIL rather than assuming "no goal" — a corrupt event log at stage 11
stops for human eyes instead of silently passing.

**Evaluator.** `evaluateGoal(projectRoot, runId, options)` builds the rollup in memory (persists
nothing), reads only structured JSON (never prose), always layers harness checks over the criteria
Expand Down
41 changes: 38 additions & 3 deletions src/core/harness/goal-check.js
Original file line number Diff line number Diff line change
Expand Up @@ -520,9 +520,33 @@ async function activeGoalContext(runDir) {
if (existsSync(join(runDir, GOAL_DEFINITION_FILE))) {
return { active: true, source: GOAL_DEFINITION_FILE };
}
const events = await readJsonlIfPresent(join(runDir, 'events.jsonl'));
const loopEvent = events.find((event) => GOAL_ACTIVE_LOOP_EVENT_TYPES.has(event?.type));
if (loopEvent) return { active: true, source: `loop event ${loopEvent.type}` };
// Distinguish "no events file" (a genuine non-goal run) from "events file
// present but unreadable/corrupt". In the latter case we CANNOT tell whether
// a `pipeline loop --goal <recipe>` drove this run — the proof lives only in
// events.jsonl — so the tolerant `[]` fallback would silently disarm the gate
// (fail-open). Report indeterminate instead and let the gate fail closed
// (#200; matches goal 4 "never silently degraded" and the #82 data-integrity
// philosophy). A legitimately empty events.jsonl (brand-new run) is NOT
// corrupt — only non-empty content that yields zero parseable events, or a
// read failure despite the file existing, is.
const eventsPath = join(runDir, 'events.jsonl');
if (existsSync(eventsPath)) {
let raw;
try {
raw = await readFile(eventsPath, 'utf8');
} catch {
return { active: false, indeterminate: true, reason: 'events.jsonl exists but could not be read' };
}
const nonEmptyLines = raw.split('\n').filter((line) => line.trim());
const events = nonEmptyLines.flatMap((line) => {
try { return [JSON.parse(line)]; } catch { return []; }
});
if (nonEmptyLines.length > 0 && events.length === 0) {
return { active: false, indeterminate: true, reason: `events.jsonl has ${nonEmptyLines.length} line(s) but none are parseable JSON` };
}
const loopEvent = events.find((event) => GOAL_ACTIVE_LOOP_EVENT_TYPES.has(event?.type));
if (loopEvent) return { active: true, source: `loop event ${loopEvent.type}` };
}
return { active: false, source: null };
}

Expand All @@ -538,6 +562,17 @@ export async function validateStageGoalEvaluation({ runDir, stageIds = [] } = {}
return summarize([], { goal_active: false, goal_source: null });
}
const goalContext = await activeGoalContext(runDir);
if (goalContext.indeterminate) {
// Fail closed: goal activity can't be determined because the run's event
// log is unreadable, so we refuse to pass the gate on the assumption there
// is no goal (#200). A corrupt events.jsonl at stage 11 is a real
// data-integrity problem worth stopping for, not a silent PASS.
return summarize([{
name: 'goal_activity_indeterminate',
status: 'FAIL',
evidence: `cannot determine goal activity: ${goalContext.reason} — refusing to pass stage-11 validation on unreadable run state (fail-closed)`,
}], { goal_active: false, goal_source: null, goal_activity_indeterminate: true });
}
if (!goalContext.active) {
return summarize([{
name: 'goal_evaluation_not_required',
Expand Down
21 changes: 21 additions & 0 deletions tests/harness-goal-check.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -737,6 +737,27 @@ test('validateStageGoalEvaluation: non-stage-11 tasks and goal-less runs stay un
assert.equal(junk.checks.length, 0);
});

test('validateStageGoalEvaluation: a corrupt events.jsonl fails closed instead of disarming the gate (#200)', async () => {
const projectRoot = mkdtempSync(path.join(os.tmpdir(), 'rstack-goal-'));

// No goal.json — goal activity can only be proven from events.jsonl. A run
// whose event log exists but is unparseable must NOT silently read as "no
// active goal": we can't tell, so fail closed.
const corruptDir = seedRun(projectRoot, 'run-corrupt', { tasks: [task('001', 'PASS')] });
writeFileSync(path.join(corruptDir, 'events.jsonl'), 'not json\n{ truncated at the sta\n');
const corrupt = await validateStageGoalEvaluation({ runDir: corruptDir, stageIds: ['11-feedback-loop'] });
assert.equal(corrupt.ok, false, 'corrupt event log fails the gate');
assert.equal(corrupt.goal_activity_indeterminate, true);
assert.ok(corrupt.issues.some((check) => check.name === 'goal_activity_indeterminate'));

// A legitimately empty events.jsonl (brand-new run) is NOT corrupt — the
// section stays optional, a single informational PASS.
const emptyDir = seedRun(projectRoot, 'run-empty', { tasks: [task('001', 'PASS')], events: [] });
const empty = await validateStageGoalEvaluation({ runDir: emptyDir, stageIds: ['11-feedback-loop'] });
assert.equal(empty.ok, true, 'an empty event log is a clean non-goal run, not a failure');
assert.equal(empty.checks[0].name, 'goal_evaluation_not_required');
});

test('validateStageGoalEvaluation: a goal-driven run FAILs a missing or malformed goal_evaluation (#196)', async () => {
const projectRoot = mkdtempSync(path.join(os.tmpdir(), 'rstack-goal-'));
const goal = { goal_id: 'fixture', criteria: [{ id: 'c1', kind: 'judge', question: 'Is it done?' }] };
Expand Down
Loading