diff --git a/apps/code/src/main/bootstrap.ts b/apps/code/src/main/bootstrap.ts index 24f5e98b49..725340b0b8 100644 --- a/apps/code/src/main/bootstrap.ts +++ b/apps/code/src/main/bootstrap.ts @@ -58,6 +58,12 @@ app.commandLine.appendSwitch("enable-logging", "file"); app.commandLine.appendSwitch("log-file", chromiumLogPath); app.commandLine.appendSwitch("log-level", "0"); +// Allow programmatic audio playback without a prior user gesture. The agent +// speaks (and completion sounds ring) autonomously, with no click at that +// moment, so Chromium's default gesture requirement would silently reject +// HTMLMediaElement.play(). Must be set before app "ready". +app.commandLine.appendSwitch("autoplay-policy", "no-user-gesture-required"); + // In dev, expose the renderer over CDP (:9222 by default) for the // test-electron-app skill. electron-vite launches Electron itself, so this is // set in-process rather than via a CLI flag. POSTHOG_CODE_CDP_PORT matches the diff --git a/apps/code/src/main/di/bindings.ts b/apps/code/src/main/di/bindings.ts index ba0f490822..1c3db763fa 100644 --- a/apps/code/src/main/di/bindings.ts +++ b/apps/code/src/main/di/bindings.ts @@ -186,6 +186,10 @@ import type { ISecureStoreService, SECURE_STORE_SERVICE, } from "@posthog/workspace-server/services/secure-store/identifiers"; +import type { + ISpeechSynthesizer, + SPEECH_SYNTHESIZER_SERVICE, +} from "@posthog/workspace-server/services/speech/identifiers"; import type { SUSPENSION_FILE_WATCHER, SUSPENSION_SERVICE, @@ -460,6 +464,7 @@ export interface MainBindings { [MAIN_SECURE_STORE_BACKEND]: typeof rendererStore; [MAIN_SECURE_STORE_SERVICE]: SecureStoreService; [SECURE_STORE_SERVICE]: ISecureStoreService; + [SPEECH_SYNTHESIZER_SERVICE]: ISpeechSynthesizer; [LOGS_SERVICE]: ILogsService; [MAIN_ENCRYPTION_SERVICE]: EncryptionService; [MAIN_DISCORD_PRESENCE_SERVICE]: DiscordPresenceService; diff --git a/apps/code/src/main/di/container.ts b/apps/code/src/main/di/container.ts index bca03305ef..2750e20687 100644 --- a/apps/code/src/main/di/container.ts +++ b/apps/code/src/main/di/container.ts @@ -192,6 +192,7 @@ import { SECURE_STORE_SERVICE } from "@posthog/workspace-server/services/secure- import { shellModule } from "@posthog/workspace-server/services/shell/shell.module"; import { skillsModule } from "@posthog/workspace-server/services/skills/skills.module"; import { skillsMarketplaceModule } from "@posthog/workspace-server/services/skills-marketplace/skills-marketplace.module"; +import { SPEECH_SYNTHESIZER_SERVICE } from "@posthog/workspace-server/services/speech/identifiers"; import { SUSPENSION_FILE_WATCHER, SUSPENSION_SERVICE, @@ -258,6 +259,7 @@ import { DiscordPresenceService } from "../services/discord-presence/service"; import { EncryptionService } from "../services/encryption/service"; import { SecureStoreService } from "../services/secure-store/service"; import { settingsStore } from "../services/settingsStore"; +import { ElevenLabsSpeechService } from "../services/speech/service"; import { WorkspaceServerService } from "../services/workspace-server/service"; import { getUserDataDir, isDevBuild } from "../utils/env"; import { logger } from "../utils/logger"; @@ -717,6 +719,10 @@ container .to(SecureStoreService) .inSingletonScope(); container.bind(SECURE_STORE_SERVICE).toService(MAIN_SECURE_STORE_SERVICE); +container + .bind(SPEECH_SYNTHESIZER_SERVICE) + .to(ElevenLabsSpeechService) + .inSingletonScope(); container.bind(LOGS_SERVICE).toDynamicValue((ctx) => { const ws = ctx.get(MAIN_WORKSPACE_CLIENT); return { diff --git a/apps/code/src/main/services/speech/service.ts b/apps/code/src/main/services/speech/service.ts new file mode 100644 index 0000000000..a9ef588887 --- /dev/null +++ b/apps/code/src/main/services/speech/service.ts @@ -0,0 +1,96 @@ +import { logger } from "@main/utils/logger"; +import { + type ISecureStoreService, + SECURE_STORE_SERVICE, +} from "@posthog/workspace-server/services/secure-store/identifiers"; +import { + ELEVENLABS_API_KEY_STORE_KEY, + type ISpeechSynthesizer, + type SpeechSynthesisResult, +} from "@posthog/workspace-server/services/speech/identifiers"; +import { inject, injectable } from "inversify"; + +const log = logger.scope("speech"); + +// Expressive default voice (Eleven v3). Overridable per call via voiceId. +const DEFAULT_VOICE_ID = "goT3UYdM9bhm0n2lmKQx"; +const MODEL_ID = "eleven_v3"; +const OUTPUT_FORMAT = "mp3_44100_128"; +// Cache synthesized audio by voice+text so repeated lines (e.g. the "finished" +// backstop for a given task) aren't re-billed/re-fetched. Bounded LRU — audio +// is ~100KB, so this caps at a few MB. +const CACHE_MAX = 32; + +/** + * ElevenLabs-backed speech synthesizer. Reads the API key from encrypted secure + * storage and returns MP3 bytes to the renderer, which plays them (host-neutral + * playback). When no key is configured or synthesis fails, returns null so the + * renderer falls back to the system voice. Best-effort — never throws. + */ +@injectable() +export class ElevenLabsSpeechService implements ISpeechSynthesizer { + // voice+text -> audioBase64; insertion-ordered for LRU eviction. + private readonly cache = new Map(); + + constructor( + @inject(SECURE_STORE_SERVICE) + private readonly secureStore: ISecureStoreService, + ) {} + + async synthesize( + text: string, + voiceId?: string, + ): Promise { + const apiKey = this.secureStore.getItem(ELEVENLABS_API_KEY_STORE_KEY); + if (!apiKey) { + log.info("No ElevenLabs key configured — using system voice"); + return null; + } + const voice = voiceId?.trim() || DEFAULT_VOICE_ID; + + const cacheKey = `${voice}:${text}`; + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + // Refresh recency (delete + re-set moves it to the end). + this.cache.delete(cacheKey); + this.cache.set(cacheKey, cached); + log.info(`Speech cache hit (voice=${voice}, chars=${text.length})`); + return { audioBase64: cached, mimeType: "audio/mpeg" }; + } + + log.info( + `Synthesizing via ElevenLabs (voice=${voice}, chars=${text.length})`, + ); + try { + const res = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${voice}?output_format=${OUTPUT_FORMAT}`, + { + method: "POST", + headers: { "xi-api-key": apiKey, "content-type": "application/json" }, + body: JSON.stringify({ + text, + model_id: MODEL_ID, + voice_settings: { stability: 0.5 }, + }), + }, + ); + if (!res.ok) { + const detail = await res.text().catch(() => ""); + log.warn( + `ElevenLabs failed: HTTP ${res.status} ${detail.slice(0, 300)}`, + ); + return null; + } + const buffer = Buffer.from(await res.arrayBuffer()); + const audioBase64 = buffer.toString("base64"); + this.cache.set(cacheKey, audioBase64); + if (this.cache.size > CACHE_MAX) { + this.cache.delete(this.cache.keys().next().value as string); + } + return { audioBase64, mimeType: "audio/mpeg" }; + } catch (error) { + log.warn("ElevenLabs synthesis error", error); + return null; + } + } +} diff --git a/apps/code/src/main/trpc/router.ts b/apps/code/src/main/trpc/router.ts index 41688352bd..c6bd789ffe 100644 --- a/apps/code/src/main/trpc/router.ts +++ b/apps/code/src/main/trpc/router.ts @@ -42,6 +42,7 @@ import { shellRouter } from "@posthog/host-router/routers/shell.router"; import { skillsRouter } from "@posthog/host-router/routers/skills.router"; import { slackIntegrationRouter } from "@posthog/host-router/routers/slack-integration.router"; import { sleepRouter } from "@posthog/host-router/routers/sleep.router"; +import { speechRouter } from "@posthog/host-router/routers/speech.router"; import { suspensionRouter } from "@posthog/host-router/routers/suspension.router"; import { uiRouter } from "@posthog/host-router/routers/ui.router"; import { updatesRouter } from "@posthog/host-router/routers/updates.router"; @@ -98,6 +99,7 @@ export const trpcRouter = router({ suspension: suspensionRouter, secureStore: secureStoreRouter, shell: shellRouter, + speech: speechRouter, skills: skillsRouter, slackIntegration: slackIntegrationRouter, ui: uiRouter, diff --git a/apps/code/src/renderer/desktop-contributions.ts b/apps/code/src/renderer/desktop-contributions.ts index 4ad7cb89f7..de7dcc41d5 100644 --- a/apps/code/src/renderer/desktop-contributions.ts +++ b/apps/code/src/renderer/desktop-contributions.ts @@ -6,6 +6,7 @@ import { githubConnectModule } from "@posthog/core/integrations/githubConnect.mo import { onboardingModule } from "@posthog/core/onboarding/onboarding.module"; import { setupCoreModule } from "@posthog/core/setup/setup.module"; import { skillsCoreModule } from "@posthog/core/skills/skills.module"; +import { speechCoreModule } from "@posthog/core/speech/speech.module"; import { CONTRIBUTION } from "@posthog/di/contribution"; import { agentUiModule } from "@posthog/ui/features/agent/agent.module"; import { authUiModule } from "@posthog/ui/features/auth/auth.module"; @@ -48,6 +49,7 @@ export function registerDesktopContributions(): void { setupCoreModule, setupUiModule, skillsCoreModule, + speechCoreModule, workspaceUiModule, ]) { container.load(module); diff --git a/apps/code/src/renderer/desktop-services.ts b/apps/code/src/renderer/desktop-services.ts index 20e84feea6..f4e878c880 100644 --- a/apps/code/src/renderer/desktop-services.ts +++ b/apps/code/src/renderer/desktop-services.ts @@ -42,6 +42,12 @@ import { type SessionService, } from "@posthog/core/sessions/sessionService"; import { SETUP_STORE } from "@posthog/core/setup/identifiers"; +import { + SPEECH_SETTINGS_PROVIDER, + SPEECH_USER_NAME_PROVIDER, + type SpeechSettingsProvider, + type UserNameProvider, +} from "@posthog/core/speech/identifiers"; import { resolveService } from "@posthog/di/container"; import { ROOT_LOGGER, type RootLogger } from "@posthog/di/logger"; import { @@ -49,6 +55,7 @@ import { NOTIFICATIONS_SERVICE, type NotificationTarget, } from "@posthog/platform/notifications"; +import { type ISpeech, SPEECH_SERVICE } from "@posthog/platform/speech"; import { type Adapter, AUTORESEARCH_FLAG, @@ -58,6 +65,7 @@ import { AUTH_SIDE_EFFECTS, type IAuthSideEffects, } from "@posthog/ui/features/auth/identifiers"; +import { authKeys } from "@posthog/ui/features/auth/useCurrentUser"; import { FEATURE_FLAGS, type FeatureFlags, @@ -77,7 +85,9 @@ import { ACTIVE_VIEW_PROVIDER, type IActiveView, type INotificationSettings, + type ISpeechNotifySettings, NOTIFICATION_SETTINGS_PROVIDER, + SPEECH_NOTIFY_SETTINGS, } from "@posthog/ui/features/notifications/identifiers"; import { OnboardingGithubConnectClient } from "@posthog/ui/features/onboarding/githubConnectClientImpl"; import { @@ -85,14 +95,26 @@ import { type AgentPromptSender, } from "@posthog/ui/features/sessions/agentPromptSender"; import { useSettingsStore } from "@posthog/ui/features/settings/settingsStore"; +import { + type ISpeechKeyStore, + SPEECH_KEY_STORE, +} from "@posthog/ui/features/settings/speechKeyStore"; import { getCurrentMatches } from "@posthog/ui/router/navigationBridge"; import { HEDGEHOG_MODE_HOST } from "@posthog/ui/shell/hedgehogModeHost"; import { posthogFeatureFlags } from "@posthog/ui/shell/posthogAnalyticsImpl"; +import type { ImperativeQueryClient } from "@posthog/ui/shell/queryClient"; import { IMPERATIVE_QUERY_CLIENT } from "@posthog/ui/shell/queryClient"; import { FILE_PATH_RESOLVER, type FilePathResolver, } from "@posthog/ui/utils/getFilePath"; +import { + isSpeechSupported, + playAudioBase64, + speakSystemVoice, + stopSpeech, +} from "@posthog/ui/utils/speech"; +import { ELEVENLABS_API_KEY_STORE_KEY } from "@posthog/workspace-server/services/speech/identifiers"; import { container } from "@renderer/di/container"; import { RendererAuthSideEffects } from "@renderer/platform-adapters/auth-side-effects"; import { gitCacheKeyProvider } from "@renderer/platform-adapters/git-cache-keys"; @@ -329,6 +351,86 @@ container.bind(FILE_WATCHER_CLIENT).toConstantValue({ stop: (repoPath: string) => trpcClient.fileWatcher.stop.mutate({ repoPath }), }); +// Spoken notifications: synthesize in the host (ElevenLabs, key stays there), +// play in the renderer from a blob URL (host-neutral). Fall back to the system +// voice when no key is set or synthesis fails. speak() resolves when playback +// ends, so the core queue serializes utterances. +const speechLog = logger.scope("speech-adapter"); +container.bind(SPEECH_SERVICE).toConstantValue({ + isSupported: () => isSpeechSupported(), + speak: async (text, opts) => { + try { + const result = await hostTrpcClient.speech.synthesize.query({ + text, + voiceId: opts?.voiceId || undefined, + }); + if (result?.audioBase64) { + await playAudioBase64(result.audioBase64, result.mimeType); + return; + } + } catch (err) { + speechLog.warn("Synthesis failed; using system voice", err); + } + await speakSystemVoice(text); + }, + stop: () => stopSpeech(), +}); + +container + .bind(SPEECH_SETTINGS_PROVIDER) + .toConstantValue({ + get: () => { + const s = useSettingsStore.getState(); + return { + enabled: s.spokenNotifications, + voiceId: s.elevenLabsVoiceId || undefined, + }; + }, + }); + +container.bind(SPEECH_USER_NAME_PROVIDER).toConstantValue({ + getFirstName: () => { + try { + const qc = container.get(IMPERATIVE_QUERY_CLIENT); + for (const [, data] of qc.getQueriesData({ + queryKey: authKeys.currentUsers(), + })) { + const first = ( + data as { first_name?: string | null } | undefined + )?.first_name?.trim(); + if (first) return first; + } + } catch { + // best-effort — no name means we just skip the "Hey " prefix + } + return undefined; + }, +}); + +container.bind(SPEECH_KEY_STORE).toConstantValue({ + save: (apiKey) => + hostTrpcClient.secureStore.setItem + .query({ key: ELEVENLABS_API_KEY_STORE_KEY, value: apiKey }) + .then(() => {}), + clear: () => + hostTrpcClient.secureStore.removeItem + .query({ key: ELEVENLABS_API_KEY_STORE_KEY }) + .then(() => {}), +}); + +container.bind(SPEECH_NOTIFY_SETTINGS).toConstantValue({ + get: () => { + const s = useSettingsStore.getState(); + return { + enabled: s.spokenNotifications, + needsInput: s.spokenNotifyNeedsInput, + completion: s.spokenNotifyCompletion, + progress: s.spokenNotifyProgress, + focusMode: s.spokenFocusMode, + }; + }, +}); + container .bind(FEATURE_FLAGS) .toConstantValue(posthogFeatureFlags); diff --git a/apps/code/src/renderer/di/bindings.ts b/apps/code/src/renderer/di/bindings.ts index 4fdea08b65..dbdca0c7df 100644 --- a/apps/code/src/renderer/di/bindings.ts +++ b/apps/code/src/renderer/di/bindings.ts @@ -97,6 +97,12 @@ import { import { type ISetupStore, SETUP_STORE } from "@posthog/core/setup/identifiers"; import { SKILLS_WORKSPACE_CLIENT } from "@posthog/core/skills/identifiers"; import type { SkillsWorkspaceClient } from "@posthog/core/skills/teamSkillsService"; +import { + SPEECH_SETTINGS_PROVIDER, + SPEECH_USER_NAME_PROVIDER, + type SpeechSettingsProvider, + type UserNameProvider, +} from "@posthog/core/speech/identifiers"; import { TASK_CREATION_EFFECTS, TASK_CREATION_HOST, @@ -137,6 +143,7 @@ import { type INotifications, NOTIFICATIONS_SERVICE, } from "@posthog/platform/notifications"; +import { type ISpeech, SPEECH_SERVICE } from "@posthog/platform/speech"; import { AUTH_SIDE_EFFECTS, type IAuthSideEffects, @@ -188,7 +195,9 @@ import { ACTIVE_VIEW_PROVIDER, type IActiveView, type INotificationSettings, + type ISpeechNotifySettings, NOTIFICATION_SETTINGS_PROVIDER, + SPEECH_NOTIFY_SETTINGS, } from "@posthog/ui/features/notifications/identifiers"; import { AGENT_PROMPT_SENDER, @@ -202,6 +211,10 @@ import { DEV_MODE_CLIENT, type DevModeClient, } from "@posthog/ui/features/settings/devModeClient"; +import { + type ISpeechKeyStore, + SPEECH_KEY_STORE, +} from "@posthog/ui/features/settings/speechKeyStore"; import { SHELL_CLIENT, type ShellClient, @@ -316,6 +329,11 @@ export interface RendererBindings { [NOTIFICATIONS_SERVICE]: INotifications; [NOTIFICATION_SETTINGS_PROVIDER]: INotificationSettings; [ACTIVE_VIEW_PROVIDER]: IActiveView; + [SPEECH_SERVICE]: ISpeech; + [SPEECH_SETTINGS_PROVIDER]: SpeechSettingsProvider; + [SPEECH_USER_NAME_PROVIDER]: UserNameProvider; + [SPEECH_NOTIFY_SETTINGS]: ISpeechNotifySettings; + [SPEECH_KEY_STORE]: ISpeechKeyStore; [FILE_WATCHER_CLIENT]: FileWatcherClient; [FEATURE_FLAGS]: FeatureFlags; [AUTH_SIDE_EFFECTS]: IAuthSideEffects; diff --git a/docs/superpowers/specs/2026-07-02-quiet-backstop-narration-design.md b/docs/superpowers/specs/2026-07-02-quiet-backstop-narration-design.md new file mode 100644 index 0000000000..946639bb44 --- /dev/null +++ b/docs/superpowers/specs/2026-07-02-quiet-backstop-narration-design.md @@ -0,0 +1,59 @@ +# Quiet backstop narration when viewing the task + +Date: 2026-07-02 +Status: approved + +## Problem + +Every permission request fires the deterministic speech backstop +(`speakDeterministic(taskRunId, session, "needs_input")` in +`packages/core/src/sessions/sessionService.ts`). `shouldSpeak` in +`packages/ui/src/features/notifications/speechRouting.ts` lets every +`needs_input` line bypass focus routing (`if (kind === "needs_input") return +true`), and the queue treats them as priority lines that are never coalesced. +Result: consecutive permission prompts each speak "needs your input" while the +user is already looking at the chat approving them. + +## Design + +Distinguish the two speakers sharing the speech pipe and gate them +differently: + +- `source: "agent"` — lines from the agent's `speak` tool call. Keep current + behavior: `needs_input` always plays; `done`/`progress` follow + `spokenFocusMode`. +- `source: "backstop"` — deterministic lines ("finished", "needs your input") + fired by turn-complete and permission events. Never play when the focus + channel is `"suppress"` (app focused and the user is viewing that task). + Otherwise follow `spokenFocusMode` as today. + +`"suppress"` is per-task: a permission prompt on a task the user is *not* +viewing (app focused, different tab) still speaks — that is the multi-agent +case the voice exists for. + +## Changes + +1. `packages/ui/src/features/notifications/speechRouting.ts` — add + `SpeechSource = "agent" | "backstop"`; `shouldSpeak(kind, source, channel, + settings)`: backstop lines return `false` when `channel === "suppress"`; + agent `needs_input` keeps its unconditional pass. +2. `packages/ui/src/features/notifications/speechNotifier.ts` — add `source` + to `SpeakRequest`, pass through to `shouldSpeak`. +3. `packages/core/src/sessions/sessionService.ts` — add `source` to the + `enqueueSpeech` dep signature; `speakDeterministic` sends `"backstop"`, + the speak-tool handler sends `"agent"`. +4. `packages/ui/src/features/sessions/sessionServiceHost.ts` — type flows + through unchanged (verify). +5. `packages/ui/src/features/notifications/speechRouting.test.ts` — cover the + new matrix: backstop suppressed on `"suppress"`, backstop follows focus + mode on `"toast"`/`"native"`, agent `needs_input` unaffected. + +No settings UI change. No queue changes. + +## Rejected alternatives + +- Gate all `needs_input` by focus mode (no source flag): also mutes the + agent's intentional "Hey Jon" lines. +- Debounce/dedupe backstop lines in the queue: treats the symptom; focus + suppression already kills repeats once the user is present, and serial + prompts while away should each speak. diff --git a/packages/agent/src/adapters/claude/mcp/local-tools.test.ts b/packages/agent/src/adapters/claude/mcp/local-tools.test.ts index 606ff66c4f..062aede9fc 100644 --- a/packages/agent/src/adapters/claude/mcp/local-tools.test.ts +++ b/packages/agent/src/adapters/claude/mcp/local-tools.test.ts @@ -20,10 +20,29 @@ describe("createLocalToolsMcpServer", () => { } }); - it("returns undefined when no tool's gate passes (desktop run)", () => { - expect( - createLocalToolsMcpServer({ cwd: "/repo", token: "ghs_x" }, undefined), - ).toBeUndefined(); + it("exposes only the always-on tools on a desktop run (no cloud-only tools)", async () => { + const server = createLocalToolsMcpServer( + { cwd: "/repo", token: "ghs_x" }, + undefined, + ); + if (!server) { + throw new Error("expected the local-tools server to be registered"); + } + + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.instance.connect(serverTransport); + const client = new Client({ name: "test", version: "1.0.0" }); + await client.connect(clientTransport); + + const { tools } = await client.listTools(); + const names = tools.map((t) => t.name); + // `speak` is always on (narration works on desktop and cloud alike). + expect(names).toContain("speak"); + // Signed-git tools are cloud-only and must not leak into a desktop run. + expect(names).not.toContain("git_signed_commit"); + + await client.close(); }); it("exposes git_signed_commit over MCP in a cloud run with a token", async () => { diff --git a/packages/agent/src/adapters/claude/session/instructions.ts b/packages/agent/src/adapters/claude/session/instructions.ts index 0928da65da..3f79a9c7c4 100644 --- a/packages/agent/src/adapters/claude/session/instructions.ts +++ b/packages/agent/src/adapters/claude/session/instructions.ts @@ -43,5 +43,30 @@ Optimize for the fewest shell round trips. - Never rerun a command solely to reproduce output you already have. `; +const SPOKEN_NARRATION = ` +# Spoken Narration + +You have a \`speak\` tool (\`mcp__posthog-code-tools__speak\`) that says a short line out loud. The user is usually looking at another window, so this — not the transcript — is how they actually receive what you did. Answering only in text leaves them staring at a silent tab. + +**Hard rule, not a suggestion: never end a turn silently.** Every turn that answers a question, finishes a request, or blocks on the user MUST include a \`speak\` call. And call it BEFORE you write your final text reply — while you're still working — not after. Once you've written the answer the turn feels done and you'll just stop, dropping the \`speak\`; calling it first, mid-turn, is the only reliable ordering. + +Call \`speak\` with: +- \`kind: "needs_input"\` when you are blocked and need the user: a question, a decision, a confirmation, or an error only they can resolve. This is the most important case. +- \`kind: "done"\` when you finish the user's request. Say the actual RESULT, not just that you're done. If the request had a concrete answer or headline number, that answer IS the line: "ARR is about forty-three million dollars", "the login bug was a stale session cookie", "all tests pass and the PR is up". A bare "finished" wastes the moment — the app already signals the task is done, so give them the takeaway. +- \`kind: "progress"\` when you learn something the user would genuinely want to hear mid-task: a notable finding, a surprising number, or a meaningful new phase. Don't narrate routine steps or every file edit — but don't hoard interesting information either. If you'd mention it to a colleague looking over your shoulder, speak it. + +How to phrase the line: +- Say just the message — one short sentence. Lead with the substance ("ARR is about forty-three million dollars"), not preamble. +- Spell things that don't read aloud well: round and word out numbers ("forty-three million", not "$43,512,900"), skip symbols and long IDs. +- Do NOT prefix it with the task name or the user's name yourself. The app automatically prepends the current task ("PostHog Code task '…' —") so the user knows which agent is talking, and for \`needs_input\` lines it addresses the user by their real name. You don't know the user's name — leave that to the app. +- Specific, never generic. +- Be theatrical: use expressive audio tags in [square brackets] — [laughs], [sighs], [groans], [excited], [whispers], [clears throat] — 1-3 per line, matched to the moment. The system-voice fallback strips tags automatically, so they never hurt. +`; + export const APPENDED_INSTRUCTIONS = - BRANCH_NAMING + PULL_REQUEST_LINKS + PLAN_MODE + MCP_TOOLS + SHELL_EFFICIENCY; + BRANCH_NAMING + + PULL_REQUEST_LINKS + + PLAN_MODE + + MCP_TOOLS + + SHELL_EFFICIENCY + + SPOKEN_NARRATION; diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts index 1a65e20dbd..1c4d532134 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts @@ -90,11 +90,22 @@ describe("buildLocalToolsServer", () => { ).toBeNull(); }); - it("returns null when no tool's gate passes (desktop run)", () => { + it("exposes only the always-on tools on a desktop run (no cloud-only tools)", () => { process.env.GH_TOKEN = "ghs_test"; - expect( - buildLocalToolsServer({ cwd: "/repo" }, { environment: "local" }), - ).toBeNull(); + const server = buildLocalToolsServer( + { cwd: "/repo" }, + { environment: "local" }, + ); + + expect(server).not.toBeNull(); + const enabled = + server?.env.find((e) => e.name === "POSTHOG_LOCAL_TOOLS_ENABLED") + ?.value ?? ""; + const names = enabled.split(","); + // `speak` is always on (narration works on desktop and cloud alike). + expect(names).toContain("speak"); + // Signed-git tools are cloud-only and must not leak into a desktop run. + expect(names).not.toContain("git_signed_commit"); }); }); diff --git a/packages/agent/src/adapters/local-tools/index.ts b/packages/agent/src/adapters/local-tools/index.ts index 04302ae215..c2aab7139d 100644 --- a/packages/agent/src/adapters/local-tools/index.ts +++ b/packages/agent/src/adapters/local-tools/index.ts @@ -4,6 +4,7 @@ import { listReposTool } from "./tools/list-repos"; import { signedCommitTool } from "./tools/signed-commit"; import { signedMergeTool } from "./tools/signed-merge"; import { signedRewriteTool } from "./tools/signed-rewrite"; +import { speakTool } from "./tools/speak"; export { LOCAL_TOOLS_MCP_NAME, @@ -21,6 +22,7 @@ export const LOCAL_TOOLS: LocalTool[] = [ signedRewriteTool, listReposTool, cloneRepoTool, + speakTool, ]; /** Tools whose gate passes for the given context — the set to actually expose. */ diff --git a/packages/agent/src/adapters/local-tools/tools/speak.test.ts b/packages/agent/src/adapters/local-tools/tools/speak.test.ts new file mode 100644 index 0000000000..32ee140745 --- /dev/null +++ b/packages/agent/src/adapters/local-tools/tools/speak.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { enabledLocalTools } from "../index"; +import { SPEAK_TOOL_NAME, speakSchema, speakTool } from "./speak"; + +describe("speak tool", () => { + it.each([ + { name: "desktop", environment: "local" as const }, + { name: "cloud", environment: "cloud" as const }, + ])("is always exposed ($name)", ({ environment }) => { + const tools = enabledLocalTools({ cwd: "/repo" }, { environment }); + expect(tools.some((t) => t.name === SPEAK_TOOL_NAME)).toBe(true); + }); + + it("is exposed even with no gate meta", () => { + const tools = enabledLocalTools({ cwd: "/repo" }, undefined); + expect(tools.some((t) => t.name === SPEAK_TOOL_NAME)).toBe(true); + }); + + it("stays visible without ToolSearch (alwaysLoad)", () => { + expect(speakTool.alwaysLoad).toBe(true); + }); + + it("validates a well-formed line", () => { + expect( + speakSchema.text.safeParse("[excited] tests are green!").success, + ).toBe(true); + }); + + it("rejects empty text", () => { + expect(speakSchema.text.safeParse("").success).toBe(false); + }); + + it.each(["needs_input", "done", "progress"])("accepts kind %s", (kind) => { + expect(speakSchema.kind.safeParse(kind).success).toBe(true); + }); + + it("rejects an unknown kind", () => { + expect(speakSchema.kind.safeParse("chatter").success).toBe(false); + }); + + it("handler acknowledges without side effects", async () => { + const result = await speakTool.handler( + { cwd: "/repo" }, + { text: "hello", kind: "done" }, + ); + expect(result.isError).toBeUndefined(); + expect(result.content[0]).toMatchObject({ type: "text" }); + }); +}); diff --git a/packages/agent/src/adapters/local-tools/tools/speak.ts b/packages/agent/src/adapters/local-tools/tools/speak.ts new file mode 100644 index 0000000000..5559dc1609 --- /dev/null +++ b/packages/agent/src/adapters/local-tools/tools/speak.ts @@ -0,0 +1,63 @@ +import { z } from "zod"; +import { defineLocalTool, type LocalToolResult } from "../registry"; + +export const SPEAK_TOOL_NAME = "speak"; + +export const speakSchema = { + text: z + .string() + .min(1) + .max(400) + .describe( + "The message to say out loud — just the content, one short sentence. Do " + + "NOT add a task-name prefix or the user's name yourself; the app " + + "prepends the current task (\"PostHog Code task '…' —\") and, for " + + "needsUser lines, addresses the user by their real name automatically. " + + 'So say e.g. "moving on to search the database" or "I need your call ' + + 'on which branch to use". Use expressive audio tags in [square ' + + "brackets] like [laughs], [sighs], [excited]; they are stripped " + + "automatically by the system-voice fallback.", + ), + kind: z + .enum(["needs_input", "done", "progress"]) + .describe( + "Why you're speaking. 'needs_input': you're blocked and need the user " + + "(a question, decision, confirmation, or an error only they can " + + 'resolve) — highest priority, spoken as "Hey , …", never ' + + "dropped. 'done': you've finished the user's request. 'progress': " + + "you've moved to a meaningful new phase. The user controls which of " + + "these are spoken and may mute 'progress' entirely, so reserve it for " + + "genuine phase changes — never routine steps.", + ), +}; + +export const SPEAK_TOOL_DESCRIPTION = + "Say a short line out loud to the user via text-to-speech — how you hand " + + "them information while they look at another window. Lean toward using it. " + + "Call it when you are BLOCKED and need them (kind 'needs_input'); when you " + + "FINISH their request (kind 'done') — and there, say the actual RESULT or " + + "answer, not just that you're done; and when you learn something they'd want " + + "to hear mid-task (kind 'progress') — a notable finding or number. Don't " + + "narrate routine steps or every file edit, but don't hoard useful results " + + "either. Playback is best-effort and serialized across all running agents; " + + "this tool returns immediately and never blocks your work."; + +/** + * A no-op-on-the-agent-side narration tool. The handler runs inside the agent + * process (local subprocess or cloud sandbox) which cannot reach the user's + * speakers, so it just acknowledges. The desktop renderer observes the + * surfaced `tool_call` (carrying `text`/`needsUser` in its rawInput) and routes + * it to the speech queue, exactly like completion/permission notifications are + * pure side effects off the event stream. Always enabled — gating happens on + * the consumer side via the user's "spoken narration" setting. + */ +export const speakTool = defineLocalTool({ + name: SPEAK_TOOL_NAME, + description: SPEAK_TOOL_DESCRIPTION, + schema: speakSchema, + alwaysLoad: true, + isEnabled: () => true, + handler: async (): Promise => { + return { content: [{ type: "text", text: "ok" }] }; + }, +}); diff --git a/packages/core/src/sessions/acpNotifications.ts b/packages/core/src/sessions/acpNotifications.ts index 81e0cb25ab..b5649776ba 100644 --- a/packages/core/src/sessions/acpNotifications.ts +++ b/packages/core/src/sessions/acpNotifications.ts @@ -22,6 +22,11 @@ export const POSTHOG_NOTIFICATIONS = { PERMISSION_RESOLVED: "_posthog/permission_resolved", } as const; +// Qualified id of the agent's `speak` narration tool, as it appears on the +// surfaced tool_call (`_meta.claudeCode.toolName`). Mirrors the agent's +// LOCAL_TOOLS_MCP_NAME + tool name; kept here so core doesn't import @posthog/agent. +export const SPEAK_TOOL_QUALIFIED_NAME = "mcp__posthog-code-tools__speak"; + type PosthogNotification = (typeof POSTHOG_NOTIFICATIONS)[keyof typeof POSTHOG_NOTIFICATIONS]; diff --git a/packages/core/src/sessions/cloudTaskUpdateNotifications.test.ts b/packages/core/src/sessions/cloudTaskUpdateNotifications.test.ts index 75a61e4f83..3bbc6d0e9c 100644 --- a/packages/core/src/sessions/cloudTaskUpdateNotifications.test.ts +++ b/packages/core/src/sessions/cloudTaskUpdateNotifications.test.ts @@ -125,6 +125,7 @@ function createHarness() { let onUpdate: ((update: CloudTaskUpdatePayload) => void) | undefined; const notifyPromptComplete = vi.fn(); const notifyPermissionRequest = vi.fn(); + const enqueueSpeech = vi.fn(); const markActivity = vi.fn(); const noopLog = { info: vi.fn(), @@ -138,6 +139,7 @@ function createHarness() { log: noopLog, notifyPromptComplete, notifyPermissionRequest, + enqueueSpeech, taskViewedApi: { markActivity }, getPersistedConfigOptions: () => undefined, setPersistedConfigOptions: vi.fn(), @@ -177,6 +179,7 @@ function createHarness() { sendUpdate: (update: CloudTaskUpdatePayload) => onUpdate?.(update), notifyPromptComplete, notifyPermissionRequest, + enqueueSpeech, markActivity, }; } @@ -253,6 +256,9 @@ describe("cloud task update notifications", () => { TASK_ID, 45_000, ); + expect(harness.enqueueSpeech).toHaveBeenCalledWith( + expect.objectContaining({ kind: "done", source: "backstop" }), + ); expect(harness.markActivity).toHaveBeenCalledTimes(1); }); @@ -269,6 +275,9 @@ describe("cloud task update notifications", () => { snapshot(); expect(harness.notifyPermissionRequest).toHaveBeenCalledTimes(1); + expect(harness.enqueueSpeech).toHaveBeenCalledWith( + expect.objectContaining({ kind: "needs_input", source: "backstop" }), + ); snapshot(); snapshot(); diff --git a/packages/core/src/sessions/sessionEventBatching.test.ts b/packages/core/src/sessions/sessionEventBatching.test.ts index e9af1af617..c1d47f2c64 100644 --- a/packages/core/src/sessions/sessionEventBatching.test.ts +++ b/packages/core/src/sessions/sessionEventBatching.test.ts @@ -87,6 +87,7 @@ function createHarness() { log: noopLog, notifyPromptComplete, notifyPermissionRequest: vi.fn(), + enqueueSpeech: vi.fn(), taskViewedApi: { markActivity: vi.fn() }, getPersistedConfigOptions: () => undefined, setPersistedConfigOptions: vi.fn(), diff --git a/packages/core/src/sessions/sessionService.ts b/packages/core/src/sessions/sessionService.ts index 3c4f1b9559..fba0b3ffb6 100644 --- a/packages/core/src/sessions/sessionService.ts +++ b/packages/core/src/sessions/sessionService.ts @@ -40,7 +40,12 @@ import { isTerminalStatus, type Task, } from "@posthog/shared/domain-types"; -import { isNotification, POSTHOG_NOTIFICATIONS } from "./acpNotifications"; +import type { SpeechKind, SpeechSource } from "../speech/identifiers"; +import { + isNotification, + POSTHOG_NOTIFICATIONS, + SPEAK_TOOL_QUALIFIED_NAME, +} from "./acpNotifications"; import { createAppendOnlyTracker } from "./appendOnlyTracker"; import type { CloudArtifactClient, @@ -279,6 +284,14 @@ export interface SessionServiceDeps { buildPermissionToolMetadata: (...args: any[]) => any; notifyPermissionRequest: (...args: any[]) => any; notifyPromptComplete: (...args: any[]) => any; + enqueueSpeech: (request: { + text: string; + taskTitle: string; + taskId?: string; + kind: SpeechKind; + source: SpeechSource; + addressByName?: boolean; + }) => void; getIsOnline: () => boolean; fetchAuthState: () => Promise; getAuthenticatedClient: () => Promise; @@ -585,6 +598,26 @@ export class SessionService { private cloudRunIdleTracker: CloudRunIdleTracker; private nextCloudTaskWatchToken = 0; private supersededRunIds = new Set(); + // Spoken narration: the `speak` tool's { text, kind } args stream in across + // multiple tool_call_updates (partial input_json_delta), so early events carry + // a truncated text like "The quick b". Track speak tool calls by id and + // accumulate their latest args; enqueue once the call reaches a terminal + // status (full text), then delete the entry — which also dedupes re-fires. + // A `null` value means "identified as speak, args not streamed in yet". + // Keyed by taskRunId first so a session teardown can drop any of its + // still-streaming speak calls (see unsubscribeFromChannel); the inner map is + // keyed by toolCallId. + private speakCalls = new Map< + string, + Map + >(); + // When the agent last narrated `done`/`needs_input` per task run (event ts). + // The deterministic completion/needs-input backstops compare this against the + // turn's start so they don't double up on a moment the agent already voiced. + private agentSpokeAt = new Map< + string, + { needs_input: number; done: number } + >(); private subscriptions = new Map< string, { @@ -1697,6 +1730,10 @@ export class SessionService { subscription?.permission?.unsubscribe(); this.subscriptions.delete(taskRunId); this.liveTurnContent.delete(taskRunId); + this.agentSpokeAt.delete(taskRunId); + // Drop any speak calls still mid-stream for this run (never reached a + // terminal status, so they were never enqueued or deleted above). + this.speakCalls.delete(taskRunId); } /** @@ -1736,6 +1773,8 @@ export class SessionService { this.sessionLastUsedAt.clear(); this.cloudPermissionRequestIds.clear(); this.liveTurnContent.clear(); + this.speakCalls.clear(); + this.agentSpokeAt.clear(); this.cloudLogGapReconciler.clear(); this.dispatchingCloudQueues.clear(); this.scheduledCloudQueueFlushes.clear(); @@ -1875,6 +1914,7 @@ export class SessionService { session.taskId, turnStartedAtTs ? acpMsg.ts - turnStartedAtTs : undefined, ); + this.speakDeterministic(taskRunId, session, "done"); } this.d.taskViewedApi.markActivity(session.taskId); this.finalizeTurnContent(taskRunId, "turn_complete", acpMsg.ts); @@ -1943,6 +1983,36 @@ export class SessionService { } } + /** + * Deterministic backstop for the two moments the user must not miss. Fired + * from the turn-complete and permission events (which happen every time), + * unless the agent already narrated that same moment this turn via the speak + * tool — in which case its expressive line stands. Routes through the same + * speech channel (focus + settings gating + serialized queue). + */ + private speakDeterministic( + taskRunId: string, + session: { + taskTitle: string; + taskId: string; + promptStartedAt: number | null; + }, + kind: "done" | "needs_input", + ): void { + const turnStart = session.promptStartedAt ?? 0; + const spokeAt = this.agentSpokeAt.get(taskRunId)?.[kind] ?? 0; + if (turnStart > 0 && spokeAt >= turnStart) return; // agent already voiced it + // Deterministic backstop stays plain — no "Hey ," greeting. + this.d.enqueueSpeech({ + text: kind === "done" ? "finished" : "needs your input", + taskTitle: session.taskTitle, + taskId: session.taskId, + kind, + source: "backstop", + addressByName: false, + }); + } + private handleSessionEvent(taskRunId: string, acpMsg: AcpMessage): void { const session = this.d.store.getSessions()[taskRunId]; if (!session) return; @@ -2000,6 +2070,9 @@ export class SessionService { session.taskId, turnStartedAtTs ? acpMsg.ts - turnStartedAtTs : undefined, ); + if (stopReason === "end_turn") { + this.speakDeterministic(taskRunId, session, "done"); + } } this.d.taskViewedApi.markActivity(session.taskId); @@ -2027,6 +2100,80 @@ export class SessionService { this.d.log.info("Session config options updated", { taskRunId }); } + // Spoken narration: the agent's `speak` tool call surfaces here. The + // initial tool_call names the tool but arrives with empty args; the + // assembled { text, kind } stream in on later tool_call_updates with the + // same toolCallId. So we remember speak toolCallIds, accumulate the latest + // args, and enqueue once the call completes (with the full text). + if ( + params?.update?.sessionUpdate === "tool_call" || + params?.update?.sessionUpdate === "tool_call_update" + ) { + const update = params.update as { + toolCallId?: string; + status?: string; + _meta?: { + claudeCode?: { toolName?: string; parentToolCallId?: string }; + }; + rawInput?: { text?: unknown; kind?: unknown }; + }; + const id = update.toolCallId; + if (id) { + const speakCalls = this.speakCalls.get(taskRunId); + // Only the top-level agent narrates. A `speak` from a sub-agent + // (spawned via the Task tool) carries a parentToolCallId; ignoring + // those prevents several sub-agents talking over each other. + if ( + update._meta?.claudeCode?.toolName === SPEAK_TOOL_QUALIFIED_NAME && + !update._meta.claudeCode.parentToolCallId && + !speakCalls?.has(id) + ) { + const calls = speakCalls ?? new Map(); + calls.set(id, null); + this.speakCalls.set(taskRunId, calls); + } + const calls = this.speakCalls.get(taskRunId); + if (calls?.has(id)) { + // Accumulate the latest args — text grows across streamed updates. + const text = update.rawInput?.text; + if (typeof text === "string" && text.trim().length > 0) { + const rawKind = update.rawInput?.kind; + const kind: SpeechKind = + rawKind === "needs_input" || + rawKind === "done" || + rawKind === "progress" + ? rawKind + : "progress"; + calls.set(id, { text, kind }); + } + // Speak only once the call is complete (full text). Deleting the + // entry both frees it and dedupes any later terminal event. + const pending = calls.get(id); + if (isTerminalStatus(update.status) && pending) { + calls.delete(id); + if (calls.size === 0) this.speakCalls.delete(taskRunId); + if (pending.kind !== "progress") { + const spoke = this.agentSpokeAt.get(taskRunId) ?? { + needs_input: 0, + done: 0, + }; + spoke[pending.kind] = acpMsg.ts; + this.agentSpokeAt.set(taskRunId, spoke); + } + this.d.enqueueSpeech({ + text: pending.text, + taskTitle: session.taskTitle, + taskId: session.taskId, + kind: pending.kind, + source: "agent", + // Agent-authored line: allowed to address the user by name. + addressByName: true, + }); + } + } + } + } + // Handle context usage updates if (params?.update?.sessionUpdate === "usage_update") { const update = params.update as { @@ -2146,6 +2293,7 @@ export class SessionService { this.d.store.setPendingPermissions(taskRunId, newPermissions); this.d.taskViewedApi.markActivity(session.taskId); this.d.notifyPermissionRequest(session.taskTitle, session.taskId); + this.speakDeterministic(taskRunId, session, "needs_input"); } private handleCloudPermissionRequest( @@ -2194,6 +2342,7 @@ export class SessionService { this.d.store.setPendingPermissions(taskRunId, newPermissions); this.d.taskViewedApi.markActivity(session.taskId); this.d.notifyPermissionRequest(session.taskTitle, session.taskId); + this.speakDeterministic(taskRunId, session, "needs_input"); } private surfacePersistedPendingPermissions( diff --git a/packages/core/src/speech/composeUtterance.test.ts b/packages/core/src/speech/composeUtterance.test.ts new file mode 100644 index 0000000000..3d9a9e3645 --- /dev/null +++ b/packages/core/src/speech/composeUtterance.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import { composeUtterance, firstNameFromLabel } from "./composeUtterance"; + +describe("firstNameFromLabel", () => { + it.each([ + { label: "Jon McCallum", expected: "Jon" }, + { label: " Jon ", expected: "Jon" }, + { label: "jonathon@posthog.com", expected: undefined }, + { label: "", expected: undefined }, + { label: undefined, expected: undefined }, + ])("extracts $expected from $label", ({ label, expected }) => { + expect(firstNameFromLabel(label)).toBe(expected); + }); +}); + +describe("composeUtterance", () => { + it("prefixes the task name", () => { + expect( + composeUtterance({ text: "moving on to search", taskTitle: "fix login" }), + ).toBe("PostHog Code task 'fix login' — moving on to search"); + }); + + it("addresses the user by name for agent needs-user lines", () => { + expect( + composeUtterance({ + text: "I need your call on which branch", + taskTitle: "search index", + needsUser: true, + addressByName: true, + firstName: "Jon", + }), + ).toBe( + "PostHog Code task 'search index' — Hey Jon, I need your call on which branch", + ); + }); + + it("omits the greeting on the deterministic backstop (no addressByName)", () => { + expect( + composeUtterance({ + text: "needs your input", + taskTitle: "deploy", + needsUser: true, + firstName: "Jon", + }), + ).toBe("PostHog Code task 'deploy' — needs your input"); + }); + + it("normalizes an agent-added greeting into the real name", () => { + expect( + composeUtterance({ + text: "Hey, blocked on the API key", + taskTitle: "deploy", + needsUser: true, + addressByName: true, + firstName: "Jon", + }), + ).toBe("PostHog Code task 'deploy' — Hey Jon, blocked on the API key"); + }); + + it("does not add a name when it is unknown", () => { + expect( + composeUtterance({ + text: "I need your input", + taskTitle: "deploy", + needsUser: true, + }), + ).toBe("PostHog Code task 'deploy' — I need your input"); + }); + + it("does not double-prefix when the agent already added one", () => { + expect( + composeUtterance({ + text: "PostHog Code task 'x' — already prefixed", + taskTitle: "y", + }), + ).toBe("PostHog Code task 'x' — already prefixed"); + }); + + it("does not double-prefix even when addressing by name", () => { + expect( + composeUtterance({ + text: "PostHog Code task 'x' — already prefixed", + taskTitle: "y", + needsUser: true, + addressByName: true, + firstName: "Jon", + }), + ).toBe("PostHog Code task 'x' — already prefixed"); + }); + + it("truncates a long task title", () => { + const long = "a".repeat(60); + const out = composeUtterance({ text: "hi", taskTitle: long }); + expect(out).toContain("…"); + expect(out.length).toBeLessThan(`PostHog Code task '${long}' — hi`.length); + }); + + it("falls back to the body when there is no task title", () => { + expect(composeUtterance({ text: "hello", taskTitle: "" })).toBe("hello"); + }); +}); diff --git a/packages/core/src/speech/composeUtterance.ts b/packages/core/src/speech/composeUtterance.ts new file mode 100644 index 0000000000..175c45cb07 --- /dev/null +++ b/packages/core/src/speech/composeUtterance.ts @@ -0,0 +1,61 @@ +const TASK_TITLE_MAX = 40; + +export interface ComposeInput { + text: string; + taskTitle: string; + needsUser?: boolean; + firstName?: string; + /** Prepend "Hey ," — only for agent-authored lines, not the backstop. */ + addressByName?: boolean; +} + +/** + * Extract a first name from a display label. Returns undefined for empty labels + * or bare emails (we don't want "Hey jon@posthog.com"). + */ +export function firstNameFromLabel( + label: string | undefined, +): string | undefined { + if (!label) return undefined; + const trimmed = label.trim(); + if (!trimmed || trimmed.includes("@")) return undefined; + const first = trimmed.split(/\s+/)[0]; + return first || undefined; +} + +/** + * Build the spoken line from the agent's message body. The agent supplies only + * the content; the app owns the task-name prefix (so the user knows which of + * several parallel agents is talking) and, for needs-user lines, addresses the + * user by their real name. Idempotent against an agent that already added a + * prefix or a "Hey" greeting. + */ +export function composeUtterance({ + text, + taskTitle, + needsUser, + firstName, + addressByName, +}: ComposeInput): string { + let body = text.trim(); + + // Agent already prefixed with a task reference — return it verbatim. Checked + // before any greeting logic so the guard stays order-independent: injecting + // "Hey ," first would push the prefix past the start of the string and + // defeat this test, producing a double prefix. + if (/^\s*posthog code task/i.test(body)) return body; + + if (needsUser && addressByName && firstName) { + // Normalize any leading greeting the agent added, then address by real name. + body = body.replace(/^\s*hey\b[\s,]*/i, "").trimStart(); + body = `Hey ${firstName}, ${body}`; + } + + const title = taskTitle.trim(); + if (!title) return body; + const shortTitle = + title.length > TASK_TITLE_MAX + ? `${title.slice(0, TASK_TITLE_MAX)}…` + : title; + return `PostHog Code task '${shortTitle}' — ${body}`; +} diff --git a/packages/core/src/speech/identifiers.ts b/packages/core/src/speech/identifiers.ts new file mode 100644 index 0000000000..20ca3d74c0 --- /dev/null +++ b/packages/core/src/speech/identifiers.ts @@ -0,0 +1,48 @@ +/** Why the agent is speaking; drives priority, greeting, and per-kind gating. */ +export type SpeechKind = "needs_input" | "done" | "progress"; + +/** + * Who authored the line: the agent's intentional `speak` tool call, or the + * deterministic backstop fired by turn-complete / permission events. Backstop + * lines are suppressed while the user is viewing the task; agent lines are not. + */ +export type SpeechSource = "agent" | "backstop"; + +/** A narration request as it arrives from the agent's `speak` tool call. */ +export interface SpeechRequest { + /** The message body the agent produced (no task prefix, no user name). */ + text: string; + /** Canonical task title, prepended by the app so the user knows who's talking. */ + taskTitle: string; + taskId?: string; + /** True when the line is a request for the user; prioritized and never dropped. */ + needsUser?: boolean; + /** Prepend "Hey ," — only for agent-authored lines, not the backstop. */ + addressByName?: boolean; +} + +/** Serializes agent narration into one-at-a-time speech. */ +export interface ISpeechQueue { + enqueue(request: SpeechRequest): void; +} + +export const SPEECH_QUEUE_SERVICE = Symbol.for("posthog.speech.queue"); + +/** Reads the user's spoken-narration settings at speak time. */ +export interface SpeechSettingsProvider { + get(): { enabled: boolean; voiceId?: string }; +} + +export const SPEECH_SETTINGS_PROVIDER = Symbol.for( + "posthog.speech.settingsProvider", +); + +/** Supplies the signed-in user's display name for "Hey " lines. */ +export interface UserNameProvider { + /** The user's first name, or undefined when unknown (e.g. only an email). */ + getFirstName(): string | undefined; +} + +export const SPEECH_USER_NAME_PROVIDER = Symbol.for( + "posthog.speech.userNameProvider", +); diff --git a/packages/core/src/speech/speech.module.ts b/packages/core/src/speech/speech.module.ts new file mode 100644 index 0000000000..be27b708ab --- /dev/null +++ b/packages/core/src/speech/speech.module.ts @@ -0,0 +1,7 @@ +import { ContainerModule } from "inversify"; +import { SPEECH_QUEUE_SERVICE } from "./identifiers"; +import { SpeechQueueService } from "./speech"; + +export const speechCoreModule = new ContainerModule(({ bind }) => { + bind(SPEECH_QUEUE_SERVICE).to(SpeechQueueService).inSingletonScope(); +}); diff --git a/packages/core/src/speech/speech.test.ts b/packages/core/src/speech/speech.test.ts new file mode 100644 index 0000000000..ec3e131ac5 --- /dev/null +++ b/packages/core/src/speech/speech.test.ts @@ -0,0 +1,159 @@ +import type { ISpeech } from "@posthog/platform/speech"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { SpeechSettingsProvider, UserNameProvider } from "./identifiers"; +import { SpeechQueueService } from "./speech"; + +/** A fake ISpeech whose `speak` resolves only when the test releases it. */ +class ControllableSpeech implements ISpeech { + spoken: string[] = []; + private resolvers: Array<() => void> = []; + + isSupported(): boolean { + return true; + } + + speak(text: string): Promise { + this.spoken.push(text); + return new Promise((resolve) => { + this.resolvers.push(resolve); + }); + } + + stop(): void {} + + /** Finish the oldest in-flight utterance. */ + finishOne(): void { + const resolve = this.resolvers.shift(); + resolve?.(); + } + + get inFlight(): number { + return this.resolvers.length; + } +} + +const noopLogger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + scope: () => noopLogger, +}; + +function make(settings: Partial<{ enabled: boolean; voiceId?: string }> = {}) { + const speech = new ControllableSpeech(); + const settingsProvider: SpeechSettingsProvider = { + get: () => ({ enabled: true, ...settings }), + }; + const userName: UserNameProvider = { getFirstName: () => "Jon" }; + const service = new SpeechQueueService( + speech, + settingsProvider, + userName, + noopLogger as never, + ); + return { speech, service }; +} + +// let the queued microtasks (drain loop) run +const tick = () => new Promise((r) => setTimeout(r, 0)); + +describe("SpeechQueueService", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("speaks one utterance at a time", async () => { + const { speech, service } = make(); + service.enqueue({ text: "one", taskTitle: "a", taskId: "1" }); + service.enqueue({ text: "two", taskTitle: "b", taskId: "2" }); + await tick(); + + expect(speech.inFlight).toBe(1); + expect(speech.spoken).toEqual(["PostHog Code task 'a' — one"]); + + speech.finishOne(); + await tick(); + expect(speech.spoken).toEqual([ + "PostHog Code task 'a' — one", + "PostHog Code task 'b' — two", + ]); + }); + + it("stays silent when narration is disabled", async () => { + const { speech, service } = make({ enabled: false }); + service.enqueue({ text: "one", taskTitle: "a", taskId: "1" }); + await tick(); + expect(speech.spoken).toEqual([]); + }); + + it("coalesces a newer line for the same queued task", async () => { + const { speech, service } = make(); + // First starts playing immediately (in-flight, not in queue). + service.enqueue({ text: "start", taskTitle: "t", taskId: "1" }); + await tick(); + // These two queue behind it; second replaces the first for task 2. + service.enqueue({ text: "stale", taskTitle: "t2", taskId: "2" }); + service.enqueue({ text: "fresh", taskTitle: "t2", taskId: "2" }); + await tick(); + + speech.finishOne(); // finish "start" + await tick(); + speech.finishOne(); // finish the task-2 line + await tick(); + + expect(speech.spoken).toEqual([ + "PostHog Code task 't' — start", + "PostHog Code task 't2' — fresh", + ]); + expect(speech.spoken).not.toContain("PostHog Code task 't2' — stale"); + }); + + it("prioritizes needs-user lines ahead of routine narration", async () => { + const { speech, service } = make(); + service.enqueue({ text: "playing", taskTitle: "t", taskId: "1" }); + await tick(); + service.enqueue({ text: "routine", taskTitle: "t2", taskId: "2" }); + service.enqueue({ + text: "urgent", + taskTitle: "t3", + taskId: "3", + needsUser: true, + addressByName: true, + }); + await tick(); + + speech.finishOne(); // finish "playing" + await tick(); + expect(speech.spoken[1]).toBe("PostHog Code task 't3' — Hey Jon, urgent"); + }); + + it("drops oldest routine lines when the queue is backed up but keeps priority", async () => { + const { speech, service } = make(); + service.enqueue({ text: "playing", taskTitle: "t", taskId: "0" }); + await tick(); + // Queue five routine lines behind the in-flight one (cap is 3). + for (let i = 1; i <= 5; i++) { + service.enqueue({ text: `r${i}`, taskTitle: `t${i}`, taskId: String(i) }); + } + service.enqueue({ + text: "urgent", + taskTitle: "tu", + taskId: "u", + needsUser: true, + addressByName: true, + }); + await tick(); + + // Drain everything. + for (let i = 0; i < 10 && speech.inFlight > 0; i++) { + speech.finishOne(); + await tick(); + } + + expect(speech.spoken).toContain("PostHog Code task 'tu' — Hey Jon, urgent"); + // Some routine lines were dropped (didn't speak all five). + const routineSpoken = speech.spoken.filter((s) => /— r\d$/.test(s)).length; + expect(routineSpoken).toBeLessThan(5); + }); +}); diff --git a/packages/core/src/speech/speech.ts b/packages/core/src/speech/speech.ts new file mode 100644 index 0000000000..467062ff29 --- /dev/null +++ b/packages/core/src/speech/speech.ts @@ -0,0 +1,135 @@ +import { + ROOT_LOGGER, + type RootLogger, + type ScopedLogger, +} from "@posthog/di/logger"; +import { type ISpeech, SPEECH_SERVICE } from "@posthog/platform/speech"; +import { inject, injectable } from "inversify"; +import { composeUtterance } from "./composeUtterance"; +import { + type ISpeechQueue, + SPEECH_QUEUE_SERVICE, + SPEECH_SETTINGS_PROVIDER, + SPEECH_USER_NAME_PROVIDER, + type SpeechRequest, + type SpeechSettingsProvider, + type UserNameProvider, +} from "./identifiers"; + +export { SPEECH_QUEUE_SERVICE }; + +/** Max queued utterances (excluding the one currently playing) before we drop. */ +const MAX_QUEUE = 3; + +interface QueuedUtterance { + text: string; + taskId?: string; + priority: boolean; +} + +/** + * Serializes agent narration so several parallel sessions never talk over each + * other. One utterance plays at a time; a newer line for a task supersedes its + * still-queued predecessor; needs-user lines jump the queue and are never + * dropped; and the queue is capped so a burst can't back up minutes of speech. + * + * This is portable orchestration (queue/dedupe/coalesce), so it lives in core + * and depends only on the host-neutral ISpeech capability plus injected + * settings/name providers — no tRPC, Node, or Electron. + */ +@injectable() +export class SpeechQueueService implements ISpeechQueue { + private readonly logger: ScopedLogger; + private readonly queue: QueuedUtterance[] = []; + private speaking = false; + + constructor( + @inject(SPEECH_SERVICE) + private readonly speech: ISpeech, + @inject(SPEECH_SETTINGS_PROVIDER) + private readonly settings: SpeechSettingsProvider, + @inject(SPEECH_USER_NAME_PROVIDER) + private readonly userName: UserNameProvider, + @inject(ROOT_LOGGER) + rootLogger: RootLogger, + ) { + this.logger = rootLogger.scope("speech"); + } + + enqueue(request: SpeechRequest): void { + const { enabled } = this.settings.get(); + if (!enabled) return; + + const text = composeUtterance({ + text: request.text, + taskTitle: request.taskTitle, + needsUser: request.needsUser, + addressByName: request.addressByName, + firstName: this.userName.getFirstName(), + }); + if (!text) return; + + const utterance: QueuedUtterance = { + text, + taskId: request.taskId, + priority: request.needsUser === true, + }; + + // Coalesce: a newer non-priority line for the same task replaces its + // still-queued predecessor so we speak the freshest narration, not a stale + // one. Priority lines are kept distinct (each needs answering). + if (!utterance.priority && utterance.taskId) { + const idx = this.queue.findIndex( + (q) => !q.priority && q.taskId === utterance.taskId, + ); + if (idx !== -1) { + this.queue[idx] = utterance; + void this.drain(); + return; + } + } + + // Priority lines jump ahead of routine narration. + if (utterance.priority) { + const firstNonPriority = this.queue.findIndex((q) => !q.priority); + if (firstNonPriority === -1) this.queue.push(utterance); + else this.queue.splice(firstNonPriority, 0, utterance); + } else { + this.queue.push(utterance); + } + + this.enforceCap(); + void this.drain(); + } + + private enforceCap(): void { + while (this.queue.length > MAX_QUEUE) { + const idx = this.queue.findIndex((q) => !q.priority); + if (idx === -1) break; // all priority — never drop + const [dropped] = this.queue.splice(idx, 1); + this.logger.warn("Dropped narration; queue backed up", { + text: dropped.text, + }); + } + } + + private async drain(): Promise { + if (this.speaking) return; + this.speaking = true; + try { + while (this.queue.length > 0) { + const next = this.queue.shift(); + if (!next) break; + const { voiceId } = this.settings.get(); + try { + await this.speech.speak(next.text, { voiceId }); + } catch (err) { + // Best-effort — a failed line must not stall the queue. + this.logger.warn("speak failed", { err }); + } + } + } finally { + this.speaking = false; + } + } +} diff --git a/packages/host-router/src/router.ts b/packages/host-router/src/router.ts index 2c0ee46c41..71f787c4c2 100644 --- a/packages/host-router/src/router.ts +++ b/packages/host-router/src/router.ts @@ -41,6 +41,7 @@ import { shellRouter } from "./routers/shell.router"; import { skillsRouter } from "./routers/skills.router"; import { slackIntegrationRouter } from "./routers/slack-integration.router"; import { sleepRouter } from "./routers/sleep.router"; +import { speechRouter } from "./routers/speech.router"; import { suspensionRouter } from "./routers/suspension.router"; import { uiRouter } from "./routers/ui.router"; import { updatesRouter } from "./routers/updates.router"; @@ -87,6 +88,7 @@ export const hostRouter = router({ provisioning: provisioningRouter, secureStore: secureStoreRouter, shell: shellRouter, + speech: speechRouter, skills: skillsRouter, slackIntegration: slackIntegrationRouter, sleep: sleepRouter, diff --git a/packages/host-router/src/routers/speech.router.ts b/packages/host-router/src/routers/speech.router.ts new file mode 100644 index 0000000000..8df43e0db9 --- /dev/null +++ b/packages/host-router/src/routers/speech.router.ts @@ -0,0 +1,23 @@ +import { publicProcedure, router } from "@posthog/host-trpc/trpc"; +import { + type ISpeechSynthesizer, + SPEECH_SYNTHESIZER_SERVICE, +} from "@posthog/workspace-server/services/speech/identifiers"; +import { z } from "zod"; + +export const speechRouter = router({ + // Synthesizes MP3 bytes (key stays in the host); the renderer plays them. + // Returns null when no key is set or synthesis fails. + synthesize: publicProcedure + .input( + z.object({ + text: z.string().min(1).max(2000), + voiceId: z.string().optional(), + }), + ) + .query(({ ctx, input }) => + ctx.container + .get(SPEECH_SYNTHESIZER_SERVICE) + .synthesize(input.text, input.voiceId), + ), +}); diff --git a/packages/platform/package.json b/packages/platform/package.json index 326da8093b..5b1ede9f13 100644 --- a/packages/platform/package.json +++ b/packages/platform/package.json @@ -56,6 +56,10 @@ "types": "./dist/notifier.d.ts", "import": "./dist/notifier.js" }, + "./speech": { + "types": "./dist/speech.d.ts", + "import": "./dist/speech.js" + }, "./notifications": { "types": "./dist/notifications.d.ts", "import": "./dist/notifications.js" diff --git a/packages/platform/src/speech.ts b/packages/platform/src/speech.ts new file mode 100644 index 0000000000..8e9b8e6340 --- /dev/null +++ b/packages/platform/src/speech.ts @@ -0,0 +1,22 @@ +export interface SpeakOptions { + /** ElevenLabs voice id to synthesize with. Adapter picks a default when omitted. */ + voiceId?: string; +} + +/** + * Host capability for speaking a line out loud (text-to-speech). Best-effort: + * `speak` must never throw and should resolve when playback finishes (or is + * skipped), so a caller can serialize utterances one at a time. The Electron + * adapter synthesizes expressive audio via ElevenLabs when a key is configured + * and falls back to the system voice otherwise. + */ +export interface ISpeech { + /** True when any playback path is available (system voice or configured key). */ + isSupported(): boolean; + /** Speak `text`; resolves when playback ends. Never rejects. */ + speak(text: string, opts?: SpeakOptions): Promise; + /** Stop any in-progress playback immediately. */ + stop(): void; +} + +export const SPEECH_SERVICE = Symbol.for("posthog.platform.speech"); diff --git a/packages/platform/tsup.config.ts b/packages/platform/tsup.config.ts index 92701b8c61..ba5543d15e 100644 --- a/packages/platform/tsup.config.ts +++ b/packages/platform/tsup.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ "src/power-manager.ts", "src/updater.ts", "src/notifier.ts", + "src/speech.ts", "src/notifications.ts", "src/context-menu.ts", "src/bundled-resources.ts", diff --git a/packages/ui/src/features/notifications/identifiers.ts b/packages/ui/src/features/notifications/identifiers.ts index de7b7d6258..5eb322a448 100644 --- a/packages/ui/src/features/notifications/identifiers.ts +++ b/packages/ui/src/features/notifications/identifiers.ts @@ -32,3 +32,14 @@ export interface IActiveView { export const ACTIVE_VIEW_PROVIDER = Symbol.for( "posthog.ui.notifications.activeView", ); + +// Reads the user's spoken-notification settings at speak time (host binds it to +// the settings store). Separate from INotificationSettings so the speech +// channel and the toast/native channel evolve independently. +export interface ISpeechNotifySettings { + get(): import("./speechRouting").SpeechGateSettings; +} + +export const SPEECH_NOTIFY_SETTINGS = Symbol.for( + "posthog.ui.speech.notifySettings", +); diff --git a/packages/ui/src/features/notifications/notifications.module.ts b/packages/ui/src/features/notifications/notifications.module.ts index f70a558d49..52c629c437 100644 --- a/packages/ui/src/features/notifications/notifications.module.ts +++ b/packages/ui/src/features/notifications/notifications.module.ts @@ -1,6 +1,8 @@ import { ContainerModule } from "inversify"; import { NotificationBus } from "./notifications"; +import { SpeechNotifier } from "./speechNotifier"; export const notificationsUiModule = new ContainerModule(({ bind }) => { bind(NotificationBus).toSelf().inSingletonScope(); + bind(SpeechNotifier).toSelf().inSingletonScope(); }); diff --git a/packages/ui/src/features/notifications/speechNotifier.ts b/packages/ui/src/features/notifications/speechNotifier.ts new file mode 100644 index 0000000000..b1b82d8f66 --- /dev/null +++ b/packages/ui/src/features/notifications/speechNotifier.ts @@ -0,0 +1,71 @@ +import { + type ISpeechQueue, + SPEECH_QUEUE_SERVICE, +} from "@posthog/core/speech/identifiers"; +import type { NotificationTarget } from "@posthog/platform/notifications"; +import { inject, injectable } from "inversify"; +import { + ACTIVE_VIEW_PROVIDER, + type IActiveView, + type ISpeechNotifySettings, + SPEECH_NOTIFY_SETTINGS, +} from "./identifiers"; +import { routeNotification } from "./routeNotification"; +import { + type SpeechKind, + type SpeechSource, + shouldSpeak, +} from "./speechRouting"; + +export interface SpeakRequest { + text: string; + kind: SpeechKind; + /** Agent `speak` tool call, or the deterministic turn/permission backstop. */ + source: SpeechSource; + taskTitle: string; + taskId?: string; + /** Address the user by name ("Hey ,") — agent lines only, not backstop. */ + addressByName?: boolean; +} + +/** + * The speech channel's decision point: applies the user's spoken-notification + * settings and focus routing (reusing routeNotification, so "quiet for the task + * I'm watching" comes for free), then hands the surviving line to the core + * SpeechQueueService for serialization. Mirrors NotificationBus but for voice. + */ +@injectable() +export class SpeechNotifier { + constructor( + @inject(ACTIVE_VIEW_PROVIDER) + private readonly view: IActiveView, + @inject(SPEECH_NOTIFY_SETTINGS) + private readonly settings: ISpeechNotifySettings, + @inject(SPEECH_QUEUE_SERVICE) + private readonly queue: ISpeechQueue, + ) {} + + speak(request: SpeakRequest): void { + const target: NotificationTarget | undefined = request.taskId + ? { kind: "task", taskId: request.taskId } + : undefined; + const channel = routeNotification({ + appFocused: this.view.hasFocus(), + viewingTarget: this.view.getActiveTarget(), + notificationTarget: target, + }); + + if ( + !shouldSpeak(request.kind, request.source, channel, this.settings.get()) + ) + return; + + this.queue.enqueue({ + text: request.text, + taskTitle: request.taskTitle, + taskId: request.taskId, + needsUser: request.kind === "needs_input", + addressByName: request.addressByName, + }); + } +} diff --git a/packages/ui/src/features/notifications/speechRouting.test.ts b/packages/ui/src/features/notifications/speechRouting.test.ts new file mode 100644 index 0000000000..61684fc94a --- /dev/null +++ b/packages/ui/src/features/notifications/speechRouting.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; +import type { NotificationChannel } from "./routeNotification"; +import { + type SpeechGateSettings, + type SpeechKind, + type SpeechSource, + shouldSpeak, +} from "./speechRouting"; + +const base: SpeechGateSettings = { + enabled: true, + needsInput: true, + completion: true, + progress: true, + focusMode: "always", +}; + +describe("shouldSpeak", () => { + it("is silent when the feature is disabled", () => { + expect( + shouldSpeak("needs_input", "agent", "native", { + ...base, + enabled: false, + }), + ).toBe(false); + }); + + it.each<[SpeechKind, keyof SpeechGateSettings]>([ + ["needs_input", "needsInput"], + ["done", "completion"], + ["progress", "progress"], + ])("respects the per-kind toggle for %s", (kind, key) => { + expect( + shouldSpeak(kind, "agent", "native", { ...base, [key]: false }), + ).toBe(false); + }); + + it("always speaks agent needs-input regardless of focus", () => { + for (const channel of ["suppress", "toast", "native"] as const) { + expect( + shouldSpeak("needs_input", "agent", channel, { + ...base, + focusMode: "app_unfocused", + }), + ).toBe(true); + } + }); + + it.each<[NotificationChannel, boolean]>([ + ["suppress", false], + ["toast", true], + ["native", true], + ])( + "backstop needs-input is silent only over the viewed task: channel %s -> %s", + (channel, expected) => { + expect( + shouldSpeak("needs_input", "backstop", channel, { + ...base, + focusMode: "app_unfocused", + }), + ).toBe(expected); + }, + ); + + it("backstop done never plays over the viewed task, even in always mode", () => { + expect( + shouldSpeak("done", "backstop", "suppress", { + ...base, + focusMode: "always", + }), + ).toBe(false); + }); + + it.each<[SpeechSource, NotificationChannel, boolean]>([ + ["agent", "suppress", false], + ["agent", "toast", true], + ["agent", "native", true], + ["backstop", "suppress", false], + ["backstop", "toast", true], + ["backstop", "native", true], + ])( + "unviewed_task: %s done on channel %s -> %s", + (source, channel, expected) => { + expect( + shouldSpeak("done", source, channel, { + ...base, + focusMode: "unviewed_task", + }), + ).toBe(expected); + }, + ); + + it.each<[NotificationChannel, boolean]>([ + ["suppress", false], + ["toast", false], + ["native", true], + ])("app_unfocused: channel %s -> %s", (channel, expected) => { + expect( + shouldSpeak("done", "agent", channel, { + ...base, + focusMode: "app_unfocused", + }), + ).toBe(expected); + }); + + it("always mode speaks agent lines on every channel", () => { + for (const channel of ["suppress", "toast", "native"] as const) { + expect( + shouldSpeak("done", "agent", channel, { ...base, focusMode: "always" }), + ).toBe(true); + } + }); +}); diff --git a/packages/ui/src/features/notifications/speechRouting.ts b/packages/ui/src/features/notifications/speechRouting.ts new file mode 100644 index 0000000000..fa3debb848 --- /dev/null +++ b/packages/ui/src/features/notifications/speechRouting.ts @@ -0,0 +1,57 @@ +import type { + SpeechKind, + SpeechSource, +} from "@posthog/core/speech/identifiers"; +import type { SpokenFocusMode } from "@posthog/ui/features/settings/settingsStore"; +import type { NotificationChannel } from "./routeNotification"; + +export type { SpeechKind, SpeechSource }; + +export interface SpeechGateSettings { + enabled: boolean; + needsInput: boolean; + completion: boolean; + progress: boolean; + focusMode: SpokenFocusMode; +} + +/** + * Whether a spoken line should play, given who authored it, the focus-routing + * channel (from routeNotification) and the user's spoken-notification + * settings. Pure so the policy is exhaustively unit-tested without the DI + * graph. + * + * Agent needs-input lines ignore focus mode entirely — a blocker is the whole + * point of the feature. Backstop lines never play over the task the user is + * already viewing: the permission dialog (or finished turn) is on screen, and + * consecutive permission prompts would otherwise narrate every approval click. + */ +export function shouldSpeak( + kind: SpeechKind, + source: SpeechSource, + channel: NotificationChannel, + s: SpeechGateSettings, +): boolean { + if (!s.enabled) return false; + + const kindEnabled = + kind === "needs_input" + ? s.needsInput + : kind === "done" + ? s.completion + : s.progress; + if (!kindEnabled) return false; + + if (source === "backstop" && channel === "suppress") return false; + + if (kind === "needs_input") return true; + + switch (s.focusMode) { + case "always": + return true; + case "unviewed_task": + return channel !== "suppress"; + case "app_unfocused": + return channel === "native"; + } +} diff --git a/packages/ui/src/features/sessions/sessionServiceHost.ts b/packages/ui/src/features/sessions/sessionServiceHost.ts index 9a57f913e7..13548c4604 100644 --- a/packages/ui/src/features/sessions/sessionServiceHost.ts +++ b/packages/ui/src/features/sessions/sessionServiceHost.ts @@ -23,6 +23,7 @@ import { fetchAuthState } from "@posthog/ui/features/auth/authQueries"; import { useUsageLimitStore } from "@posthog/ui/features/billing/usageLimitStore"; import { useAddDirectoryDialogStore } from "@posthog/ui/features/folder-picker/addDirectoryDialogStore"; import { NotificationBus } from "@posthog/ui/features/notifications/notifications"; +import { SpeechNotifier } from "@posthog/ui/features/notifications/speechNotifier"; import { useSessionAdapterStore } from "@posthog/ui/features/sessions/sessionAdapterStore"; import { getPersistedConfigOptions, @@ -92,6 +93,7 @@ function buildSessionServiceDeps(): SessionServiceDeps { taskId, durationMs, ), + enqueueSpeech: (request) => resolveService(SpeechNotifier).speak(request), getIsOnline, fetchAuthState, getAuthenticatedClient, diff --git a/packages/ui/src/features/settings/sections/NotificationsSettings.tsx b/packages/ui/src/features/settings/sections/NotificationsSettings.tsx index eb3663602c..ed6ddc4afe 100644 --- a/packages/ui/src/features/settings/sections/NotificationsSettings.tsx +++ b/packages/ui/src/features/settings/sections/NotificationsSettings.tsx @@ -4,6 +4,7 @@ import { type INotifications, NOTIFICATIONS_SERVICE, } from "@posthog/platform/notifications"; +import { type ISpeech, SPEECH_SERVICE } from "@posthog/platform/speech"; import { ANALYTICS_EVENTS, PROJECT_BLUEBIRD_FLAG } from "@posthog/shared"; import type { Task } from "@posthog/shared/domain-types"; import { useFeatureFlag } from "@posthog/ui/features/feature-flags/useFeatureFlag"; @@ -14,8 +15,13 @@ import { type CompletionSound, type CustomSound, NOTIFICATION_DEFAULTS, + type SpokenFocusMode, useSettingsStore, } from "@posthog/ui/features/settings/settingsStore"; +import { + type ISpeechKeyStore, + SPEECH_KEY_STORE, +} from "@posthog/ui/features/settings/speechKeyStore"; import { useTasks } from "@posthog/ui/features/tasks/useTasks"; import { Tooltip } from "@posthog/ui/primitives/Tooltip"; import { toast } from "@posthog/ui/primitives/toast"; @@ -360,6 +366,8 @@ export function NotificationsSettings() { )} + + void; + disabled?: boolean; +}) { + return ( + + + + ); +} + +// Voice narration: the agent speaks a short line when it needs the user or +// finishes. The master toggle gates the whole feature; sub-controls disable +// when it's off. The ElevenLabs key is written to encrypted host storage via an +// injected capability (never kept in packages/ui or the persisted blob). +function SpokenNotificationsSection() { + const { + spokenNotifications, + spokenNotifyNeedsInput, + spokenNotifyCompletion, + spokenNotifyProgress, + spokenFocusMode, + elevenLabsVoiceId, + elevenLabsKeyConfigured, + setSpokenNotifications, + setSpokenNotifyNeedsInput, + setSpokenNotifyCompletion, + setSpokenNotifyProgress, + setSpokenFocusMode, + setElevenLabsVoiceId, + setElevenLabsKeyConfigured, + } = useSettingsStore(); + + const keyStore = useServiceOptional(SPEECH_KEY_STORE); + const speech = useServiceOptional(SPEECH_SERVICE); + const [keyDraft, setKeyDraft] = useState(""); + const [savingKey, setSavingKey] = useState(false); + + const disabled = !spokenNotifications; + + const saveKey = useCallback(async () => { + if (!keyStore || !keyDraft.trim()) return; + setSavingKey(true); + try { + await keyStore.save(keyDraft.trim()); + setElevenLabsKeyConfigured(true); + setKeyDraft(""); + toast.success("ElevenLabs key saved"); + } catch { + toast.error("Couldn't save the key"); + } finally { + setSavingKey(false); + } + }, [keyStore, keyDraft, setElevenLabsKeyConfigured]); + + const clearKey = useCallback(async () => { + if (!keyStore) return; + try { + await keyStore.clear(); + setElevenLabsKeyConfigured(false); + toast.success("ElevenLabs key removed"); + } catch { + toast.error("Couldn't remove the key"); + } + }, [keyStore, setElevenLabsKeyConfigured]); + + const testVoice = useCallback(() => { + void speech?.speak( + "PostHog Code task 'demo' — [excited] this is my voice!", + { voiceId: elevenLabsVoiceId || undefined }, + ); + }, [speech, elevenLabsVoiceId]); + + return ( + <> + + Spoken notifications + + + Have the agent say a short line out loud when it needs you or finishes, + so you hear it across parallel tasks without watching the screen. Lines + are serialized so agents never talk over each other. + + + + + + + + + + + + setSpokenFocusMode(v as SpokenFocusMode)} + disabled={disabled} + size="1" + > + + + + Quiet for the task I'm viewing + + + Only when app is in background + + Always + + + + + + {elevenLabsKeyConfigured ? ( + + + Key saved + + + + ) : ( + + setKeyDraft(e.currentTarget.value)} + disabled={disabled || !keyStore} + /> + + + )} + + + + + setElevenLabsVoiceId(e.currentTarget.value)} + disabled={disabled} + /> + + + + + ); +} + // A single installed custom sound: inline-rename field, preview, and delete. function CustomSoundRow({ sound, diff --git a/packages/ui/src/features/settings/settingsStore.ts b/packages/ui/src/features/settings/settingsStore.ts index b8539e8490..d8abfa2ba8 100644 --- a/packages/ui/src/features/settings/settingsStore.ts +++ b/packages/ui/src/features/settings/settingsStore.ts @@ -27,6 +27,13 @@ export type SendMessagesWith = "enter" | "cmd+enter"; export type AutoConvertLongText = "off" | "1000" | "2500" | "5000" | "10000"; export type DiffOpenMode = "auto" | "split" | "same-pane" | "last-active-pane"; +// When spoken notifications are allowed to talk, relative to what's on screen: +// - always: speak regardless of what the user is looking at +// - unviewed_task: stay quiet for the task currently on screen +// - app_unfocused: only speak when PostHog Code isn't the focused app +// (needs-input lines ignore this so a blocker is never missed.) +export type SpokenFocusMode = "always" | "unviewed_task" | "app_unfocused"; + export type BuiltInCompletionSound = | "none" | "guitar" @@ -134,6 +141,24 @@ interface SettingsStore { removeCustomSound: (id: string) => void; renameCustomSound: (id: string, name: string) => void; + // Spoken notifications + spokenNotifications: boolean; + spokenNotifyNeedsInput: boolean; + spokenNotifyCompletion: boolean; + spokenNotifyProgress: boolean; + spokenFocusMode: SpokenFocusMode; + elevenLabsVoiceId: string; + // Mirrors whether an ElevenLabs key is stored (the key itself lives in + // encrypted secure storage, never in this persisted blob). + elevenLabsKeyConfigured: boolean; + setSpokenNotifications: (enabled: boolean) => void; + setSpokenNotifyNeedsInput: (enabled: boolean) => void; + setSpokenNotifyCompletion: (enabled: boolean) => void; + setSpokenNotifyProgress: (enabled: boolean) => void; + setSpokenFocusMode: (mode: SpokenFocusMode) => void; + setElevenLabsVoiceId: (voiceId: string) => void; + setElevenLabsKeyConfigured: (configured: boolean) => void; + // Composer / chat autoConvertLongText: AutoConvertLongText; sendMessagesWith: SendMessagesWith; @@ -208,6 +233,13 @@ export const NOTIFICATION_DEFAULTS = { completionSound: "none" as CompletionSound, completionVolume: 80, scaleSoundWithTaskLength: false, + spokenNotifications: false, + spokenNotifyNeedsInput: true, + spokenNotifyCompletion: true, + spokenNotifyProgress: false, + spokenFocusMode: "unviewed_task" as SpokenFocusMode, + elevenLabsVoiceId: "", + elevenLabsKeyConfigured: false, }; export const useSettingsStore = create()( @@ -277,6 +309,18 @@ export const useSettingsStore = create()( setToastNotifications: (enabled) => set({ toastNotifications: enabled }), setCompletionSound: (sound) => set({ completionSound: sound }), setCompletionVolume: (volume) => set({ completionVolume: volume }), + setSpokenNotifications: (enabled) => + set({ spokenNotifications: enabled }), + setSpokenNotifyNeedsInput: (enabled) => + set({ spokenNotifyNeedsInput: enabled }), + setSpokenNotifyCompletion: (enabled) => + set({ spokenNotifyCompletion: enabled }), + setSpokenNotifyProgress: (enabled) => + set({ spokenNotifyProgress: enabled }), + setSpokenFocusMode: (mode) => set({ spokenFocusMode: mode }), + setElevenLabsVoiceId: (voiceId) => set({ elevenLabsVoiceId: voiceId }), + setElevenLabsKeyConfigured: (configured) => + set({ elevenLabsKeyConfigured: configured }), setScaleSoundWithTaskLength: (enabled) => set({ scaleSoundWithTaskLength: enabled }), addCustomSound: (sound) => @@ -423,6 +467,13 @@ export const useSettingsStore = create()( completionVolume: state.completionVolume, scaleSoundWithTaskLength: state.scaleSoundWithTaskLength, customSounds: state.customSounds, + spokenNotifications: state.spokenNotifications, + spokenNotifyNeedsInput: state.spokenNotifyNeedsInput, + spokenNotifyCompletion: state.spokenNotifyCompletion, + spokenNotifyProgress: state.spokenNotifyProgress, + spokenFocusMode: state.spokenFocusMode, + elevenLabsVoiceId: state.elevenLabsVoiceId, + elevenLabsKeyConfigured: state.elevenLabsKeyConfigured, // Composer / chat autoConvertLongText: state.autoConvertLongText, diff --git a/packages/ui/src/features/settings/speechKeyStore.ts b/packages/ui/src/features/settings/speechKeyStore.ts new file mode 100644 index 0000000000..fb969a75ac --- /dev/null +++ b/packages/ui/src/features/settings/speechKeyStore.ts @@ -0,0 +1,13 @@ +/** + * Writes the ElevenLabs API key to encrypted host storage. The key never lives + * in the persisted settings blob or in packages/ui — the host binds this to the + * secure store (see apps/code desktop-services). The settings UI injects it to + * save/clear the key; a boolean "configured" flag in the settings store mirrors + * whether one is set, so the UI never reads the secret back. + */ +export interface ISpeechKeyStore { + save(apiKey: string): Promise; + clear(): Promise; +} + +export const SPEECH_KEY_STORE = Symbol.for("posthog.ui.speech.keyStore"); diff --git a/packages/ui/src/utils/speech.ts b/packages/ui/src/utils/speech.ts new file mode 100644 index 0000000000..e405f22418 --- /dev/null +++ b/packages/ui/src/utils/speech.ts @@ -0,0 +1,85 @@ +// Renderer-side speech playback. Two paths, both resolve when playback ends so +// the core queue can serialize utterances one at a time: +// - ElevenLabs MP3 bytes (synthesized in the host, key stays there) played via +// a blob object URL — NOT a data: URL, which Chromium won't fully load for a +// multi-second clip. +// - the system voice via the Web Speech API (fallback when no key is set). +// Best-effort: every path resolves rather than rejecting, and a single current +// utterance is tracked so stop() and the next line interrupt cleanly. + +let currentAudio: HTMLAudioElement | null = null; + +/** Remove [audio tags] the system voice would otherwise read aloud. */ +export function stripAudioTags(text: string): string { + return text + .replace(/\[[^\]]*\]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function base64ToBlob(base64: string, mimeType: string): Blob { + const bytes = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)); + return new Blob([bytes], { type: mimeType }); +} + +export function playAudioBase64( + base64: string, + mimeType: string, +): Promise { + return new Promise((resolve) => { + stopSpeech(); + let objectUrl: string; + try { + objectUrl = URL.createObjectURL(base64ToBlob(base64, mimeType)); + } catch { + resolve(); + return; + } + const audio = new Audio(objectUrl); + currentAudio = audio; + const done = () => { + if (currentAudio === audio) currentAudio = null; + URL.revokeObjectURL(objectUrl); + resolve(); + }; + audio.addEventListener("ended", done); + audio.addEventListener("error", done); + audio.play().catch(done); + }); +} + +export function speakSystemVoice(text: string): Promise { + return new Promise((resolve) => { + if (typeof window === "undefined" || !window.speechSynthesis) { + resolve(); + return; + } + const clean = stripAudioTags(text); + if (!clean) { + resolve(); + return; + } + const utterance = new SpeechSynthesisUtterance(clean); + utterance.onend = () => resolve(); + utterance.onerror = () => resolve(); + window.speechSynthesis.speak(utterance); + }); +} + +export function stopSpeech(): void { + if (currentAudio) { + currentAudio.pause(); + currentAudio = null; + } + if (window?.speechSynthesis) { + window.speechSynthesis.cancel(); + } +} + +export function isSpeechSupported(): boolean { + return ( + typeof window !== "undefined" && + (typeof window.speechSynthesis !== "undefined" || + typeof Audio !== "undefined") + ); +} diff --git a/packages/workspace-server/src/services/speech/identifiers.ts b/packages/workspace-server/src/services/speech/identifiers.ts new file mode 100644 index 0000000000..b169fc82f7 --- /dev/null +++ b/packages/workspace-server/src/services/speech/identifiers.ts @@ -0,0 +1,25 @@ +export const SPEECH_SYNTHESIZER_SERVICE = Symbol.for( + "posthog.workspace.speechSynthesizer", +); + +export interface SpeechSynthesisResult { + /** Base64-encoded MP3 audio bytes. */ + audioBase64: string; + mimeType: string; +} + +/** + * Synthesizes speech audio from text (the API key stays in the host). Returns + * null when no key is configured or synthesis fails, so the renderer falls back + * to the system voice. Playback itself happens in the renderer (host-neutral). + * Best-effort — never throws. + */ +export interface ISpeechSynthesizer { + synthesize( + text: string, + voiceId?: string, + ): Promise; +} + +/** Secure-store key the ElevenLabs API key is saved under. */ +export const ELEVENLABS_API_KEY_STORE_KEY = "elevenlabs.apiKey"; diff --git a/scripts/host-boundary-allowlist.json b/scripts/host-boundary-allowlist.json index c58253ebc2..0f4d8338be 100644 --- a/scripts/host-boundary-allowlist.json +++ b/scripts/host-boundary-allowlist.json @@ -34,6 +34,9 @@ "apps/code/src/main/services/secure-store/service.ts": [ "injectable-outside-host" ], + "apps/code/src/main/services/speech/service.ts": [ + "injectable-outside-host" + ], "apps/code/src/main/services/workspace-server/service.ts": [ "injectable-outside-host" ],