diff --git a/.dockerignore b/.dockerignore index f157d40a..48269e9a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,9 +14,16 @@ .env.* !.env.sample -# Documentation and repos +# Documentation, scratch, and repos (not needed to build/run the server) docs/ repos/ +scratch/ +queries/ +packs/ +*.md + +# Local tool downloads (the ./bun wrapper caches Bun here) +download/ # CLI build artifacts packages/cli/dist/ diff --git a/.env.sample b/.env.sample index 66cd90d9..bfefa1d9 100644 --- a/.env.sample +++ b/.env.sample @@ -9,7 +9,7 @@ # PostgreSQL connection string for the application database. One database holds # the auth + core control plane and every per-space me_ schema. -DATABASE_URL=postgres://postgres:postgres@localhost:5432/me +DATABASE_URL=postgres://postgres:postgres@localhost:5432/postgres # Public base URL for OAuth callbacks API_BASE_URL=http://localhost:3000 @@ -17,6 +17,22 @@ API_BASE_URL=http://localhost:3000 # OpenAI API key (or compatible provider) for generating embeddings EMBEDDING_API_KEY= +# better-auth signing secret: signs session cookies and encrypts the JWKS keys. +# Required — the server refuses to boot without it. Use a long random value, +# e.g. `openssl rand -base64 32`. +BETTER_AUTH_SECRET= + +# ----------------------------------------------------------------------------- +# Optional — Docker Compose (self-host) +# ----------------------------------------------------------------------------- + +# Password for the bundled Postgres container in compose.yaml. Required by +# `docker compose up` (the server's DATABASE_URL is built from it); ignored by +# host-run dev (`./bun run server`), which uses DATABASE_URL above. Use a +# URL-safe value (letters/digits) so it needs no encoding in the connection +# string. See SELF_HOST.md. +# POSTGRES_PASSWORD= + # ----------------------------------------------------------------------------- # OAuth Providers # ----------------------------------------------------------------------------- @@ -47,22 +63,37 @@ GOOGLE_CLIENT_SECRET= # AUTH_SCHEMA=auth # CORE_SCHEMA=core -# Cron schedule for cleaning up expired device authorizations (UTC) -# DEVICE_FLOW_CLEANUP_CRON=*/15 * * * * +# Cron schedule for the auth cleanup job, e.g. expired device authorizations +# (UTC). The legacy name DEVICE_FLOW_CLEANUP_CRON is still honored. +# AUTH_CLEANUP_CRON=*/15 * * * * + +# Directory of the built web UI to serve at root `/` (the Docker image bakes +# this in; override only for host runs with a non-default build location). +# WEB_DIST=packages/web/dist + +# Extra origins allowed to make cookie-authenticated requests (CSRF gate), +# comma-separated. The API_BASE_URL origin is always allowed; add others here +# (e.g. to permit both api.* and app.* during a cutover). +# WEB_ALLOWED_ORIGINS= # ----------------------------------------------------------------------------- # Optional — Telemetry # ----------------------------------------------------------------------------- -# Logfire token for OpenTelemetry export (omit to disable) +# Logfire token for OpenTelemetry export. Default: unset (no export). The token +# is project-scoped, so data only appears in that token's own Logfire project. # LOGFIRE_TOKEN= -# LOGFIRE_ENVIRONMENT=prod +# Deployment environment label for telemetry. Default: unset. +# Example: LOGFIRE_ENVIRONMENT=prod +# LOGFIRE_ENVIRONMENT= -# Print spans to console (useful for local development) +# Print spans to the console. Default: off — set to true to enable +# (useful for local development). # LOGFIRE_CONSOLE=true -# Disable scrubbing of sensitive fields (content, embeddings, tokens) +# Scrubbing of sensitive fields (content, embeddings, tokens) is ON by default. +# Set to false to disable. # LOGFIRE_SCRUBBING=false # ----------------------------------------------------------------------------- @@ -142,7 +173,8 @@ GOOGLE_CLIENT_SECRET= # The embedding worker uses a dedicated pool. Each setting defaults to the # corresponding application-pool value (or DATABASE_URL) when unset. -# WORKER_DATABASE_URL=postgres://postgres:postgres@localhost:5432/me +# WORKER_DATABASE_URL=postgres://postgres:postgres@localhost:5432/postgres +# Defaults to max(WORKER_COUNT, 1), not a fixed 2. # WORKER_DB_POOL_MAX=2 # WORKER_DB_POOL_IDLE_REAP_SECONDS=300 # WORKER_DB_POOL_MAX_LIFETIME=0 diff --git a/CLAUDE.md b/CLAUDE.md index d9e358d5..ea262d27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,7 +99,7 @@ Always use the `./bun` wrapper script (auto-installs the pinned Bun version): **Important — verification runs against the local Postgres**: after making code changes, run `./bun run check` (fast, no DB). Before committing, run `./bun run check:full` — it defaults to the `me-postgres` Docker container -(if it isn't running: `docker start me-postgres || ./bun run pg`). Only run +(if it isn't running: `docker start me-postgres || ./bun run pg:docker`). Only run against ghost when explicitly asked to test against ghost. CI is the strict gate: it runs every suite with `TEST_CI=1`, which disables conditional skips — any new `describe.skipIf` gate **must** include `!process.env.TEST_CI` in @@ -114,7 +114,7 @@ its condition (pattern: `packages/embedding/generate.test.ts`, `*.integration.test.ts` files run against a real PostgreSQL 18 with the required extensions (citext, ltree, pgvector, pg_textsearch). Everything defaults to the **local `me-postgres` Docker container** at 127.0.0.1:5432 -(same image CI builds; `./bun run pg` creates it). `test:db` is the focused +(same image CI builds; `./bun run pg:docker` creates it). `test:db` is the focused variant: it first reclaims orphaned test schemas, then runs **every** `*.integration.test.ts` under `packages/` (the auth/core/space migration suites plus the engine/server/worker suites), `--parallel=2`, 30s timeout: diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 78092a76..d1df76ba 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -42,7 +42,7 @@ memory-engine itself as that can be confusing for the model. ```bash ./bun install -./bun run pg +./bun run pg:docker # configure .env (see below) ./bun run setup ./bun run server @@ -63,10 +63,10 @@ memory-engine itself as that can be confusing for the model. The project requires PostgreSQL 18 with three extensions: `pgvector`, `pg_textsearch`, and `ltree`. The included Dockerfile builds a pre-configured image: ```bash -./bun run pg +./bun run pg:docker ``` -This builds the Docker image and starts a container named `me-postgres` on `localhost:5432` with trust authentication (no password). +This builds the Docker image and starts a container named `me-postgres` on `localhost:5432` with trust authentication (no password). It serves the default `postgres` database, which is the one `.env.sample` targets. Other database commands: @@ -88,7 +88,7 @@ cp .env.sample .env **Database connection** — one database holds the `auth` + `core` control plane and every per-space `me_` schema: ``` -DATABASE_URL=postgres://postgres@localhost:5432/memory_engine +DATABASE_URL=postgres://postgres@localhost:5432/postgres ``` @@ -144,7 +144,7 @@ GITHUB_CLIENT_SECRET=... ```bash # Database -DATABASE_URL=postgres://postgres@localhost:5432/memory_engine +DATABASE_URL=postgres://postgres@localhost:5432/postgres # Server API_BASE_URL=http://localhost:3000 @@ -173,12 +173,12 @@ specified in milliseconds, for example `RPC_DB_STATEMENT_TIMEOUT_MS=30000`. ./bun run setup ``` -This is idempotent (safe to run multiple times) and will: - -1. Create the `accounts` and `shard1` databases if they don't exist -2. Run account schema migrations -3. Create and activate an encryption data key -4. Bootstrap the engine database (extensions and roles) +This is idempotent (safe to run multiple times). It reads `DATABASE_URL` and +creates that database if it doesn't already exist (everything else — +bootstrap, migrations, encryption keys — happens automatically at server +startup). When `DATABASE_URL` targets `/postgres` (the `me-postgres` +container's default database, as in `.env.sample`), this is effectively a +no-op since that database already exists — running it is still harmless. ### 5. Start the server @@ -186,6 +186,45 @@ This is idempotent (safe to run multiple times) and will: ./bun run server ``` +### 5b. Run the server in Docker (optional) + +To exercise the actual production image locally — the multi-stage +`packages/server/Dockerfile` that CI builds and deploys — use the `server:*` +scripts (the Docker counterparts to `pg:*`): + +```bash +./bun run server:docker # build the image + run in the foreground (me-server) +./bun run server:build # build the image only +./bun run server:rm # force-remove the container (only needed if it leaks) +``` + +`server:docker` runs the container in the **foreground** (`--rm -t`): logs +scroll in the terminal, `Ctrl+C` triggers the server's graceful shutdown, and +the container is removed on exit. It publishes the server on `127.0.0.1:3000` +and wires it to the `me-postgres` container. Prerequisites: + +- Postgres running (`./bun run pg:docker`). +- A populated `.env` (the container is started with `--env-file .env`, so it + reads `EMBEDDING_API_KEY`, the OAuth credentials, telemetry, etc. from there). + +How it reaches Postgres: the script overrides `DATABASE_URL` to +`postgres://postgres@host.docker.internal:5432/postgres` (and adds +`--add-host=host.docker.internal:host-gateway` so the hostname resolves on +Linux too). A container can't reach the host's `localhost`, so this override +replaces the `localhost` value your `.env` uses for host-run development. +`-e` takes precedence over `--env-file`, so the rest of `.env` still applies. + +Caveats: + +- The script hardcodes port `3000`. If you set a different `PORT` in `.env`, + edit the `server:docker` script's published port and keep `API_BASE_URL` + consistent. +- The override only covers `DATABASE_URL`. If you uncomment + `WORKER_DATABASE_URL` in `.env`, it must also use `host.docker.internal`. +- `docker run --env-file` parses literal `KEY=VALUE` lines — no quotes, no + `$VAR` expansion (unlike Bun's `.env` loader). `.env.sample` is already + compatible. + ### 6. Test with the CLI In another terminal: @@ -207,11 +246,14 @@ After login, the server URL is stored as the default in `~/.config/me/credential | Command | Description | |---|---| -| `./bun run server` | Start the server | -| `./bun run setup` | Create databases, run migrations, bootstrap engine | -| `./bun run pg` | Build and start PostgreSQL in Docker | +| `./bun run server` | Start the server (on the host) | +| `./bun run setup` | Ensure the `DATABASE_URL` database exists | +| `./bun run pg:docker` | Build and start PostgreSQL in Docker | | `./bun run pg:rm` | Stop and remove the PostgreSQL container | | `./bun run psql` | Connect to PostgreSQL with psql | +| `./bun run server:build` | Build the server Docker image (`me-server`) | +| `./bun run server:docker` | Build + run the server in Docker (vs `me-postgres`) | +| `./bun run server:rm` | Stop and remove the server container | | `./bun run test` | Run all package tests (unit + integration, vs local Postgres by default) | | `./bun run check` | Fast inner loop: typecheck + lint + unit tests (no database) | | `./bun run check:full` | Everything: check + full suite + e2e (vs local Postgres by default) | diff --git a/README.md b/README.md index 79d10879..d52c66a9 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,12 @@ Memory Engine runs as an MCP server that AI agents connect to over stdio. Each a - **tstzrange** for temporal queries - **Tree-scoped access grants** evaluated in the search SQL (no RLS) +## Self-hosting + +Want to run your own Memory Engine backend? See **[Self-Hosting](SELF_HOST.md)** +— a Docker Compose stack (server + PostgreSQL), built from a tagged release, +plus building the `me` CLI from source to connect to it. + ## Documentation - [Getting Started](docs/getting-started.md) -- install, login, first memory diff --git a/SELF_HOST.md b/SELF_HOST.md new file mode 100644 index 00000000..d8b97e9c --- /dev/null +++ b/SELF_HOST.md @@ -0,0 +1,152 @@ +# Self-Hosting Memory Engine + +Run your own Memory Engine backend with Docker Compose, and build the `me` CLI +from source to use it. + +There are **two pieces, and you need both**: + +1. **The backend stack** — the server + PostgreSQL, started with the + `compose.yaml` in this repo. This is just the backend; on its own it does + nothing useful. +2. **A client** — the `me` CLI (or any MCP client) pointed at the backend. The + stack is only reachable once a client connects to it. + +A typical setup keeps `docker compose up` running in one terminal and uses the +`me` CLI from another. + +## Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) with Compose (`docker compose`). +- A Git checkout of this repository (the bundled `./bun` wrapper installs the + pinned Bun version for you — no separate Bun install needed). +- An embedding API key — an OpenAI key, or a compatible provider. +- At least one OAuth app for login — GitHub and/or Google. + +## 0. Get the code at a tagged release + +Build **everything** — the compose stack and the CLI — from the same tagged +release commit. The server and CLI version counters diverge over time and the +client/server handshake enforces a minimum version on each side, so a single +released commit is the only combination guaranteed to be compatible. Don't run a +real deployment off `main`, which can sit mid-flight between releases. + +```bash +git clone https://github.com/timescale/memory-engine +cd memory-engine +git fetch --tags +# Check out the latest server release (server/vX.Y.Z): +git checkout "$(git tag -l 'server/v*' | sort -V | tail -1)" +``` + +To pin a specific version instead, browse the +[releases](https://github.com/timescale/memory-engine/releases) and +`git checkout server/vX.Y.Z`. + +## 1. Configure `.env` + +```bash +cp .env.sample .env +``` + +Set the required values: + +- `EMBEDDING_API_KEY` — your OpenAI (or compatible) API key. +- `POSTGRES_PASSWORD` — the password for the bundled Postgres container. Use a + URL-safe value (letters/digits) so it needs no encoding in the connection + string. `compose.yaml` refuses to start if this is unset. +- `API_BASE_URL=http://localhost:3000` — keep this consistent with the + published port (see [Notes](#notes--troubleshooting)). +- At least one OAuth provider (`GITHUB_CLIENT_ID`/`GITHUB_CLIENT_SECRET` and/or + `GOOGLE_CLIENT_ID`/`GOOGLE_CLIENT_SECRET`). When creating the app, set the + authorized callback URL to match `API_BASE_URL`: + - **GitHub** — create at + ([guide](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app)). + Callback: `http://localhost:3000/api/v1/auth/callback/github` + - **Google** — create at + ([guide](https://developers.google.com/identity/protocols/oauth2/web-server#creatingcred)). + Callback: `http://localhost:3000/api/v1/auth/callback/google` + +## 2. Start the backend + +```bash +docker compose up --build +``` + +This builds the Postgres image (`docker/Dockerfile.postgres`) and the server +image (`packages/server/Dockerfile`), then starts both. The server: + +- listens on , +- runs database migrations automatically on boot, +- connects to Postgres over the internal compose network (the database is + **not** exposed to the host by default). + +Postgres data persists in the `pgdata` Docker volume across restarts. + +## 3. Build the CLI + +From the same checkout (so the CLI matches your server version): + +```bash +./bun install +./bun run install:local +``` + +`install:local` builds the `me` binary and installs it to `~/.local/bin` (set +`ME_INSTALL_DIR` to override). Add that directory to your `PATH` if it isn't +already: + +```bash +export PATH="$HOME/.local/bin:$PATH" +``` + +Alternatively, just build it and run the binary in place: + +```bash +./bun run build # produces packages/cli/dist/me +./packages/cli/dist/me --help +``` + +## 4. Connect and log in + +Point the CLI at your self-hosted server and authenticate: + +```bash +export ME_SERVER=http://localhost:3000 +me login +``` + +`me login` opens a browser to complete the OAuth flow with the provider you +configured. After logging in, your server URL is saved as the default, so later +commands don't need `ME_SERVER`. Try it: + +```bash +me memory create "Auth uses bcrypt with cost 12" --tree share.design.auth +me memory search "how does authentication work" +``` + +Keep `docker compose up` running while you use the CLI — the backend stack and +the CLI are separate processes. + +## Lifecycle + +| Task | Command | +|---|---| +| Stop (keep data) | `docker compose down` | +| Stop and wipe the database | `docker compose down -v` (deletes the `pgdata` volume) | +| Follow server logs | `docker compose logs -f server` | +| Update | `git fetch --tags && git checkout server/vX.Y.Z`, then `docker compose up --build` and rebuild the CLI (`./bun run install:local`) | + +## Notes / troubleshooting + +- **Keep ports consistent.** `API_BASE_URL`, the server's `PORT` (default 3000), + and the published port in `compose.yaml` (`3000:3000`) must agree. To run on a + different port, change all three, and update your OAuth callback URLs to match. +- **OAuth callback must match `API_BASE_URL` exactly**, including the port. +- **`POSTGRES_PASSWORD` only takes effect on first database init** (an empty + `pgdata` volume). Changing it later has no effect unless you wipe the volume + (`docker compose down -v`) or alter the role inside Postgres. +- **The database isn't exposed to the host** by default. To connect with `psql`, + uncomment the `ports` block under the `postgres` service in `compose.yaml`. +- **Version mismatch errors** at login or on RPC calls usually mean the CLI and + server were built from different commits. Rebuild both from the same + `server/v*` tag (see [step 0](#0-get-the-code-at-a-tagged-release)). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..dab44d65 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,55 @@ +# Evaluate / self-host stack: PostgreSQL + the Memory Engine server. +# +# Usage: +# cp .env.sample .env # set EMBEDDING_API_KEY, POSTGRES_PASSWORD, an OAuth provider +# docker compose up --build +# +# Server: http://localhost:3000 +# +# This stack is the backend only. To use it you also need a client — the `me` +# CLI (or another MCP client) — pointed at http://localhost:3000. See +# SELF_HOST.md for the full walkthrough (including building the CLI). +services: + postgres: + build: + context: ./docker + dockerfile: Dockerfile.postgres + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + restart: unless-stopped + # The DB is reachable only on the internal compose network. To use psql from + # the host, uncomment below (conflicts with `./bun run pg:docker`, which + # already binds 5432): + # ports: + # - "127.0.0.1:5432:5432" + + server: + build: + context: . + dockerfile: packages/server/Dockerfile + env_file: .env + # Override both DB URLs from .env (host-oriented localhost) so the server + # reaches the postgres service over the compose network. `environment` + # takes precedence over `env_file`. + environment: + DATABASE_URL: postgres://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/postgres + WORKER_DATABASE_URL: postgres://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/postgres + ports: + - "3000:3000" + depends_on: + postgres: + condition: service_healthy + restart: unless-stopped + +volumes: + pgdata: diff --git a/package.json b/package.json index 16a4561c..2c877fbe 100644 --- a/package.json +++ b/package.json @@ -24,13 +24,16 @@ "lint": "biome check", "me": "./bun run packages/cli/index.ts", "migrate:db": "./bun scripts/migrate-db.ts", - "pg": "./bun run pg:build && docker run -d --name me-postgres -e POSTGRES_HOST_AUTH_METHOD=trust -p 127.0.0.1:5432:5432 me-postgres", "pg:build": "docker build -t me-postgres -f docker/Dockerfile.postgres docker/", + "pg:docker": "./bun run pg:build && docker run -d --name me-postgres -e POSTGRES_HOST_AUTH_METHOD=trust -p 127.0.0.1:5432:5432 me-postgres", "pg:rm": "docker rm -f me-postgres", "psql": "psql postgresql://postgres@127.0.0.1:5432/postgres", "release:client": "./bun scripts/release-client.ts", "release:server": "./bun scripts/release-server.ts", "server": "./bun run packages/server/index.ts", + "server:build": "docker build -t me-server -f packages/server/Dockerfile .", + "server:docker": "./bun run server:build && docker run --rm -t --name me-server --add-host=host.docker.internal:host-gateway -p 127.0.0.1:3000:3000 --env-file .env -e DATABASE_URL=postgres://postgres@host.docker.internal:5432/postgres me-server", + "server:rm": "docker rm -f me-server", "setup": "./bun scripts/setup.ts", "test": "TEST_DATABASE_URL=\"${TEST_DATABASE_URL:-postgresql://postgres@127.0.0.1:5432/postgres}\" ./bun test packages --timeout 30000 --parallel=2", "test:db": "./bun run test:db:clean && find packages -name '*.integration.test.ts' -print0 | xargs -0 ./bun test --parallel=2 --timeout 30000", diff --git a/scripts/setup.ts b/scripts/setup.ts index f02a3360..18258711 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -10,7 +10,7 @@ * which the server doesn't do. * * Prerequisites: - * 1. Postgres running (`./bun run pg`) + * 1. Postgres running (`./bun run pg:docker`) * 2. .env filled in (DATABASE_URL) * * Usage: