diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index a60f89c..af2c201 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -58,16 +58,16 @@ verifier through environment variables on the api component, e.g.: ```yaml api: - extraEnv: - - name: CODEAPI_AUTH_PROVIDER - value: librechat-jwt - - name: CODEAPI_JWT_PUBLIC_KEY # single PEM/base64-DER verifier key - valueFrom: - secretKeyRef: - name: codeapi-jwt-verifier - key: public-key - - name: CODEAPI_JWT_KID - value: my-key-id + extraEnv: + - name: CODEAPI_AUTH_PROVIDER + value: librechat-jwt + - name: CODEAPI_JWT_PUBLIC_KEY # single PEM/base64-DER verifier key + valueFrom: + secretKeyRef: + name: codeapi-jwt-verifier + key: public-key + - name: CODEAPI_JWT_KID + value: my-key-id ``` `CODEAPI_JWT_PUBLIC_KEYS_DIR` (a mounted directory of PEM files) and @@ -78,14 +78,35 @@ For development only, `LOCAL_MODE=true` bypasses authentication — see **TLS to Redis.** Set `REDIS_TLS=true` via `extraEnv` on each component when your Redis (e.g. a managed cache) requires TLS. +**Redis Cluster mode (GCP Memorystore cluster, AWS ElastiCache cluster).** Set +`redis.cluster.enabled=true` and provide the comma-separated startup nodes in +`redis.cluster.nodes`. The chart will set `USE_REDIS_CLUSTER=true` and populate +`REDIS_HOST` with the node list on every component. For TLS with a CA certificate +(recommended for GCP Memorystore), also set: + +```yaml +redis: + tls: + enabled: true + caSecretName: my-memorystore-secret # Secret that holds the CA cert + caKey: ca # Key inside the Secret + caMountPath: /etc/redis-tls/ca.crt # Mount path in each pod + useAlternativeDnsLookup: true # Required for GCP Memorystore cluster +``` + +The CA certificate is mounted from the Secret into every component pod and +passed to ioredis via `REDIS_CA`. No sidecar is needed. + ## Quick Start (Local Development) ### 1. Start Minikube + ```bash minikube start --cpus=4 --memory=8192 ``` ### 2. Build Images Inside Minikube + ```bash # Point docker to minikube's daemon eval $(minikube docker-env) @@ -100,6 +121,7 @@ docker build -t codeapi-package-init:latest -f docker/Dockerfile.package-init . ``` ### 3. Install Dependencies & Deploy + ```bash cd helm/codeapi @@ -151,6 +173,7 @@ kubectl rollout restart deployment/codeapi-sandbox-runner ``` ### 5. Access the API + ```bash # Port forward (in another terminal) kubectl port-forward svc/codeapi-api 3112:3112 @@ -164,6 +187,7 @@ curl http://localhost:3112/v1/health ## Commands Reference ### Startup + ```bash # Start minikube minikube start @@ -176,6 +200,7 @@ kubectl port-forward svc/codeapi-api 3112:3112 ``` ### Check Status + ```bash # View all pods kubectl get pods @@ -190,6 +215,7 @@ kubectl describe pod ``` ### Scaling + ```bash # Scale the sandbox execution tier kubectl scale deployment/codeapi-sandbox-runner --replicas=10 @@ -200,6 +226,7 @@ helm upgrade codeapi ./helm/codeapi -f ./helm/codeapi/values-local.yaml \ ``` ### Update After Code Changes + ```bash # Rebuild images (must be in minikube docker env) eval $(minikube docker-env) @@ -212,6 +239,7 @@ kubectl rollout restart deployment/codeapi-sandbox-runner ``` ### Teardown + ```bash # Uninstall the Helm release (removes all K8s resources) helm uninstall codeapi @@ -228,12 +256,14 @@ minikube delete ## Testing ### Health Check + ```bash curl http://localhost:3112/v1/health # Expected: OK ``` ### Execute Python Code + ```bash curl -X POST http://localhost:3112/v1/exec \ -H "Content-Type: application/json" \ @@ -242,6 +272,7 @@ curl -X POST http://localhost:3112/v1/exec \ ``` ### Verify Horizontal Scaling + ```bash # Check which service-worker processed the job kubectl logs deployment/codeapi-service-worker --tail=5 @@ -299,6 +330,7 @@ kubectl logs deployment/codeapi-service-worker --tail=5 ## Troubleshooting ### Pod stuck in `ErrImageNeverPull` + ```bash # Images must be built inside minikube's docker eval $(minikube docker-env) @@ -307,6 +339,7 @@ kubectl rollout restart deployment/ ``` ### Pod stuck in `CrashLoopBackOff` + ```bash # Check logs kubectl logs --previous @@ -314,6 +347,7 @@ kubectl describe pod ``` ### "runtime is unknown" error + ```bash # Language packages PVC is empty. Check if the init job ran: kubectl get jobs -l app.kubernetes.io/component=package-init @@ -327,12 +361,14 @@ kubectl rollout restart deployment/codeapi-sandbox-runner ``` ### Connection refused on port 3112 + ```bash # Make sure port-forward is running kubectl port-forward svc/codeapi-api 3112:3112 ``` ### MinIO `ImagePullBackOff` (production values) + ```bash # The Bitnami MinIO chart may reference unavailable image tags. # For local dev, values-local.yaml uses minio.useSimple=true which diff --git a/helm/codeapi/templates/_helpers.tpl b/helm/codeapi/templates/_helpers.tpl index ce81df7..e8032b2 100644 --- a/helm/codeapi/templates/_helpers.tpl +++ b/helm/codeapi/templates/_helpers.tpl @@ -152,11 +152,15 @@ app.kubernetes.io/component: tool-call-server {{- end }} {{/* -Redis host - either from subchart or external +Redis host - either from subchart or external. +In cluster mode this renders the comma-separated node list from redis.cluster.nodes +(falling back to redis.external.host for single-node external deployments). */}} {{- define "codeapi.redis.host" -}} {{- if .Values.redis.enabled }} {{- printf "%s-redis-master" .Release.Name }} +{{- else if and .Values.redis.cluster.enabled .Values.redis.cluster.nodes }} +{{- .Values.redis.cluster.nodes }} {{- else }} {{- .Values.redis.external.host }} {{- end }} @@ -173,6 +177,70 @@ Redis port {{- end }} {{- end }} +{{/* +USE_REDIS_CLUSTER value – "true" when redis.cluster.enabled or when +redis.cluster.nodes contains a comma (auto-detect multiple nodes). +*/}} +{{- define "codeapi.redis.clusterEnabled" -}} +{{- if .Values.redis.cluster.enabled }} +{{- "true" }} +{{- else if and .Values.redis.cluster.nodes (contains "," .Values.redis.cluster.nodes) }} +{{- "true" }} +{{- else }} +{{- "false" }} +{{- end }} +{{- end }} + +{{/* +Emit the Redis TLS + CA environment variables and volume mount for each +component that needs it. Renders nothing when redis.tls.enabled is false. +Usage: {{ include "codeapi.redis.tlsEnv" . }} +*/}} +{{- define "codeapi.redis.tlsEnv" -}} +{{- if .Values.redis.tls.enabled }} +- name: REDIS_TLS + value: "true" +{{- if .Values.redis.tls.caSecretName }} +- name: REDIS_CA + value: {{ .Values.redis.tls.caMountPath | quote }} +{{- end }} +{{- end }} +{{- if .Values.redis.useAlternativeDnsLookup }} +- name: REDIS_USE_ALTERNATIVE_DNS_LOOKUP + value: "true" +{{- end }} +{{- end }} + +{{/* +Volume definition for the Redis CA certificate secret. +Renders nothing when redis.tls.caSecretName is empty. +Usage: {{ include "codeapi.redis.caVolume" . }} +*/}} +{{- define "codeapi.redis.caVolume" -}} +{{- if and .Values.redis.tls.enabled .Values.redis.tls.caSecretName }} +- name: redis-ca + secret: + secretName: {{ .Values.redis.tls.caSecretName }} + items: + - key: {{ .Values.redis.tls.caKey }} + path: ca.crt +{{- end }} +{{- end }} + +{{/* +VolumeMount for the Redis CA certificate inside a container. +Renders nothing when redis.tls.caSecretName is empty. +Usage: {{ include "codeapi.redis.caVolumeMount" . }} +*/}} +{{- define "codeapi.redis.caVolumeMount" -}} +{{- if and .Values.redis.tls.enabled .Values.redis.tls.caSecretName }} +- name: redis-ca + mountPath: {{ .Values.redis.tls.caMountPath | quote }} + subPath: ca.crt + readOnly: true +{{- end }} +{{- end }} + {{/* Redis NetworkPolicy egress. Kubernetes NetworkPolicy cannot match DNS names, so external Redis can be scoped with CIDRs when available; otherwise the chart diff --git a/helm/codeapi/templates/api-deployment.yaml b/helm/codeapi/templates/api-deployment.yaml index 0c801ab..2de5257 100644 --- a/helm/codeapi/templates/api-deployment.yaml +++ b/helm/codeapi/templates/api-deployment.yaml @@ -51,7 +51,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password - # Service URLs + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: FILE_SERVER_URL value: "http://{{ include "codeapi.fullname" . }}-file-server:{{ .Values.fileServer.service.port }}" - name: TOOL_CALL_SERVER_URL @@ -87,6 +89,14 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.api.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 6e39490..96a5e37 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -69,6 +69,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} livenessProbe: httpGet: path: /live @@ -83,6 +86,14 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.egressGateway.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/file-server-deployment.yaml b/helm/codeapi/templates/file-server-deployment.yaml index 1dd9516..e542cd0 100644 --- a/helm/codeapi/templates/file-server-deployment.yaml +++ b/helm/codeapi/templates/file-server-deployment.yaml @@ -110,6 +110,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: CODEAPI_INTERNAL_SERVICE_TOKEN valueFrom: secretKeyRef: @@ -135,6 +138,14 @@ spec: timeoutSeconds: 5 resources: {{- toYaml .Values.fileServer.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/tool-call-server-deployment.yaml b/helm/codeapi/templates/tool-call-server-deployment.yaml index b5596c9..f648712 100644 --- a/helm/codeapi/templates/tool-call-server-deployment.yaml +++ b/helm/codeapi/templates/tool-call-server-deployment.yaml @@ -44,6 +44,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: CODEAPI_INTERNAL_SERVICE_TOKEN valueFrom: secretKeyRef: @@ -70,6 +73,14 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.toolCallServer.resources | nindent 12 }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index ff23008..7fa373c 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -136,6 +136,9 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: USE_REDIS_CLUSTER + value: {{ include "codeapi.redis.clusterEnabled" . | quote }} + {{- include "codeapi.redis.tlsEnv" . | nindent 12 }} - name: CODEAPI_INTERNAL_SERVICE_TOKEN valueFrom: secretKeyRef: @@ -163,6 +166,10 @@ spec: {{- with .Values.workerSandbox.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} + {{- with (include "codeapi.redis.caVolumeMount" . | trim) }} + volumeMounts: + {{- . | nindent 12 }} + {{- end }} livenessProbe: httpGet: path: /health @@ -183,6 +190,10 @@ spec: limits: {{- toYaml . | nindent 14 }} {{- end }} + {{- with (include "codeapi.redis.caVolume" . | trim) }} + volumes: + {{- . | nindent 8 }} + {{- end }} {{- with $serviceWorkerNodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index ce16c34..49865d4 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -9,37 +9,37 @@ # GLOBAL SETTINGS # ============================================================================= global: - # Image pull secrets (if using private registry) - imagePullSecrets: [] - # Storage class for persistent volumes - storageClass: "" + # Image pull secrets (if using private registry) + imagePullSecrets: [] + # Storage class for persistent volumes + storageClass: '' # Shared token for internal service-to-service requests between the public API, # file-server, tool-call-server, worker, and sandbox API. Override for every # production install. internalServiceAuth: - token: "changeme-in-production" + token: 'changeme-in-production' hardenedSandboxMode: true otel: - enabled: false - # OTLP/HTTP collector endpoint, e.g. "http://opentelemetry-collector.observability:4318". - # Required when enabled=true. - exporterOtlpEndpoint: "" - exporterOtlpTracesEndpoint: "" - # Comma-separated OTLP headers. Percent-encode commas and equals signs in - # values, matching the OTLP env-var convention. - exporterOtlpHeaders: "" - resourceAttributes: "" + enabled: false + # OTLP/HTTP collector endpoint, e.g. "http://opentelemetry-collector.observability:4318". + # Required when enabled=true. + exporterOtlpEndpoint: '' + exporterOtlpTracesEndpoint: '' + # Comma-separated OTLP headers. Percent-encode commas and equals signs in + # values, matching the OTLP env-var convention. + exporterOtlpHeaders: '' + resourceAttributes: '' # Encrypted grants/handles owned by the egress gateway. Sandbox-runner, # service-api, and service-worker never receive this secret. egressGrant: - secret: "changeme-egress-grant-secret-32-bytes-minimum" - ttlSeconds: 900 - ledgerRequired: true - ledgerTtlGraceSeconds: 300 + secret: 'changeme-egress-grant-secret-32-bytes-minimum' + ttlSeconds: 900 + ledgerRequired: true + ledgerTtlGraceSeconds: 300 # Worker signs sandbox execute requests with this private key; sandbox-runner # receives only the public verifier so a runner compromise cannot mint new @@ -51,59 +51,59 @@ egressGrant: # workerSandbox.enabled=true and these are unset. For minikube local dev, # values-local.yaml ships a test-only keypair. executionManifest: - privateKey: "" - publicKey: "" - ttlSeconds: 300 + privateKey: '' + publicKey: '' + ttlSeconds: 300 # ============================================================================= # API SERVICE (HTTP handlers, scales based on traffic) # ============================================================================= api: - enabled: true - replicaCount: 2 # Start with 2 API pods - - image: - repository: codeapi-api - tag: latest - pullPolicy: IfNotPresent # Use Always in production - - # Resource limits - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - - # Service configuration - service: - type: ClusterIP - port: 3112 - - httpJsonLimit: 50mb - - # Ingress (optional, for external access) - ingress: - enabled: false - className: "" - annotations: {} - hosts: - - host: codeapi.local - paths: - - path: / - pathType: Prefix - tls: [] - - # Horizontal Pod Autoscaler - autoscaling: - enabled: false - minReplicas: 2 - maxReplicas: 10 - targetCPUUtilizationPercentage: 70 + enabled: true + replicaCount: 2 # Start with 2 API pods - # Extra environment variables - extraEnv: [] + image: + repository: codeapi-api + tag: latest + pullPolicy: IfNotPresent # Use Always in production + + # Resource limits + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + # Service configuration + service: + type: ClusterIP + port: 3112 + + httpJsonLimit: 50mb + + # Ingress (optional, for external access) + ingress: + enabled: false + className: '' + annotations: {} + hosts: + - host: codeapi.local + paths: + - path: / + pathType: Prefix + tls: [] + + # Horizontal Pod Autoscaler + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + + # Extra environment variables + extraEnv: [] # ============================================================================= # WORKER-SANDBOX (Job processors + MicroVM Sandbox, scales based on queue depth) @@ -111,329 +111,355 @@ api: # Only requires /dev/kvm — no privileged mode, no Linux capabilities. # ============================================================================= workerSandbox: - enabled: true - replicaCount: 3 - - workerImage: - repository: codeapi-worker - tag: latest - pullPolicy: IfNotPresent - - sandboxImage: - repository: codeapi-sandbox-runner - tag: latest - pullPolicy: IfNotPresent - - # Enable libkrun microVM isolation. Requires /dev/kvm on the node. - # Set to false to fall back to chroot + NsJail (no guest kernel). - # Override to false until the node group has nested virtualisation enabled. - kvmEnabled: true - - # KVM device group ID for legacy hostPath mode. - kvmGid: 108 - - # Custom seccomp profile (defense-in-depth on top of NsJail's inner Kafel - # policy). When enabled, pods reference Localhost: profiles/nsjail.json, - # which must be pre-installed on every node at - # /var/lib/kubelet/seccomp/profiles/nsjail.json - see seccomp/README.md - # for the DaemonSet installer pattern. When disabled, KVM-mode pods use - # RuntimeDefault (still safer than no profile) and non-KVM-mode pods use - # Unconfined (forced by NsJail's pivot_root requirement). - seccomp: - enabled: false + enabled: true + replicaCount: 3 - # Optional Kubernetes device-plugin path for non-privileged KVM access. The - # device plugin should be deployed separately; this chart only requests its - # extended resource for sandbox-runner. - kvmDevicePlugin: - enabled: false - # Extended resource advertised by your KVM device plugin, e.g. - # "devices.kubevirt.io/kvm". Required when enabled=true (install fails - # fast when unset). - resourceName: "" - - # MicroVM launcher configuration (only used when kvmEnabled: true) - launcher: - vcpus: 2 - ramMib: 2048 - logLevel: 3 - filterVsockEnotconn: true - - # Resource limits (higher because it runs code + microVM) - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 2000m - memory: 3Gi - - serviceAccount: - create: true - name: "" - annotations: {} - - sandboxServiceAccount: - create: true - name: "" - automountServiceAccountToken: false - annotations: {} - - # Optional per-deployment scheduling. Empty values fall back to the chart-wide - # nodeSelector/tolerations/affinity below. Use sandboxRunner to place only the - # untrusted-code tier on KVM-capable nodes. - serviceWorker: - # Defaults to workerSandbox.replicaCount when unset. - # Keep this as fixed HA capacity; scale sandboxRunner for execution throughput. - replicaCount: null - inheritSharedScheduling: true - nodeSelector: {} - tolerations: [] - affinity: {} - - sandboxRunner: - # Defaults to workerSandbox.replicaCount when unset. - replicaCount: null - # Restart a sandbox-runner before launcher/VMM host-side fd retention can - # exhaust the process file descriptor limit. Set 0 to disable. - fdLivenessLimit: 40000 - inheritSharedScheduling: true - nodeSelector: {} - tolerations: [] - affinity: {} - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 20 - targetCPUUtilizationPercentage: 70 - targetMemoryUtilizationPercentage: 80 - - # Worker configuration - config: - # Throughput-mode defaults for launcher.vcpus=2. Bash/Node/Bun and - # bash-launched Python run through the "other" queue; direct Python is kept - # as a small compatibility lane. - pythonConcurrency: 1 - otherConcurrency: 8 - jobWindow: 1000 - jobTimeout: 25000 # 25 seconds - - # Health check port for the worker - healthPort: 3113 - - # Sandbox configuration (NsJail-based, uses SANDBOX_* env vars) - sandbox: - port: 2000 - maxProcessCount: 100 - runCpuTime: 15000 - runTimeout: 15000 - compileCpuTime: 10000 - compileTimeout: 10000 - outputMaxSize: 65536 - maxOpenFiles: 2048 - executeBodyLimit: 50mb - disableNetworking: true - maxConcurrentJobs: 8 - perJobUids: true - jobUidBase: 200000 - jobGidBase: 200000 - # Defaults to maxConcurrentJobs when unset. - jobUidCount: null - workspaceReaperMaxAgeSeconds: 3600 - - # Language runtime packages (mount from host or PVC) - packages: - # Use a PVC for packages - persistence: - enabled: true - size: 10Gi - accessMode: ReadWriteOnce - storageClass: "" - # If you have pre-built packages, specify the existing claim - existingClaim: "" - - # Init job to populate packages on first install - # Runs as a Helm pre-install/pre-upgrade hook - # Compiles Python from source and downloads Node/Bun binaries - initJob: - enabled: true - forceRebuild: false - image: - repository: codeapi-package-init + workerImage: + repository: codeapi-worker tag: latest pullPolicy: IfNotPresent - pythonVersion: "3.14.4" - nodeVersion: "24.15.0" - bunVersion: "1.3.14" - bashPackageVersion: "5.2.0" - backoffLimit: 2 - ttlSecondsAfterFinished: 3600 - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 2000m - memory: 4Gi - # Extra environment variables for worker containers - extraEnv: [] + sandboxImage: + repository: codeapi-sandbox-runner + tag: latest + pullPolicy: IfNotPresent - # Extra environment variables for the sandbox-runner container only. Keep - # control-plane, storage, Redis, and gateway encryption secrets out of this - # list; use it only for sandbox verifier/runtime knobs such as - # SANDBOX_LOG_LEVEL. Secret/token-like names are rejected by template - # validation; the manifest public verifier is chart-managed. - sandboxExtraEnv: [] + # Enable libkrun microVM isolation. Requires /dev/kvm on the node. + # Set to false to fall back to chroot + NsJail (no guest kernel). + # Override to false until the node group has nested virtualisation enabled. + kvmEnabled: true + + # KVM device group ID for legacy hostPath mode. + kvmGid: 108 + + # Custom seccomp profile (defense-in-depth on top of NsJail's inner Kafel + # policy). When enabled, pods reference Localhost: profiles/nsjail.json, + # which must be pre-installed on every node at + # /var/lib/kubelet/seccomp/profiles/nsjail.json - see seccomp/README.md + # for the DaemonSet installer pattern. When disabled, KVM-mode pods use + # RuntimeDefault (still safer than no profile) and non-KVM-mode pods use + # Unconfined (forced by NsJail's pivot_root requirement). + seccomp: + enabled: false + + # Optional Kubernetes device-plugin path for non-privileged KVM access. The + # device plugin should be deployed separately; this chart only requests its + # extended resource for sandbox-runner. + kvmDevicePlugin: + enabled: false + # Extended resource advertised by your KVM device plugin, e.g. + # "devices.kubevirt.io/kvm". Required when enabled=true (install fails + # fast when unset). + resourceName: '' + + # MicroVM launcher configuration (only used when kvmEnabled: true) + launcher: + vcpus: 2 + ramMib: 2048 + logLevel: 3 + filterVsockEnotconn: true + + # Resource limits (higher because it runs code + microVM) + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 3Gi + + serviceAccount: + create: true + name: '' + annotations: {} + + sandboxServiceAccount: + create: true + name: '' + automountServiceAccountToken: false + annotations: {} + + # Optional per-deployment scheduling. Empty values fall back to the chart-wide + # nodeSelector/tolerations/affinity below. Use sandboxRunner to place only the + # untrusted-code tier on KVM-capable nodes. + serviceWorker: + # Defaults to workerSandbox.replicaCount when unset. + # Keep this as fixed HA capacity; scale sandboxRunner for execution throughput. + replicaCount: null + inheritSharedScheduling: true + nodeSelector: {} + tolerations: [] + affinity: {} + + sandboxRunner: + # Defaults to workerSandbox.replicaCount when unset. + replicaCount: null + # Restart a sandbox-runner before launcher/VMM host-side fd retention can + # exhaust the process file descriptor limit. Set 0 to disable. + fdLivenessLimit: 40000 + inheritSharedScheduling: true + nodeSelector: {} + tolerations: [] + affinity: {} + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 20 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + + # Worker configuration + config: + # Throughput-mode defaults for launcher.vcpus=2. Bash/Node/Bun and + # bash-launched Python run through the "other" queue; direct Python is kept + # as a small compatibility lane. + pythonConcurrency: 1 + otherConcurrency: 8 + jobWindow: 1000 + jobTimeout: 25000 # 25 seconds + + # Health check port for the worker + healthPort: 3113 + + # Sandbox configuration (NsJail-based, uses SANDBOX_* env vars) + sandbox: + port: 2000 + maxProcessCount: 100 + runCpuTime: 15000 + runTimeout: 15000 + compileCpuTime: 10000 + compileTimeout: 10000 + outputMaxSize: 65536 + maxOpenFiles: 2048 + executeBodyLimit: 50mb + disableNetworking: true + maxConcurrentJobs: 8 + perJobUids: true + jobUidBase: 200000 + jobGidBase: 200000 + # Defaults to maxConcurrentJobs when unset. + jobUidCount: null + workspaceReaperMaxAgeSeconds: 3600 + + # Language runtime packages (mount from host or PVC) + packages: + # Use a PVC for packages + persistence: + enabled: true + size: 10Gi + accessMode: ReadWriteOnce + storageClass: '' + # If you have pre-built packages, specify the existing claim + existingClaim: '' + + # Init job to populate packages on first install + # Runs as a Helm pre-install/pre-upgrade hook + # Compiles Python from source and downloads Node/Bun binaries + initJob: + enabled: true + forceRebuild: false + image: + repository: codeapi-package-init + tag: latest + pullPolicy: IfNotPresent + pythonVersion: '3.14.4' + nodeVersion: '24.15.0' + bunVersion: '1.3.14' + bashPackageVersion: '5.2.0' + backoffLimit: 2 + ttlSecondsAfterFinished: 3600 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi + + # Extra environment variables for worker containers + extraEnv: [] + + # Extra environment variables for the sandbox-runner container only. Keep + # control-plane, storage, Redis, and gateway encryption secrets out of this + # list; use it only for sandbox verifier/runtime knobs such as + # SANDBOX_LOG_LEVEL. Secret/token-like names are rejected by template + # validation; the manifest public verifier is chart-managed. + sandboxExtraEnv: [] # ============================================================================= # FILE SERVER (S3/MinIO integration, stateless) # ============================================================================= fileServer: - enabled: true - replicaCount: 1 - - image: - repository: codeapi-file-server - tag: latest - pullPolicy: IfNotPresent - - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - - service: - type: ClusterIP - port: 3000 - - # ServiceAccount for IRSA (IAM Roles for Service Accounts) on EKS - # See: documentation/services/codeapi/deployment/irsa.md - serviceAccount: - create: true - name: "" - annotations: {} - # For IRSA on EKS, add: - # annotations: - # eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/codeapi-fileserver-role" - - # S3 configuration (when using AWS S3 instead of MinIO) - # Set minio.enabled=false when using external S3 - s3: - enabled: false - endpoint: "s3.amazonaws.com" - region: "us-east-1" - bucket: "" - useSSL: true - # Use IRSA for authentication (no static credentials needed) - useIrsa: false + enabled: true + replicaCount: 1 + + image: + repository: codeapi-file-server + tag: latest + pullPolicy: IfNotPresent + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + + service: + type: ClusterIP + port: 3000 + + # ServiceAccount for IRSA (IAM Roles for Service Accounts) on EKS + # See: documentation/services/codeapi/deployment/irsa.md + serviceAccount: + create: true + name: '' + annotations: {} + # For IRSA on EKS, add: + # annotations: + # eks.amazonaws.com/role-arn: "arn:aws:iam::ACCOUNT_ID:role/codeapi-fileserver-role" + + # S3 configuration (when using AWS S3 instead of MinIO) + # Set minio.enabled=false when using external S3 + s3: + enabled: false + endpoint: 's3.amazonaws.com' + region: 'us-east-1' + bucket: '' + useSSL: true + # Use IRSA for authentication (no static credentials needed) + useIrsa: false # ============================================================================= # TOOL CALL SERVER (programmatic tool calling, stateless) # ============================================================================= toolCallServer: - enabled: true - replicaCount: 1 - - image: - repository: codeapi-tool-call-server - tag: latest - pullPolicy: IfNotPresent - - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - - service: - type: ClusterIP - port: 3033 - - config: - requestTimeout: 300000 - sessionExpiry: 600 + enabled: true + replicaCount: 1 + + image: + repository: codeapi-tool-call-server + tag: latest + pullPolicy: IfNotPresent + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + + service: + type: ClusterIP + port: 3033 + + config: + requestTimeout: 300000 + sessionExpiry: 600 # ============================================================================= # EGRESS GATEWAY (only sandbox-runner outbound capability endpoint) # ============================================================================= egressGateway: - enabled: true - replicaCount: 1 + enabled: true + replicaCount: 1 - image: - repository: codeapi-egress-gateway - tag: latest - pullPolicy: IfNotPresent + image: + repository: codeapi-egress-gateway + tag: latest + pullPolicy: IfNotPresent - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi - service: - type: ClusterIP - port: 3190 + service: + type: ClusterIP + port: 3190 - config: - maxToolCallBytes: 1048576 + config: + maxToolCallBytes: 1048576 # ============================================================================= # REDIS (Job queue and state store) # ============================================================================= redis: - enabled: true # Set to false if using external Redis - - # Bitnami Redis chart configuration - architecture: standalone # Use 'replication' for HA - auth: - enabled: true - password: "changeme-in-production" - - master: - persistence: - enabled: true - size: 1Gi - - # If using external Redis, configure here: - # external: - # host: redis.example.com - # port: 6379 - # password: secret + enabled: true # Set to false if using external Redis + + # Bitnami Redis chart configuration + architecture: standalone # Use 'replication' for HA + auth: + enabled: true + password: 'changeme-in-production' + + master: + persistence: + enabled: true + size: 1Gi + + # If using external Redis, configure here: + # external: + # host: redis.example.com + # port: 6379 + # password: secret + + # Redis Cluster mode (e.g. GCP Memorystore cluster, AWS ElastiCache cluster). + # When enabled the service components connect using ioredis Cluster mode. + cluster: + enabled: false + # Comma-separated list of "host:port" startup nodes for ioredis Cluster + # discovery. Populate this when redis.enabled=false and the external Redis + # is a cluster endpoint. Example: + # nodes: "node1.example.com:6379,node2.example.com:6379,node3.example.com:6379" + nodes: '' + + # TLS settings for external managed Redis (e.g. GCP Memorystore with TLS). + # Ignored when redis.enabled=true (in-cluster Redis uses plain TCP). + tls: + enabled: false + # Name of a Kubernetes Secret that holds the CA certificate. + # The Secret must contain the key specified in caKey. + caSecretName: '' + caKey: 'ca' + # Mount path inside each pod for the CA certificate file. + caMountPath: '/etc/redis-tls/ca.crt' + + # Alternative DNS lookup – set to true for GCP Memorystore cluster or + # AWS ElastiCache clusters where standard DNS resolution causes TLS + # hostname-mismatch errors. Mirrors REDIS_USE_ALTERNATIVE_DNS_LOOKUP. + useAlternativeDnsLookup: false # ============================================================================= # MINIO (S3-compatible storage) # ============================================================================= minio: - enabled: true # Set to false if using external S3/MinIO + enabled: true # Set to false if using external S3/MinIO - # Bitnami MinIO chart configuration - mode: standalone # Use 'distributed' for HA - auth: - rootUser: minioadmin - rootPassword: "changeme-in-production" + # Bitnami MinIO chart configuration + mode: standalone # Use 'distributed' for HA + auth: + rootUser: minioadmin + rootPassword: 'changeme-in-production' - persistence: - enabled: true - size: 10Gi + persistence: + enabled: true + size: 10Gi - defaultBuckets: "codeapi-files" + defaultBuckets: 'codeapi-files' - # If using external MinIO/S3: - # external: - # endpoint: s3.amazonaws.com - # accessKey: AKIAXXXXXXXX - # secretKey: xxxxxxxx - # bucket: my-bucket - # useSSL: true + # If using external MinIO/S3: + # external: + # endpoint: s3.amazonaws.com + # accessKey: AKIAXXXXXXXX + # secretKey: xxxxxxxx + # bucket: my-bucket + # useSSL: true # ============================================================================= # COMMON SETTINGS @@ -450,40 +476,40 @@ affinity: {} # Pod disruption budgets podDisruptionBudget: - api: - enabled: false - minAvailable: 1 - workerSandbox: - enabled: false - minAvailable: 2 + api: + enabled: false + minAvailable: 1 + workerSandbox: + enabled: false + minAvailable: 2 # ============================================================================= # PROMETHEUS METRICS # ============================================================================= metrics: - enabled: false # Set to true to create PodMonitors for Prometheus scraping - interval: "30s" - scrapeTimeout: "10s" + enabled: false # Set to true to create PodMonitors for Prometheus scraping + interval: '30s' + scrapeTimeout: '10s' # ============================================================================= # NETWORK POLICIES # ============================================================================= networkPolicy: - enabled: true - otel: enabled: true - port: 4318 - # Match these to your collector's namespace and pod labels. - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: observability - podSelector: - matchLabels: - app.kubernetes.io/name: opentelemetry-collector - redis: - # Used only when redis.enabled=false. If empty, NetworkPolicy permits TCP - # egress on the configured Redis port to any destination because standard - # Kubernetes NetworkPolicy cannot target redis.external.host by DNS name. - externalCIDRs: [] - toolCallServer: - allowExternalEgress: false # Allow tool call server to reach external APIs + otel: + enabled: true + port: 4318 + # Match these to your collector's namespace and pod labels. + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: opentelemetry-collector + redis: + # Used only when redis.enabled=false. If empty, NetworkPolicy permits TCP + # egress on the configured Redis port to any destination because standard + # Kubernetes NetworkPolicy cannot target redis.external.host by DNS name. + externalCIDRs: [] + toolCallServer: + allowExternalEgress: false # Allow tool call server to reach external APIs diff --git a/service/.env.example b/service/.env.example index e06ec4c..6095ca2 100644 --- a/service/.env.example +++ b/service/.env.example @@ -8,6 +8,19 @@ FILE_SERVER_PORT=3000 REDIS_HOST=localhost REDIS_PORT=6379 REDIS_PASSWORD=mysecretpassword +# Redis Cluster mode (GCP Memorystore cluster, AWS ElastiCache cluster, etc.) +# Set USE_REDIS_CLUSTER=true to enable, or provide a comma-separated list of +# host:port startup nodes in REDIS_HOST (cluster mode is then auto-detected). +# Example cluster REDIS_HOST: node1.example.com:6379,node2.example.com:6379 +# USE_REDIS_CLUSTER=false +# TLS: set REDIS_TLS=true for managed Redis with TLS (e.g. GCP Memorystore). +# REDIS_TLS=false +# CA certificate path for TLS with certificate validation (recommended over +# REDIS_TLS alone which disables cert verification). +# REDIS_CA=/path/to/redis-ca.crt +# Alternative DNS lookup required for AWS ElastiCache clusters with TLS and +# for some GCP Memorystore cluster configurations. +# REDIS_USE_ALTERNATIVE_DNS_LOOKUP=false # ----------------------------------------------------------------------------- # Stripe: https://shipfa.st/docs/features/payments diff --git a/service/src/config.ts b/service/src/config.ts index 5abc57c..f129846 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -4,157 +4,235 @@ import { nanoid } from 'nanoid'; import type * as t from './types'; import { Languages } from './enum'; -export const languageConfig: Record = { - [Languages.bash]: { language: 'bash', version: '5.2.0', fileName: 'script.sh' }, - [Languages.js]: { language: 'bun-js', version: '1.3.14', fileName: 'index.js' }, - [Languages.node]: { language: 'node', version: '24.15.0', fileName: 'index.js' }, - [Languages.py]: { language: 'python', version: '3.14.4', fileName: 'main.py' }, - [Languages.ts]: { language: 'bun-ts', version: '1.3.14', fileName: 'main.ts' }, +export const languageConfig: Record< + Languages | string, + t.LanguageConfig | undefined +> = { + [Languages.bash]: { + language: 'bash', + version: '5.2.0', + fileName: 'script.sh', + }, + [Languages.js]: { + language: 'bun-js', + version: '1.3.14', + fileName: 'index.js', + }, + [Languages.node]: { + language: 'node', + version: '24.15.0', + fileName: 'index.js', + }, + [Languages.py]: { + language: 'python', + version: '3.14.4', + fileName: 'main.py', + }, + [Languages.ts]: { + language: 'bun-ts', + version: '1.3.14', + fileName: 'main.ts', + }, }; const languageAliases: Record = { - // Python - python: Languages.py, - py: Languages.py, - - // JavaScript (Bun) - javascript: Languages.js, - js: Languages.js, - 'bun-js': Languages.js, - bun: Languages.js, - - // JavaScript (Node.js) - node: Languages.node, - nodejs: Languages.node, - 'node-js': Languages.node, - 'node-javascript': Languages.node, - - // TypeScript (Bun) - typescript: Languages.ts, - ts: Languages.ts, - 'bun-ts': Languages.ts, - 'bun-typescript': Languages.ts, - - // Bash - bash: Languages.bash, - sh: Languages.bash, + // Python + python: Languages.py, + py: Languages.py, + + // JavaScript (Bun) + javascript: Languages.js, + js: Languages.js, + 'bun-js': Languages.js, + bun: Languages.js, + + // JavaScript (Node.js) + node: Languages.node, + nodejs: Languages.node, + 'node-js': Languages.node, + 'node-javascript': Languages.node, + + // TypeScript (Bun) + typescript: Languages.ts, + ts: Languages.ts, + 'bun-ts': Languages.ts, + 'bun-typescript': Languages.ts, + + // Bash + bash: Languages.bash, + sh: Languages.bash, }; export function resolveLanguage(lang: string): Languages | undefined { - return languageAliases[lang.toLowerCase()]; + return languageAliases[lang.toLowerCase()]; } const defaultJobTimeoutMs = Number(process.env.JOB_TIMEOUT) || 300000; -const defaultMaxFileSize = Number(process.env.MAX_FILE_SIZE) || 25 * 1024 * 1024; -const defaultExecutionManifestTtlSeconds = Math.min(Math.ceil((defaultJobTimeoutMs + 60000) / 1000), 600); +const defaultMaxFileSize = + Number(process.env.MAX_FILE_SIZE) || 25 * 1024 * 1024; +const defaultExecutionManifestTtlSeconds = Math.min( + Math.ceil((defaultJobTimeoutMs + 60000) / 1000), + 600, +); const EGRESS_GRANT_GRACE_MS = 10 * 60 * 1000; -export function resolveEgressGrantTtlSeconds(rawTtlSeconds: string | undefined, jobTimeoutMs: number): number { - const defaultTtlSeconds = Math.max(1, Math.ceil((jobTimeoutMs + EGRESS_GRANT_GRACE_MS) / 1000)); - if (rawTtlSeconds == null || rawTtlSeconds.trim() === '') { - return defaultTtlSeconds; - } +export function resolveEgressGrantTtlSeconds( + rawTtlSeconds: string | undefined, + jobTimeoutMs: number, +): number { + const defaultTtlSeconds = Math.max( + 1, + Math.ceil((jobTimeoutMs + EGRESS_GRANT_GRACE_MS) / 1000), + ); + if (rawTtlSeconds == null || rawTtlSeconds.trim() === '') { + return defaultTtlSeconds; + } - const configuredTtlSeconds = Number(rawTtlSeconds); - if (!Number.isFinite(configuredTtlSeconds) || configuredTtlSeconds <= 0) { - return defaultTtlSeconds; - } + const configuredTtlSeconds = Number(rawTtlSeconds); + if (!Number.isFinite(configuredTtlSeconds) || configuredTtlSeconds <= 0) { + return defaultTtlSeconds; + } - return Math.max(1, Math.ceil(configuredTtlSeconds)); + return Math.max(1, Math.ceil(configuredTtlSeconds)); } export const env = { - PORT: process.env.SERVICE_PORT ?? 3112, - LOCAL_MODE: process.env.LOCAL_MODE === 'true', - HARDENED_SANDBOX_MODE: process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', - INSTANCE_ID: process.env.INSTANCE_ID ?? nanoid(), - HTTP_JSON_LIMIT: process.env.CODEAPI_HTTP_JSON_LIMIT ?? '50mb', - SANDBOX_ENDPOINT: process.env.SANDBOX_ENDPOINT ?? 'http://localhost:2000/api/v2', - EGRESS_GATEWAY_URL: process.env.EGRESS_GATEWAY_URL ?? '', - FILE_SERVER_URL: process.env.FILE_SERVER_URL ?? 'http://localhost:3000', - TOOL_CALL_SERVER_URL: process.env.TOOL_CALL_SERVER_URL ?? 'http://localhost:3033', - EGRESS_GATEWAY_PORT: Number(process.env.EGRESS_GATEWAY_PORT) || 3190, - EGRESS_GATEWAY_FILE_SERVER_URL: process.env.EGRESS_GATEWAY_FILE_SERVER_URL ?? process.env.FILE_SERVER_URL ?? 'http://localhost:3000', - EGRESS_GATEWAY_TOOL_CALL_SERVER_URL: process.env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL ?? process.env.TOOL_CALL_SERVER_URL ?? 'http://localhost:3033', - EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES: Number(process.env.EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES) || 1024 * 1024, - EGRESS_GATEWAY_MAX_FILE_BYTES: Number(process.env.EGRESS_GATEWAY_MAX_FILE_BYTES ?? process.env.SANDBOX_MAX_FILE_SIZE) || 10_000_000, - EGRESS_GATEWAY_MAX_PATH_LENGTH: Number(process.env.EGRESS_GATEWAY_MAX_PATH_LENGTH ?? process.env.SANDBOX_MAX_PATH_LENGTH) || 256, - EGRESS_GATEWAY_MAX_NESTING_DEPTH: Number(process.env.EGRESS_GATEWAY_MAX_NESTING_DEPTH ?? process.env.SANDBOX_MAX_NESTING_DEPTH) || 10, - EGRESS_GATEWAY_REQUEST_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS) || 30_000, - EGRESS_GATEWAY_REVOKE_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS) || 5_000, - EGRESS_LEDGER_REQUIRED: process.env.CODEAPI_EGRESS_LEDGER_REQUIRED === 'true' || process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', - EGRESS_LEDGER_TTL_GRACE_SECONDS: Number(process.env.CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS) || 300, - EGRESS_GRANT_SECRET: process.env.CODEAPI_EGRESS_GRANT_SECRET ?? '', - EGRESS_GRANT_TTL_SECONDS: resolveEgressGrantTtlSeconds(process.env.EGRESS_GRANT_TTL_SECONDS, defaultJobTimeoutMs), - PYTHON_CONCURRENCY: Number(process.env.PYTHON_CONCURRENCY) || 1, - OTHER_CONCURRENCY: Number(process.env.OTHER_CONCURRENCY) || 8, - JOB_WINDOW: Number(process.env.JOB_WINDOW) || 1000, - MAX_UPLOAD_CHECKS: Number(process.env.MAX_UPLOAD_CHECKS) || 14, - MAX_UPLOAD_WAIT: Number(process.env.MAX_UPLOAD_WAIT) || 500, - MAX_FILE_SIZE: defaultMaxFileSize, - JOB_TIMEOUT: defaultJobTimeoutMs, // 5 minutes (increased for complex matplotlib rendering) - // Execution Rate Limits - EXEC_LIMIT_WINDOW: Number(process.env.RATE_LIMIT_WINDOW) || 30 * 1000, // 30 seconds - EXEC_MAX_REQUESTS: Number(process.env.MAX_REQUESTS) || 20, // execution requests per window - // Upload Rate Limits - UPLOAD_LIMIT_WINDOW: Number(process.env.UPLOAD_LIMIT_WINDOW) || 5 * 60 * 1000, // 5 minutes - UPLOAD_MAX_REQUESTS: Number(process.env.UPLOAD_MAX_REQUESTS) || 30, // 30 uploads per 5 minutes - // Download Rate Limits - DOWNLOAD_LIMIT_WINDOW: Number(process.env.DOWNLOAD_LIMIT_WINDOW) || 60 * 1000, // 1 minute - DOWNLOAD_MAX_REQUESTS: Number(process.env.DOWNLOAD_MAX_REQUESTS) || 60, // 60 downloads per minute - // Files List Rate Limits - FETCH_LIMIT_WINDOW: Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, // 1 minute - FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute - // Redis Key Cache Config - SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, - /** Strict tenant isolation. When true, sessionKey resolution fails closed - * (500) on requests whose auth context lacks `tenantId`, instead of - * silently falling back to the `'legacy'` tenant prefix. Default OFF in - * code so single-tenant deploys without an auth tenancy concept keep - * working; multi-tenant deploys MUST set this to `true` before any tenant - * is multi-homed, otherwise a missing tenantId would silently bucket - * cross-tenant requests under the same `'legacy'` prefix. */ - TENANT_ISOLATION_STRICT: process.env.CODEAPI_TENANT_ISOLATION_STRICT === 'true', - // Signed execution manifests. Prefer private/public key mode for split-runner - // deployments so sandbox-runner receives only a verifier, not a signing secret. - EXECUTION_MANIFEST_PRIVATE_KEY: process.env.CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY ?? '', - EXECUTION_MANIFEST_PUBLIC_KEY: process.env.CODEAPI_EXECUTION_MANIFEST_PUBLIC_KEY ?? '', - // Legacy HMAC fallback for non-split deployments. Do not mount into sandbox-runner. - EXECUTION_MANIFEST_SECRET: process.env.CODEAPI_EXECUTION_MANIFEST_SECRET ?? '', - EXECUTION_MANIFEST_TTL_SECONDS: Math.min( - Number(process.env.EXECUTION_MANIFEST_TTL_SECONDS) || defaultExecutionManifestTtlSeconds, - 600, - ), - EXECUTION_MANIFEST_MAX_UPLOAD_BYTES: Number(process.env.EXECUTION_MANIFEST_MAX_UPLOAD_BYTES) || defaultMaxFileSize, - EXECUTION_MANIFEST_MAX_OUTPUT_FILES: Number(process.env.EXECUTION_MANIFEST_MAX_OUTPUT_FILES) || 50, - EXECUTION_MANIFEST_MAX_REQUESTS: Number(process.env.EXECUTION_MANIFEST_MAX_REQUESTS) || 1000, - // Redis - Alternative DNS Lookup for AWS ElastiCache TLS connections - REDIS_USE_ALTERNATIVE_DNS_LOOKUP: process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true', - /** - * Programmatic Tool Calling execution model. - * - `replay` (default): Temporal-style replay. Sandbox exits between round-trips; - * tool results are persisted in Redis and replayed into a fresh sandbox on each - * continuation until the code either completes or surfaces new tool calls. - * Safe to scale horizontally since all state lives in Redis. - * - `blocking`: legacy path. Sandbox process stays alive across tool round-trips - * via a long-polling HTTP callback through the Tool Call Server. Retained as - * an explicit opt-in during rollout; scheduled for removal in a follow-up. - */ - PTC_MODE: (process.env.PTC_MODE === 'blocking' ? 'blocking' : 'replay') as 'replay' | 'blocking', - PTC_DEBUG: process.env.PTC_DEBUG === 'true', + PORT: process.env.SERVICE_PORT ?? 3112, + LOCAL_MODE: process.env.LOCAL_MODE === 'true', + HARDENED_SANDBOX_MODE: process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', + INSTANCE_ID: process.env.INSTANCE_ID ?? nanoid(), + HTTP_JSON_LIMIT: process.env.CODEAPI_HTTP_JSON_LIMIT ?? '50mb', + SANDBOX_ENDPOINT: + process.env.SANDBOX_ENDPOINT ?? 'http://localhost:2000/api/v2', + EGRESS_GATEWAY_URL: process.env.EGRESS_GATEWAY_URL ?? '', + FILE_SERVER_URL: process.env.FILE_SERVER_URL ?? 'http://localhost:3000', + TOOL_CALL_SERVER_URL: + process.env.TOOL_CALL_SERVER_URL ?? 'http://localhost:3033', + EGRESS_GATEWAY_PORT: Number(process.env.EGRESS_GATEWAY_PORT) || 3190, + EGRESS_GATEWAY_FILE_SERVER_URL: + process.env.EGRESS_GATEWAY_FILE_SERVER_URL ?? + process.env.FILE_SERVER_URL ?? + 'http://localhost:3000', + EGRESS_GATEWAY_TOOL_CALL_SERVER_URL: + process.env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL ?? + process.env.TOOL_CALL_SERVER_URL ?? + 'http://localhost:3033', + EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES: + Number(process.env.EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES) || 1024 * 1024, + EGRESS_GATEWAY_MAX_FILE_BYTES: + Number( + process.env.EGRESS_GATEWAY_MAX_FILE_BYTES ?? + process.env.SANDBOX_MAX_FILE_SIZE, + ) || 10_000_000, + EGRESS_GATEWAY_MAX_PATH_LENGTH: + Number( + process.env.EGRESS_GATEWAY_MAX_PATH_LENGTH ?? + process.env.SANDBOX_MAX_PATH_LENGTH, + ) || 256, + EGRESS_GATEWAY_MAX_NESTING_DEPTH: + Number( + process.env.EGRESS_GATEWAY_MAX_NESTING_DEPTH ?? + process.env.SANDBOX_MAX_NESTING_DEPTH, + ) || 10, + EGRESS_GATEWAY_REQUEST_TIMEOUT_MS: + Number(process.env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS) || 30_000, + EGRESS_GATEWAY_REVOKE_TIMEOUT_MS: + Number(process.env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS) || 5_000, + EGRESS_LEDGER_REQUIRED: + process.env.CODEAPI_EGRESS_LEDGER_REQUIRED === 'true' || + process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', + EGRESS_LEDGER_TTL_GRACE_SECONDS: + Number(process.env.CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS) || 300, + EGRESS_GRANT_SECRET: process.env.CODEAPI_EGRESS_GRANT_SECRET ?? '', + EGRESS_GRANT_TTL_SECONDS: resolveEgressGrantTtlSeconds( + process.env.EGRESS_GRANT_TTL_SECONDS, + defaultJobTimeoutMs, + ), + PYTHON_CONCURRENCY: Number(process.env.PYTHON_CONCURRENCY) || 1, + OTHER_CONCURRENCY: Number(process.env.OTHER_CONCURRENCY) || 8, + JOB_WINDOW: Number(process.env.JOB_WINDOW) || 1000, + MAX_UPLOAD_CHECKS: Number(process.env.MAX_UPLOAD_CHECKS) || 14, + MAX_UPLOAD_WAIT: Number(process.env.MAX_UPLOAD_WAIT) || 500, + MAX_FILE_SIZE: defaultMaxFileSize, + JOB_TIMEOUT: defaultJobTimeoutMs, // 5 minutes (increased for complex matplotlib rendering) + // Execution Rate Limits + EXEC_LIMIT_WINDOW: Number(process.env.RATE_LIMIT_WINDOW) || 30 * 1000, // 30 seconds + EXEC_MAX_REQUESTS: Number(process.env.MAX_REQUESTS) || 20, // execution requests per window + // Upload Rate Limits + UPLOAD_LIMIT_WINDOW: + Number(process.env.UPLOAD_LIMIT_WINDOW) || 5 * 60 * 1000, // 5 minutes + UPLOAD_MAX_REQUESTS: Number(process.env.UPLOAD_MAX_REQUESTS) || 30, // 30 uploads per 5 minutes + // Download Rate Limits + DOWNLOAD_LIMIT_WINDOW: + Number(process.env.DOWNLOAD_LIMIT_WINDOW) || 60 * 1000, // 1 minute + DOWNLOAD_MAX_REQUESTS: Number(process.env.DOWNLOAD_MAX_REQUESTS) || 60, // 60 downloads per minute + // Files List Rate Limits + FETCH_LIMIT_WINDOW: Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, // 1 minute + FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute + // Redis Key Cache Config + SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, + /** Strict tenant isolation. When true, sessionKey resolution fails closed + * (500) on requests whose auth context lacks `tenantId`, instead of + * silently falling back to the `'legacy'` tenant prefix. Default OFF in + * code so single-tenant deploys without an auth tenancy concept keep + * working; multi-tenant deploys MUST set this to `true` before any tenant + * is multi-homed, otherwise a missing tenantId would silently bucket + * cross-tenant requests under the same `'legacy'` prefix. */ + TENANT_ISOLATION_STRICT: + process.env.CODEAPI_TENANT_ISOLATION_STRICT === 'true', + // Signed execution manifests. Prefer private/public key mode for split-runner + // deployments so sandbox-runner receives only a verifier, not a signing secret. + EXECUTION_MANIFEST_PRIVATE_KEY: + process.env.CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY ?? '', + EXECUTION_MANIFEST_PUBLIC_KEY: + process.env.CODEAPI_EXECUTION_MANIFEST_PUBLIC_KEY ?? '', + // Legacy HMAC fallback for non-split deployments. Do not mount into sandbox-runner. + EXECUTION_MANIFEST_SECRET: + process.env.CODEAPI_EXECUTION_MANIFEST_SECRET ?? '', + EXECUTION_MANIFEST_TTL_SECONDS: Math.min( + Number(process.env.EXECUTION_MANIFEST_TTL_SECONDS) || + defaultExecutionManifestTtlSeconds, + 600, + ), + EXECUTION_MANIFEST_MAX_UPLOAD_BYTES: + Number(process.env.EXECUTION_MANIFEST_MAX_UPLOAD_BYTES) || + defaultMaxFileSize, + EXECUTION_MANIFEST_MAX_OUTPUT_FILES: + Number(process.env.EXECUTION_MANIFEST_MAX_OUTPUT_FILES) || 50, + EXECUTION_MANIFEST_MAX_REQUESTS: + Number(process.env.EXECUTION_MANIFEST_MAX_REQUESTS) || 1000, + // Redis - Cluster mode (GCP Memorystore cluster, AWS ElastiCache cluster, etc.) + USE_REDIS_CLUSTER: process.env.USE_REDIS_CLUSTER === 'true', + // Redis - Alternative DNS Lookup for AWS ElastiCache TLS connections + REDIS_USE_ALTERNATIVE_DNS_LOOKUP: + process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true', + /** + * Programmatic Tool Calling execution model. + * - `replay` (default): Temporal-style replay. Sandbox exits between round-trips; + * tool results are persisted in Redis and replayed into a fresh sandbox on each + * continuation until the code either completes or surfaces new tool calls. + * Safe to scale horizontally since all state lives in Redis. + * - `blocking`: legacy path. Sandbox process stays alive across tool round-trips + * via a long-polling HTTP callback through the Tool Call Server. Retained as + * an explicit opt-in during rollout; scheduled for removal in a follow-up. + */ + PTC_MODE: (process.env.PTC_MODE === 'blocking' ? 'blocking' : 'replay') as + | 'replay' + | 'blocking', + PTC_DEBUG: process.env.PTC_DEBUG === 'true', }; const default_run_memory_limit = 256 * 1024 * 1024; type PlanLimit = { - run_memory_limit?: number; - max_file_size?: number; + run_memory_limit?: number; + max_file_size?: number; }; type PlanLimits = { - default: Required; + default: Required; } & { - [key: string]: PlanLimit | undefined; + [key: string]: PlanLimit | undefined; }; /** @@ -162,26 +240,38 @@ type PlanLimits = { * JSON object keyed by the `plan_id` JWT claim. Unknown or absent plan ids * fall back to the default tier, which is the only entry defined in code. */ -export function parsePlanLimits(raw: string | undefined): Record { - if (raw == null || raw.trim() === '') { - return {}; - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (error) { - throw new Error(`CODEAPI_PLAN_LIMITS is not valid JSON: ${(error as Error).message}`); - } - if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error('CODEAPI_PLAN_LIMITS must be a JSON object keyed by plan id'); - } - return parsed as Record; +export function parsePlanLimits( + raw: string | undefined, +): Record { + if (raw == null || raw.trim() === '') { + return {}; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new Error( + `CODEAPI_PLAN_LIMITS is not valid JSON: ${(error as Error).message}`, + ); + } + if ( + parsed === null || + typeof parsed !== 'object' || + Array.isArray(parsed) + ) { + throw new Error( + 'CODEAPI_PLAN_LIMITS must be a JSON object keyed by plan id', + ); + } + return parsed as Record; } export const planLimits: PlanLimits = { - ...parsePlanLimits(process.env.CODEAPI_PLAN_LIMITS), - default: { - run_memory_limit: Number(process.env.SANDBOX_RUN_MEMORY_LIMIT) || default_run_memory_limit, - max_file_size: env.MAX_FILE_SIZE, - }, + ...parsePlanLimits(process.env.CODEAPI_PLAN_LIMITS), + default: { + run_memory_limit: + Number(process.env.SANDBOX_RUN_MEMORY_LIMIT) || + default_run_memory_limit, + max_file_size: env.MAX_FILE_SIZE, + }, }; diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index 24dda87..6ceaec3 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -1,343 +1,432 @@ -import IORedis from 'ioredis'; -import type { CommonRedisOptions } from 'ioredis'; -import type * as tls from 'tls'; +import type { Redis, Cluster } from 'ioredis'; import { env } from './config'; import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; +import { + createRedisConnection, + isClusterMode, + type RedisClient, +} from './redis-connection'; type LedgerStatus = 'active' | 'revoked'; export interface EgressLedgerRecord { - grant_id: string; - exec_id: string; - status: LedgerStatus; - exp: number; - revoked_at?: number; - revoke_reason?: string; - input_files: EgressGrantClaims['input_files']; - read_sessions: string[]; - output_session_id: string; - max_upload_bytes: number; - max_output_files: number; - max_requests: number; - request_count: number; - read_count: number; - upload_count: number; - tool_call_count: number; - uploaded_bytes: number; - output_file_ids: string[]; + grant_id: string; + exec_id: string; + status: LedgerStatus; + exp: number; + revoked_at?: number; + revoke_reason?: string; + input_files: EgressGrantClaims['input_files']; + read_sessions: string[]; + output_session_id: string; + max_upload_bytes: number; + max_output_files: number; + max_requests: number; + request_count: number; + read_count: number; + upload_count: number; + tool_call_count: number; + uploaded_bytes: number; + output_file_ids: string[]; } -let redis: IORedis | null = null; +let redis: RedisClient | null = null; const LEDGER_MUTATION_ATTEMPTS = 32; -const LEDGER_MUTATION_POOL_SIZE = Math.max(1, Number(process.env.CODEAPI_EGRESS_LEDGER_MUTATION_CONNECTIONS) || 32); +const LEDGER_MUTATION_POOL_SIZE = Math.max( + 1, + Number(process.env.CODEAPI_EGRESS_LEDGER_MUTATION_CONNECTIONS) || 32, +); type MutationConnectionWaiter = { - resolve: (client: IORedis) => void; - reject: (error: Error) => void; + resolve: (client: RedisClient) => void; + reject: (error: Error) => void; }; -const mutationConnections = new Set(); -let idleMutationConnections: IORedis[] = []; +const mutationConnections = new Set(); +let idleMutationConnections: RedisClient[] = []; let mutationConnectionWaiters: MutationConnectionWaiter[] = []; -export function setEgressLedgerRedisForTest(client: IORedis | null): void { - resetMutationConnections(); - redis = client; +export function setEgressLedgerRedisForTest( + client: Redis | Cluster | null, +): void { + resetMutationConnections(); + redis = client; } function ledgerKey(grantId: string): string { - return `codeapi:egress:grant:${grantId}`; + return `codeapi:egress:grant:${grantId}`; } function ttlSeconds(exp: number): number { - return Math.max(1, exp - Math.floor(Date.now() / 1000) + env.EGRESS_LEDGER_TTL_GRACE_SECONDS); + return Math.max( + 1, + exp - + Math.floor(Date.now() / 1000) + + env.EGRESS_LEDGER_TTL_GRACE_SECONDS, + ); } -function redisConnection(): IORedis { - if (redis) return redis; - const retryStrategy: CommonRedisOptions['retryStrategy'] = times => { +const ledgerRetryStrategy = (times: number): number | null => { if (times > 5) return null; return 2000; - }; - redis = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - maxRetriesPerRequest: 1, - retryStrategy, - enableReadyCheck: true, - connectTimeout: 10000, - ...redisKeepAliveOptions(), - tls: process.env.REDIS_TLS === 'true' - ? { rejectUnauthorized: false } as tls.ConnectionOptions - : undefined, - ...(env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}), - }); - redis.on('error', error => logger.error('Egress ledger Redis error', { error })); - return redis; +}; + +function redisConnection(): RedisClient { + if (redis) return redis; + redis = createRedisConnection({ + maxRetriesPerRequest: 1, + retryStrategy: ledgerRetryStrategy, + enableReadyCheck: true, + }); + redis.on('error', error => + logger.error('Egress ledger Redis error', { error }), + ); + return redis; } function resetMutationConnections(): void { - const resetError = new Error('Egress ledger Redis connection reset'); - for (const waiter of mutationConnectionWaiters) { - waiter.reject(resetError); - } - mutationConnectionWaiters = []; - idleMutationConnections = []; - for (const client of mutationConnections) { - client.disconnect(); - } - mutationConnections.clear(); + const resetError = new Error('Egress ledger Redis connection reset'); + for (const waiter of mutationConnectionWaiters) { + waiter.reject(resetError); + } + mutationConnectionWaiters = []; + idleMutationConnections = []; + for (const client of mutationConnections) { + client.disconnect(); + } + mutationConnections.clear(); } -async function dedicatedMutationConnection(): Promise { - while (idleMutationConnections.length > 0) { - const client = idleMutationConnections.pop()!; - if (client.status !== 'end') { - return client; +async function dedicatedMutationConnection(): Promise { + while (idleMutationConnections.length > 0) { + const client = idleMutationConnections.pop()!; + if (client.status !== 'end') { + return client; + } + mutationConnections.delete(client); } - mutationConnections.delete(client); - } - if (mutationConnections.size < LEDGER_MUTATION_POOL_SIZE) { - return createMutationConnection(); - } + if (mutationConnections.size < LEDGER_MUTATION_POOL_SIZE) { + return createMutationConnection(); + } - return new Promise((resolve, reject) => { - mutationConnectionWaiters.push({ resolve, reject }); - }); + return new Promise((resolve, reject) => { + mutationConnectionWaiters.push({ resolve, reject }); + }); } -function createMutationConnection(): IORedis { - const client = redisConnection().duplicate(); - mutationConnections.add(client); - client.on('error', error => logger.error('Egress ledger mutation Redis error', { error })); - return client; +function createMutationConnection(): RedisClient { + // Cluster does not expose .duplicate(); create a fresh connection instead. + const client: RedisClient = isClusterMode() + ? createRedisConnection({ + maxRetriesPerRequest: 1, + retryStrategy: ledgerRetryStrategy, + enableReadyCheck: true, + }) + : (redisConnection() as Redis).duplicate(); + mutationConnections.add(client); + client.on('error', error => + logger.error('Egress ledger mutation Redis error', { error }), + ); + return client; } -function releaseMutationConnection(client: IORedis): void { - if (!mutationConnections.has(client) || client.status === 'end') { - mutationConnections.delete(client); +function releaseMutationConnection(client: RedisClient): void { + if (!mutationConnections.has(client) || client.status === 'end') { + mutationConnections.delete(client); + const waiter = mutationConnectionWaiters.shift(); + if (waiter) { + try { + waiter.resolve(createMutationConnection()); + } catch (error) { + waiter.reject( + error instanceof Error ? error : new Error(String(error)), + ); + } + } + return; + } + const waiter = mutationConnectionWaiters.shift(); if (waiter) { - try { - waiter.resolve(createMutationConnection()); - } catch (error) { - waiter.reject(error instanceof Error ? error : new Error(String(error))); - } + waiter.resolve(client); + return; } - return; - } - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - waiter.resolve(client); - return; - } - - idleMutationConnections.push(client); + idleMutationConnections.push(client); } function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise(resolve => setTimeout(resolve, ms)); } export async function pingEgressLedger(): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().ping(); + if (!env.EGRESS_LEDGER_REQUIRED) return; + await redisConnection().ping(); } function recordFromGrant(grant: EgressGrantClaims): EgressLedgerRecord { - return { - grant_id: grant.grant_id, - exec_id: grant.exec_id, - status: 'active', - exp: grant.exp, - input_files: grant.input_files, - read_sessions: grant.read_sessions, - output_session_id: grant.output_session_id, - max_upload_bytes: grant.max_upload_bytes, - max_output_files: grant.max_output_files, - max_requests: grant.max_requests, - request_count: 0, - read_count: 0, - upload_count: 0, - tool_call_count: 0, - uploaded_bytes: 0, - output_file_ids: [], - }; + return { + grant_id: grant.grant_id, + exec_id: grant.exec_id, + status: 'active', + exp: grant.exp, + input_files: grant.input_files, + read_sessions: grant.read_sessions, + output_session_id: grant.output_session_id, + max_upload_bytes: grant.max_upload_bytes, + max_output_files: grant.max_output_files, + max_requests: grant.max_requests, + request_count: 0, + read_count: 0, + upload_count: 0, + tool_call_count: 0, + uploaded_bytes: 0, + output_file_ids: [], + }; } -export async function createEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); - } - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - ); +export async function createEgressLedger( + grant: EgressGrantClaims, +): Promise { + if (!grant.grant_id) { + throw new EgressGrantError('malformed', 'Egress grant id is required'); + } + if (!env.EGRESS_LEDGER_REQUIRED) return; + await redisConnection().set( + ledgerKey(grant.grant_id), + JSON.stringify(recordFromGrant(grant)), + 'EX', + ttlSeconds(grant.exp), + ); } -export async function ensureEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); - } - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - 'NX', - ); +export async function ensureEgressLedger( + grant: EgressGrantClaims, +): Promise { + if (!grant.grant_id) { + throw new EgressGrantError('malformed', 'Egress grant id is required'); + } + if (!env.EGRESS_LEDGER_REQUIRED) return; + await redisConnection().set( + ledgerKey(grant.grant_id), + JSON.stringify(recordFromGrant(grant)), + 'EX', + ttlSeconds(grant.exp), + 'NX', + ); } async function loadRecord(grantId: string): Promise { - const raw = await redisConnection().get(ledgerKey(grantId)); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - return JSON.parse(raw) as EgressLedgerRecord; + const raw = await redisConnection().get(ledgerKey(grantId)); + if (!raw) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger record is missing', + ); + } + return JSON.parse(raw) as EgressLedgerRecord; } -function assertActive(record: EgressLedgerRecord, grant: Pick): void { - if (record.grant_id !== grant.grant_id || record.exec_id !== grant.exec_id) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record does not match token'); - } - if (record.status !== 'active') { - throw new EgressGrantError('scope_mismatch', 'Egress grant has been revoked'); - } - if (record.exp <= Math.floor(Date.now() / 1000)) { - throw new EgressGrantError('expired', 'Egress grant is expired'); - } +function assertActive( + record: EgressLedgerRecord, + grant: Pick, +): void { + if ( + record.grant_id !== grant.grant_id || + record.exec_id !== grant.exec_id + ) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger record does not match token', + ); + } + if (record.status !== 'active') { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant has been revoked', + ); + } + if (record.exp <= Math.floor(Date.now() / 1000)) { + throw new EgressGrantError('expired', 'Egress grant is expired'); + } } async function mutateRecord( - grant: EgressGrantClaims, - mutate: (record: EgressLedgerRecord) => void, + grant: EgressGrantClaims, + mutate: (record: EgressLedgerRecord) => void, ): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) { - return recordFromGrant(grant); - } - const client = await dedicatedMutationConnection(); - const key = ledgerKey(grant.grant_id); - try { - for (let i = 0; i < LEDGER_MUTATION_ATTEMPTS; i++) { - await client.watch(key); - let record: EgressLedgerRecord; - try { - const raw = await client.get(key); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - record = JSON.parse(raw) as EgressLedgerRecord; - assertActive(record, grant); - mutate(record); - if (record.request_count > record.max_requests) { - throw new EgressGrantError('scope_mismatch', 'Egress grant request budget exceeded'); + if (!env.EGRESS_LEDGER_REQUIRED) { + return recordFromGrant(grant); + } + const client = await dedicatedMutationConnection(); + const key = ledgerKey(grant.grant_id); + try { + for (let i = 0; i < LEDGER_MUTATION_ATTEMPTS; i++) { + await client.watch(key); + let record: EgressLedgerRecord; + try { + const raw = await client.get(key); + if (!raw) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger record is missing', + ); + } + record = JSON.parse(raw) as EgressLedgerRecord; + assertActive(record, grant); + mutate(record); + if (record.request_count > record.max_requests) { + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant request budget exceeded', + ); + } + } catch (error) { + await client.unwatch().catch(unwatchError => { + logger.warn( + 'Failed to clear egress ledger WATCH after rejected mutation', + { error: unwatchError }, + ); + }); + throw error; + } + const result = await client + .multi() + .set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)) + .exec(); + if (result) return record; + if (i < LEDGER_MUTATION_ATTEMPTS - 1) { + await sleep(Math.min(25, i + 1)); + } } - } catch (error) { - await client.unwatch().catch(unwatchError => { - logger.warn('Failed to clear egress ledger WATCH after rejected mutation', { error: unwatchError }); + } finally { + await client.unwatch().catch(error => { + logger.warn( + 'Failed to clear egress ledger WATCH before returning mutation connection', + { error }, + ); }); - throw error; - } - const result = await client.multi() - .set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)) - .exec(); - if (result) return record; - if (i < LEDGER_MUTATION_ATTEMPTS - 1) { - await sleep(Math.min(25, i + 1)); - } + releaseMutationConnection(client); } - } finally { - await client.unwatch().catch(error => { - logger.warn('Failed to clear egress ledger WATCH before returning mutation connection', { error }); - }); - releaseMutationConnection(client); - } - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger update conflicted'); + throw new EgressGrantError( + 'scope_mismatch', + 'Egress grant ledger update conflicted', + ); } -export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return recordFromGrant(grant); - const record = await loadRecord(grant.grant_id); - assertActive(record, grant); - return record; +export async function assertEgressGrantActive( + grant: EgressGrantClaims, +): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return recordFromGrant(grant); + const record = await loadRecord(grant.grant_id); + assertActive(record, grant); + return record; } -export async function recordEgressRead(grant: EgressGrantClaims): Promise { - await mutateRecord(grant, record => { - record.request_count += 1; - record.read_count += 1; - }); +export async function recordEgressRead( + grant: EgressGrantClaims, +): Promise { + await mutateRecord(grant, record => { + record.request_count += 1; + record.read_count += 1; + }); } export async function reserveEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; + grant: EgressGrantClaims; + fileId: string; + bytes: number; }): Promise { - await mutateRecord(args.grant, record => { - if (args.bytes > Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES)) { - throw new EgressGrantError('scope_mismatch', 'Upload exceeds per-file egress byte limit'); - } - if (record.output_file_ids.includes(args.fileId)) { - throw new EgressGrantError('scope_mismatch', 'Output file id has already been used for this grant'); - } - if (record.output_file_ids.length >= record.max_output_files) { - throw new EgressGrantError('scope_mismatch', 'Output file count budget exceeded'); - } - const aggregateLimit = Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES) * record.max_output_files; - if (record.uploaded_bytes + args.bytes > aggregateLimit) { - throw new EgressGrantError('scope_mismatch', 'Aggregate upload byte budget exceeded'); - } - record.request_count += 1; - record.upload_count += 1; - record.uploaded_bytes += args.bytes; - record.output_file_ids.push(args.fileId); - }); + await mutateRecord(args.grant, record => { + if ( + args.bytes > + Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES) + ) { + throw new EgressGrantError( + 'scope_mismatch', + 'Upload exceeds per-file egress byte limit', + ); + } + if (record.output_file_ids.includes(args.fileId)) { + throw new EgressGrantError( + 'scope_mismatch', + 'Output file id has already been used for this grant', + ); + } + if (record.output_file_ids.length >= record.max_output_files) { + throw new EgressGrantError( + 'scope_mismatch', + 'Output file count budget exceeded', + ); + } + const aggregateLimit = + Math.min( + record.max_upload_bytes, + env.EGRESS_GATEWAY_MAX_FILE_BYTES, + ) * record.max_output_files; + if (record.uploaded_bytes + args.bytes > aggregateLimit) { + throw new EgressGrantError( + 'scope_mismatch', + 'Aggregate upload byte budget exceeded', + ); + } + record.request_count += 1; + record.upload_count += 1; + record.uploaded_bytes += args.bytes; + record.output_file_ids.push(args.fileId); + }); } export async function releaseEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; + grant: EgressGrantClaims; + fileId: string; + bytes: number; }): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return; - await mutateRecord(args.grant, record => { - record.uploaded_bytes = Math.max(0, record.uploaded_bytes - args.bytes); - record.upload_count = Math.max(0, record.upload_count - 1); - record.request_count = Math.max(0, record.request_count - 1); - record.output_file_ids = record.output_file_ids.filter(id => id !== args.fileId); - }); + if (!env.EGRESS_LEDGER_REQUIRED) return; + await mutateRecord(args.grant, record => { + record.uploaded_bytes = Math.max(0, record.uploaded_bytes - args.bytes); + record.upload_count = Math.max(0, record.upload_count - 1); + record.request_count = Math.max(0, record.request_count - 1); + record.output_file_ids = record.output_file_ids.filter( + id => id !== args.fileId, + ); + }); } -export async function recordEgressToolCall(grantId: string | undefined, executionId: string): Promise { - if (!env.EGRESS_LEDGER_REQUIRED || !grantId) return; - const grant = { grant_id: grantId, exec_id: executionId } as EgressGrantClaims; - await mutateRecord(grant, record => { - record.request_count += 1; - record.tool_call_count += 1; - }); +export async function recordEgressToolCall( + grantId: string | undefined, + executionId: string, +): Promise { + if (!env.EGRESS_LEDGER_REQUIRED || !grantId) return; + const grant = { + grant_id: grantId, + exec_id: executionId, + } as EgressGrantClaims; + await mutateRecord(grant, record => { + record.request_count += 1; + record.tool_call_count += 1; + }); } -export async function revokeEgressLedger(grantId: string, reason: string): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) return; - const key = ledgerKey(grantId); - const raw = await redisConnection().get(key); - if (!raw) return; - const record = JSON.parse(raw) as EgressLedgerRecord; - record.status = 'revoked'; - record.revoked_at = Math.floor(Date.now() / 1000); - record.revoke_reason = reason; - await redisConnection().set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)); +export async function revokeEgressLedger( + grantId: string, + reason: string, +): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + const key = ledgerKey(grantId); + const raw = await redisConnection().get(key); + if (!raw) return; + const record = JSON.parse(raw) as EgressLedgerRecord; + record.status = 'revoked'; + record.revoked_at = Math.floor(Date.now() / 1000); + record.revoke_reason = reason; + await redisConnection().set( + key, + JSON.stringify(record), + 'EX', + ttlSeconds(record.exp), + ); } diff --git a/service/src/file-server.ts b/service/src/file-server.ts index dd89bd2..a1a3b40 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,6 +1,5 @@ import b from 'busboy'; import path from 'path'; -import IORedis from 'ioredis'; import express from 'express'; import { Client } from 'minio'; import { nanoid } from 'nanoid'; @@ -8,15 +7,17 @@ import { PassThrough } from 'stream'; import { pipeline } from 'stream/promises'; import type { BucketItem, BucketItemStat, ClientOptions } from 'minio'; import type { Readable } from 'stream'; -import type * as tls from 'tls'; import type * as t from './types'; import { metricsHandler, fileUploads, fileDownloads } from './metrics'; import { httpMetricsMiddleware } from './middleware/httpMetrics'; -import { internalServiceAuthEnabled, requireInternalServiceAuth } from './internal-service-auth'; +import { + internalServiceAuthEnabled, + requireInternalServiceAuth, +} from './internal-service-auth'; import { shutdownTelemetry, traceHttpRequest } from './telemetry'; import logger from './fileServerLogger'; import { env } from './config'; -import { redisKeepAliveOptions } from './redis-options'; +import { createRedisConnection } from './redis-connection'; const { INSTANCE_ID } = env; @@ -27,155 +28,162 @@ app.use(httpMetricsMiddleware); const bucketName = process.env.MINIO_BUCKET ?? 'test-bucket'; -type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; +type IamProviderModule = { + IamAwsProvider?: new (opts: object) => unknown; + default?: new (opts: object) => unknown; +}; async function createMinioClient(): Promise { - const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; - const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); - const useIrsa = irsaExplicit || irsaEnvVars; - - const baseConfig: ClientOptions = { - endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', - port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), - useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', - region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', - }; - - if (useIrsa) { - logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { - tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, - roleArn: process.env.AWS_ROLE_ARN, - region: baseConfig.region, - }); + const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; + const irsaEnvVars = + Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && + Boolean(process.env.AWS_ROLE_ARN); + const useIrsa = irsaExplicit || irsaEnvVars; + + const baseConfig: ClientOptions = { + endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', + port: + process.env.MINIO_NO_PORT?.toLowerCase() === 'true' + ? undefined + : parseInt(process.env.MINIO_PORT ?? '9000'), + useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', + region: + process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', + }; - /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module - * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) - */ - let IamAwsProviderClass: new (opts: object) => unknown; - try { - const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (primaryError) { - try { - // Fallback for bun: resolve path using require if available (CJS context) - let resolvePath = 'node_modules/minio/'; + if (useIrsa) { + logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { + tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, + roleArn: process.env.AWS_ROLE_ARN, + region: baseConfig.region, + }); + + /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module + * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) + */ + let IamAwsProviderClass: new (opts: object) => unknown; try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); - } catch { - // require.resolve not available (ESM context), use default path + const mod = + (await import('minio/dist/main/IamAwsProvider.js')) as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (primaryError) { + try { + // Fallback for bun: resolve path using require if available (CJS context) + let resolvePath = 'node_modules/minio/'; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + resolvePath = require + .resolve('minio') + .replace(/dist\/.*$/, ''); + } catch { + // require.resolve not available (ESM context), use default path + } + const mod = (await import( + `${resolvePath}dist/main/IamAwsProvider.js` + )) as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (fallbackError) { + logger.error('Failed to load IamAwsProvider', { + primaryError, + fallbackError, + }); + throw new Error( + 'Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.', + ); + } } - const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (fallbackError) { - logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); - throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); - } - } - const credentialsProvider = new IamAwsProviderClass({}); + const credentialsProvider = new IamAwsProviderClass({}); + return new Client({ + ...baseConfig, + credentialsProvider: + credentialsProvider as ClientOptions['credentialsProvider'], + }); + } + + logger.info('Using explicit credentials for MinIO/S3 authentication'); return new Client({ - ...baseConfig, - credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], + ...baseConfig, + accessKey: process.env.MINIO_ACCESS_KEY ?? '', + secretKey: process.env.MINIO_SECRET_KEY ?? '', + sessionToken: process.env.MINIO_SESSION_TOKEN, }); - } - - logger.info('Using explicit credentials for MinIO/S3 authentication'); - return new Client({ - ...baseConfig, - accessKey: process.env.MINIO_ACCESS_KEY ?? '', - secretKey: process.env.MINIO_SECRET_KEY ?? '', - sessionToken: process.env.MINIO_SESSION_TOKEN, - }); } let minioClient: Client; let storageInitialized = false; -const useAltDnsLookup = process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true'; - -const redisClient = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - enableReadyCheck: false, - tls: process.env.REDIS_TLS === 'true' ? { - // For self-signed certificates - rejectUnauthorized: false - } as tls.ConnectionOptions : undefined, - connectTimeout: 10000, - ...redisKeepAliveOptions(), - maxRetriesPerRequest: 3, - retryStrategy(times: number): number { - const delay = Math.min(times * 500, 2000); - return delay; - }, - reconnectOnError(err: Error): boolean { - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; - }, - // Alternative DNS lookup for AWS ElastiCache TLS connections - ...(useAltDnsLookup - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}) +const redisClient = createRedisConnection({ + enableReadyCheck: false, + maxRetriesPerRequest: 3, + retryStrategy(times: number): number { + return Math.min(times * 500, 2000); + }, + reconnectOnError(err: Error): boolean { + return err.message.includes('READONLY'); + }, }); -redisClient.on('error', (err) => { - logger.error('Redis Client Error', { error: err }); +redisClient.on('error', err => { + logger.error('Redis Client Error', { error: err }); }); redisClient.on('connect', () => { - logger.info('Redis Client Connected', { - host: process.env.REDIS_HOST, - port: process.env.REDIS_PORT - }); + logger.info('Redis Client Connected', { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + }); }); redisClient.on('ready', () => { - logger.info('Redis Client Ready'); + logger.info('Redis Client Ready'); }); -const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; +const minioRegion = + process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { - for (let attempt = 1; attempt <= retries; attempt++) { - try { - const exists = await minioClient.bucketExists(bucketName); - if (exists) { - logger.info('Bucket already exists'); - return; - } - await minioClient.makeBucket(bucketName, minioRegion); - logger.info('Bucket created successfully'); - return; - } catch (err: unknown) { - const error = err as { code?: string; message?: string }; - if (error.code === 'BucketAlreadyOwnedByYou') { - logger.info('Bucket already exists'); - return; - } - - if (attempt < retries) { - const backoff = delay * Math.pow(2, attempt - 1); - logger.warn(`MinIO not ready, retrying in ${backoff}ms (attempt ${attempt}/${retries})`, { error: error.message }); - await new Promise(resolve => setTimeout(resolve, backoff)); - } else { - logger.error('Failed to ensure bucket exists after all retries', { error }); - throw err; - } + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const exists = await minioClient.bucketExists(bucketName); + if (exists) { + logger.info('Bucket already exists'); + return; + } + await minioClient.makeBucket(bucketName, minioRegion); + logger.info('Bucket created successfully'); + return; + } catch (err: unknown) { + const error = err as { code?: string; message?: string }; + if (error.code === 'BucketAlreadyOwnedByYou') { + logger.info('Bucket already exists'); + return; + } + + if (attempt < retries) { + const backoff = delay * Math.pow(2, attempt - 1); + logger.warn( + `MinIO not ready, retrying in ${backoff}ms (attempt ${attempt}/${retries})`, + { error: error.message }, + ); + await new Promise(resolve => setTimeout(resolve, backoff)); + } else { + logger.error( + 'Failed to ensure bucket exists after all retries', + { error }, + ); + throw err; + } + } } - } } async function initializeStorage(): Promise { - minioClient = await createMinioClient(); - await ensureBucketExists(); - storageInitialized = true; - logger.info('Storage initialization complete'); + minioClient = await createMinioClient(); + await ensureBucketExists(); + storageInitialized = true; + logger.info('Storage initialization complete'); } /** @@ -186,243 +194,322 @@ async function initializeStorage(): Promise { * resolve with `empty: true`; non-empty streams resolve with a PassThrough * that replays the peeked first chunk and the rest of the upstream. */ -function peekStreamForEmpty(input: Readable): Promise<{ empty: true } | { empty: false; body: Readable }> { - return new Promise((resolve, reject) => { - let settled = false; - const cleanup = (): void => { - input.removeListener('data', onData); - input.removeListener('end', onEnd); - input.removeListener('error', onError); - }; - function onError(err: Error): void { - if (settled) return; - settled = true; - cleanup(); - reject(err); - } - function onEnd(): void { - if (settled) return; - settled = true; - cleanup(); - resolve({ empty: true }); - } - function onData(firstChunk: Buffer | string): void { - if (settled) return; - settled = true; - cleanup(); - input.pause(); - const passthrough = new PassThrough(); - const buf = Buffer.isBuffer(firstChunk) ? firstChunk : Buffer.from(firstChunk); - passthrough.write(buf); - input.pipe(passthrough); - input.once('error', (err) => passthrough.destroy(err)); - resolve({ empty: false, body: passthrough }); - } - input.once('data', onData); - input.once('end', onEnd); - input.once('error', onError); - }); +function peekStreamForEmpty( + input: Readable, +): Promise<{ empty: true } | { empty: false; body: Readable }> { + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = (): void => { + input.removeListener('data', onData); + input.removeListener('end', onEnd); + input.removeListener('error', onError); + }; + function onError(err: Error): void { + if (settled) return; + settled = true; + cleanup(); + reject(err); + } + function onEnd(): void { + if (settled) return; + settled = true; + cleanup(); + resolve({ empty: true }); + } + function onData(firstChunk: Buffer | string): void { + if (settled) return; + settled = true; + cleanup(); + input.pause(); + const passthrough = new PassThrough(); + const buf = Buffer.isBuffer(firstChunk) + ? firstChunk + : Buffer.from(firstChunk); + passthrough.write(buf); + input.pipe(passthrough); + input.once('error', err => passthrough.destroy(err)); + resolve({ empty: false, body: passthrough }); + } + input.once('data', onData); + input.once('end', onEnd); + input.once('error', onError); + }); } async function uploadFile( - session_id: string, - fileStream: Readable, - filename: string, - mimetype: string, - existingFileId?: string, - readOnly = false, + session_id: string, + fileStream: Readable, + filename: string, + mimetype: string, + existingFileId?: string, + readOnly = false, ): Promise { - const fileId = existingFileId ?? nanoid(); - const fileExtension = path.extname(filename); - const objectName = `${session_id}/${fileId}${fileExtension}`; - - const encodedFilename = Buffer.from(filename).toString('base64'); - - /* `X-Amz-Meta-Read-Only: true` declares this file as infrastructure the - * uploader doesn't want surfaced as a generated artifact downstream - * (e.g. skill files primed by LibreChat). Stored as an MinIO/S3 user - * metadata header so it persists with the object and is exposed on - * `getObject` / `statObject` without a separate Redis lookup. */ - const metaData: Record = { - 'Content-Type': mimetype, - 'X-Amz-Meta-Original-Filename': encodedFilename, - 'X-Amz-Meta-Original-Filename-Encoded': 'base64', - }; - if (readOnly) { - metaData['X-Amz-Meta-Read-Only'] = 'true'; - } - - /* Note: this returns UploadedObjectInfo */ - const sessionKey = await redisClient.get(`session:${session_id}`); - const peeked = await peekStreamForEmpty(fileStream); - if (peeked.empty) { - /* Empty file: explicit single PUT with size=0 — multipart fails with - * "You must specify at least one part" on zero-byte streams. */ - await minioClient.putObject(bucketName, objectName, Buffer.alloc(0), 0, metaData); - } else { - await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); - } - logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); - await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); - fileUploads.inc(); - - return { - filename, - fileId, - }; + const fileId = existingFileId ?? nanoid(); + const fileExtension = path.extname(filename); + const objectName = `${session_id}/${fileId}${fileExtension}`; + + const encodedFilename = Buffer.from(filename).toString('base64'); + + /* `X-Amz-Meta-Read-Only: true` declares this file as infrastructure the + * uploader doesn't want surfaced as a generated artifact downstream + * (e.g. skill files primed by LibreChat). Stored as an MinIO/S3 user + * metadata header so it persists with the object and is exposed on + * `getObject` / `statObject` without a separate Redis lookup. */ + const metaData: Record = { + 'Content-Type': mimetype, + 'X-Amz-Meta-Original-Filename': encodedFilename, + 'X-Amz-Meta-Original-Filename-Encoded': 'base64', + }; + if (readOnly) { + metaData['X-Amz-Meta-Read-Only'] = 'true'; + } + + /* Note: this returns UploadedObjectInfo */ + const sessionKey = await redisClient.get(`session:${session_id}`); + const peeked = await peekStreamForEmpty(fileStream); + if (peeked.empty) { + /* Empty file: explicit single PUT with size=0 — multipart fails with + * "You must specify at least one part" on zero-byte streams. */ + await minioClient.putObject( + bucketName, + objectName, + Buffer.alloc(0), + 0, + metaData, + ); + } else { + await minioClient.putObject( + bucketName, + objectName, + peeked.body, + undefined, + metaData, + ); + } + logger.info( + `[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`, + ); + await redisClient.set( + `upload:${sessionKey}${session_id}${fileId}`, + 'true', + 'EX', + env.SESSION_CACHE_TTL, + ); + fileUploads.inc(); + + return { + filename, + fileId, + }; } app.get('/metrics', metricsHandler); app.get('/health', (_req: express.Request, res: express.Response) => { - res.status(200).json({ status: 'ok' }); + res.status(200).json({ status: 'ok' }); }); app.get('/ready', async (_req: express.Request, res: express.Response) => { - const checks: { redis?: string; s3?: string; storage?: string } = {}; - let healthy = true; - - if (!storageInitialized) { - checks.storage = 'initializing'; - healthy = false; - } - - try { - await redisClient.ping(); - checks.redis = 'ok'; - } catch (error) { - logger.error('Readiness check failed - Redis:', { error }); - checks.redis = 'error'; - healthy = false; - } - - if (storageInitialized) { + const checks: { redis?: string; s3?: string; storage?: string } = {}; + let healthy = true; + + if (!storageInitialized) { + checks.storage = 'initializing'; + healthy = false; + } + try { - await minioClient.bucketExists(bucketName); - checks.s3 = 'ok'; + await redisClient.ping(); + checks.redis = 'ok'; } catch (error) { - logger.error('Readiness check failed - S3:', { error }); - checks.s3 = 'error'; - healthy = false; + logger.error('Readiness check failed - Redis:', { error }); + checks.redis = 'error'; + healthy = false; + } + + if (storageInitialized) { + try { + await minioClient.bucketExists(bucketName); + checks.s3 = 'ok'; + } catch (error) { + logger.error('Readiness check failed - S3:', { error }); + checks.s3 = 'error'; + healthy = false; + } + } else { + checks.s3 = 'pending'; + } + + if (healthy) { + res.status(200).json({ status: 'ready', checks }); + } else { + res.status(503).json({ status: 'not ready', checks }); } - } else { - checks.s3 = 'pending'; - } - - if (healthy) { - res.status(200).json({ status: 'ready', checks }); - } else { - res.status(503).json({ status: 'not ready', checks }); - } }); if (!internalServiceAuthEnabled()) { - logger.warn('CODEAPI_INTERNAL_SERVICE_TOKEN is not set; file object routes are unauthenticated'); + logger.warn( + 'CODEAPI_INTERNAL_SERVICE_TOKEN is not set; file object routes are unauthenticated', + ); } app.use('/sessions', requireInternalServiceAuth); -app.post('/sessions/:session_id/objects', async (req: express.Request, res: express.Response) => { - const { session_id } = req.params; - /** Request-level X-Read-Only flag — applies to every file in this batch. - * See `uploadFile` for semantics (infrastructure inputs that callers - * should not surface as generated artifacts). */ - const readOnlyHeader = req.headers['x-read-only']; - const readOnly = typeof readOnlyHeader === 'string' && readOnlyHeader.toLowerCase() === 'true'; - /** busboy with proper charset handling and preservePath so subdirectory - * components survive (e.g. `pptx/editing.md`); default strips to basename. */ - const busboy = b({ - headers: req.headers, - defCharset: 'utf8', - defParamCharset: 'utf8', - preservePath: true, - }); - const uploadPromises: Promise[] = []; - - busboy.on('file', (fieldname: string, file: Readable, info: b.FileInfo) => { - const { filename: combinedFilename, encoding: _e, mimeType } = info; - - // Handle the filename properly - it might be URL encoded - let decodedFilename: string; - try { - decodedFilename = decodeURIComponent(combinedFilename); - } catch (err) { - // If decoding fails, use the original filename - logger.warn(`Failed to decode filename, using original: ${combinedFilename}`, { error: err }); - decodedFilename = combinedFilename; - } - - const [fileId, ...filenameParts] = decodedFilename.split('___'); - const filename = filenameParts.join('___'); - - logger.info(`[${INSTANCE_ID}] Processing file: ${filename} with ID: ${fileId}`); - - const uploadPromise = uploadFile(session_id, file, filename, mimeType, fileId, readOnly).catch(err => { - logger.error(`[${INSTANCE_ID}] Error uploading file ${filename}:`, { error: err }); - return null; - }); - uploadPromises.push(uploadPromise); - }); - - busboy.on('finish', async () => { - try { - const results = await Promise.all(uploadPromises); - const successfulUploads = results.filter((result): result is t.UploadResult => result !== null); - - logger.info(`[${INSTANCE_ID}] Successfully uploaded ${successfulUploads.length} files for session ${session_id}`); +app.post( + '/sessions/:session_id/objects', + async (req: express.Request, res: express.Response) => { + const { session_id } = req.params; + /** Request-level X-Read-Only flag — applies to every file in this batch. + * See `uploadFile` for semantics (infrastructure inputs that callers + * should not surface as generated artifacts). */ + const readOnlyHeader = req.headers['x-read-only']; + const readOnly = + typeof readOnlyHeader === 'string' && + readOnlyHeader.toLowerCase() === 'true'; + /** busboy with proper charset handling and preservePath so subdirectory + * components survive (e.g. `pptx/editing.md`); default strips to basename. */ + const busboy = b({ + headers: req.headers, + defCharset: 'utf8', + defParamCharset: 'utf8', + preservePath: true, + }); + const uploadPromises: Promise[] = []; + + busboy.on( + 'file', + (fieldname: string, file: Readable, info: b.FileInfo) => { + const { + filename: combinedFilename, + encoding: _e, + mimeType, + } = info; + + // Handle the filename properly - it might be URL encoded + let decodedFilename: string; + try { + decodedFilename = decodeURIComponent(combinedFilename); + } catch (err) { + // If decoding fails, use the original filename + logger.warn( + `Failed to decode filename, using original: ${combinedFilename}`, + { error: err }, + ); + decodedFilename = combinedFilename; + } + + const [fileId, ...filenameParts] = decodedFilename.split('___'); + const filename = filenameParts.join('___'); + + logger.info( + `[${INSTANCE_ID}] Processing file: ${filename} with ID: ${fileId}`, + ); + + const uploadPromise = uploadFile( + session_id, + file, + filename, + mimeType, + fileId, + readOnly, + ).catch(err => { + logger.error( + `[${INSTANCE_ID}] Error uploading file ${filename}:`, + { error: err }, + ); + return null; + }); + uploadPromises.push(uploadPromise); + }, + ); + + busboy.on('finish', async () => { + try { + const results = await Promise.all(uploadPromises); + const successfulUploads = results.filter( + (result): result is t.UploadResult => result !== null, + ); + + logger.info( + `[${INSTANCE_ID}] Successfully uploaded ${successfulUploads.length} files for session ${session_id}`, + ); + + return res.status(200).json({ + message: 'success', + storage_session_id: session_id, + files: successfulUploads, + }); + } catch (err) { + logger.error('Error processing uploads:', { error: err }); + return res.status(500).send('Error uploading files.'); + } + }); - return res.status(200).json({ - message: 'success', - storage_session_id: session_id, - files: successfulUploads - }); - } catch (err) { - logger.error('Error processing uploads:', { error: err }); - return res.status(500).send('Error uploading files.'); - } - }); + busboy.on('error', error => { + logger.error( + `[${INSTANCE_ID}] Busboy error for session_id ${session_id}:`, + error, + ); + res.status(500).json({ error: 'Error processing upload' }); + }); - busboy.on('error', (error) => { - logger.error(`[${INSTANCE_ID}] Busboy error for session_id ${session_id}:`, error); - res.status(500).json({ error: 'Error processing upload' }); - }); + await pipeline(req, busboy); + }, +); + +app.put( + '/sessions/:session_id/objects/:fileId', + async (req: express.Request, res: express.Response) => { + const { session_id, fileId } = req.params; + // Decode the filename from the header if it's URL encoded + const originalFilename = req.headers['x-original-filename'] as string; + let decodedFilename = ''; + + if (originalFilename) { + try { + decodedFilename = decodeURIComponent(originalFilename); + } catch (err) { + // If decoding fails, use the original filename + logger.warn( + `Failed to decode filename header, using original: ${originalFilename}`, + { error: err }, + ); + decodedFilename = originalFilename; + } + } - await pipeline(req, busboy); -}); + const mimeType = req.headers['content-type'] as string; + const readOnlyHeader = req.headers['x-read-only']; + const readOnly = + typeof readOnlyHeader === 'string' && + readOnlyHeader.toLowerCase() === 'true'; -app.put('/sessions/:session_id/objects/:fileId', async (req: express.Request, res: express.Response) => { - const { session_id, fileId } = req.params; - // Decode the filename from the header if it's URL encoded - const originalFilename = req.headers['x-original-filename'] as string; - let decodedFilename = ''; + if (!decodedFilename || !mimeType) { + return res.status(400).json({ error: 'Missing required headers' }); + } - if (originalFilename) { - try { - decodedFilename = decodeURIComponent(originalFilename); - } catch (err) { - // If decoding fails, use the original filename - logger.warn(`Failed to decode filename header, using original: ${originalFilename}`, { error: err }); - decodedFilename = originalFilename; - } - } - - const mimeType = req.headers['content-type'] as string; - const readOnlyHeader = req.headers['x-read-only']; - const readOnly = typeof readOnlyHeader === 'string' && readOnlyHeader.toLowerCase() === 'true'; - - if (!decodedFilename || !mimeType) { - return res.status(400).json({ error: 'Missing required headers' }); - } - - try { - const result = await uploadFile(session_id, req, decodedFilename, mimeType, fileId, readOnly); - logger.info(`[${INSTANCE_ID}] File uploaded successfully: ${result.filename}`); - return res.status(200).json(result); - } catch (err) { - logger.error(`[${INSTANCE_ID}] Error uploading file ${decodedFilename}:`, { error: err }); - return res.status(500).json({ error: 'Error uploading file.' }); - } -}); + try { + const result = await uploadFile( + session_id, + req, + decodedFilename, + mimeType, + fileId, + readOnly, + ); + logger.info( + `[${INSTANCE_ID}] File uploaded successfully: ${result.filename}`, + ); + return res.status(200).json(result); + } catch (err) { + logger.error( + `[${INSTANCE_ID}] Error uploading file ${decodedFilename}:`, + { error: err }, + ); + return res.status(500).json({ error: 'Error uploading file.' }); + } + }, +); /** * Single-object metadata lookup. Returns the JSON shape callers @@ -434,289 +521,371 @@ app.put('/sessions/:session_id/objects/:fileId', async (req: express.Request, re * service-api expose only the metadata variant under sessionAuth * without conflating with the internal-only binary download. */ -app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => { - const { session_id, objectId } = req.params; - - try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - }); - } +app.get( + '/sessions/:session_id/objects/:objectId/metadata', + async (req, res) => { + const { session_id, objectId } = req.params; - const stat: Partial = await minioClient.statObject(bucketName, objectName); - const originalFilename = decodeOriginalFilename(stat.metaData, path.basename(objectName)); - - return res.status(200).json({ - name: objectName, - originalFilename, - size: stat.size, - lastModified: stat.lastModified, - etag: stat.etag, - contentType: stat.metaData?.['content-type'] ?? 'application/octet-stream', - readOnly: stat.metaData?.['read-only'] === 'true', - }); - } catch (err) { - logger.error('Error fetching object metadata:', { error: err, session_id, objectId, bucketName }); - return res.status(500).json({ - error: 'Error fetching object metadata', - details: (err as Error | undefined)?.message, - }); - } -}); + try { + const stream = minioClient.listObjects( + bucketName, + `${session_id}/${objectId}`, + true, + ); + let objectName = ''; + + for await (const obj of stream) { + if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { + objectName = obj.name; + break; + } + } + + if (!objectName) { + return res.status(404).json({ + error: 'File not found', + details: 'No matching file found', + session_id, + objectId, + }); + } + + const stat: Partial = await minioClient.statObject( + bucketName, + objectName, + ); + const originalFilename = decodeOriginalFilename( + stat.metaData, + path.basename(objectName), + ); + + return res.status(200).json({ + name: objectName, + originalFilename, + size: stat.size, + lastModified: stat.lastModified, + etag: stat.etag, + contentType: + stat.metaData?.['content-type'] ?? + 'application/octet-stream', + readOnly: stat.metaData?.['read-only'] === 'true', + }); + } catch (err) { + logger.error('Error fetching object metadata:', { + error: err, + session_id, + objectId, + bucketName, + }); + return res.status(500).json({ + error: 'Error fetching object metadata', + details: (err as Error | undefined)?.message, + }); + } + }, +); app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { - const { session_id, objectId } = req.params; - - try { - // List objects to find the correct file with extension - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - logger.warn('File not found', { session_id, objectId, bucketName }); - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - bucketName - }); - } + const { session_id, objectId } = req.params; - logger.info(`[${INSTANCE_ID}] Attempting to download: ${objectName}`); + try { + // List objects to find the correct file with extension + const stream = minioClient.listObjects( + bucketName, + `${session_id}/${objectId}`, + true, + ); + let objectName = ''; + + for await (const obj of stream) { + if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { + objectName = obj.name; + break; + } + } - const stat: Partial = await minioClient.statObject(bucketName, objectName); + if (!objectName) { + logger.warn('File not found', { session_id, objectId, bucketName }); + return res.status(404).json({ + error: 'File not found', + details: 'No matching file found', + session_id, + objectId, + bucketName, + }); + } - let originalFilename = path.basename(objectName); - if (stat.metaData?.['original-filename-encoded'] === 'base64' && stat.metaData['original-filename'] != null) { - try { - originalFilename = Buffer.from(stat.metaData['original-filename'], 'base64').toString('utf8'); - } catch (err) { - logger.warn('Failed to decode filename from metadata, using fallback', { error: err }); - originalFilename = stat.metaData['original-filename'] ?? path.basename(objectName); - } - } else if (stat.metaData?.['original-filename'] != null) { - originalFilename = stat.metaData['original-filename']; - } + logger.info(`[${INSTANCE_ID}] Attempting to download: ${objectName}`); + + const stat: Partial = await minioClient.statObject( + bucketName, + objectName, + ); + + let originalFilename = path.basename(objectName); + if ( + stat.metaData?.['original-filename-encoded'] === 'base64' && + stat.metaData['original-filename'] != null + ) { + try { + originalFilename = Buffer.from( + stat.metaData['original-filename'], + 'base64', + ).toString('utf8'); + } catch (err) { + logger.warn( + 'Failed to decode filename from metadata, using fallback', + { error: err }, + ); + originalFilename = + stat.metaData['original-filename'] ?? + path.basename(objectName); + } + } else if (stat.metaData?.['original-filename'] != null) { + originalFilename = stat.metaData['original-filename']; + } - logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); + logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); - // Explicitly remove problematic headers that might be duplicated - res.removeHeader('Transfer-Encoding'); - res.removeHeader('Date'); + // Explicitly remove problematic headers that might be duplicated + res.removeHeader('Transfer-Encoding'); + res.removeHeader('Date'); - const encodedFilename = encodeURIComponent(originalFilename); - res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${encodedFilename}`); - if (stat.metaData?.['content-type'] != null) { - res.setHeader('Content-Type', stat.metaData['content-type']); - } - /* Surface the read-only flag on download so the sandbox can plumb it - * onto its in-memory file metadata without a separate metadata fetch. - * MinIO normalizes `X-Amz-Meta-Read-Only` to `read-only` in stat.metaData. */ - if (stat.metaData?.['read-only'] === 'true') { - res.setHeader('X-Read-Only', 'true'); - } + const encodedFilename = encodeURIComponent(originalFilename); + res.setHeader( + 'Content-Disposition', + `attachment; filename*=UTF-8''${encodedFilename}`, + ); + if (stat.metaData?.['content-type'] != null) { + res.setHeader('Content-Type', stat.metaData['content-type']); + } + /* Surface the read-only flag on download so the sandbox can plumb it + * onto its in-memory file metadata without a separate metadata fetch. + * MinIO normalizes `X-Amz-Meta-Read-Only` to `read-only` in stat.metaData. */ + if (stat.metaData?.['read-only'] === 'true') { + res.setHeader('X-Read-Only', 'true'); + } - const dataStream = await minioClient.getObject(bucketName, objectName); - fileDownloads.inc(); + const dataStream = await minioClient.getObject(bucketName, objectName); + fileDownloads.inc(); - dataStream.on('data', (chunk) => { - res.write(chunk); - }); + dataStream.on('data', chunk => { + res.write(chunk); + }); - dataStream.on('end', () => { - res.end(); - }); + dataStream.on('end', () => { + res.end(); + }); - dataStream.on('error', (err) => { - logger.error('Error streaming file:', { error: err, session_id, objectId, bucketName }); - // Only send error if headers haven't been sent yet - if (!res.headersSent) { - res.status(500).json({ - error: 'Error streaming file', - details: err.message + dataStream.on('error', err => { + logger.error('Error streaming file:', { + error: err, + session_id, + objectId, + bucketName, + }); + // Only send error if headers haven't been sent yet + if (!res.headersSent) { + res.status(500).json({ + error: 'Error streaming file', + details: err.message, + }); + } else { + res.end(); + } }); - } else { - res.end(); - } - }); - } catch (err) { - logger.error('Error downloading file:', { error: err, session_id, objectId, bucketName }); - return res.status(500).json({ - error: 'Error downloading file', - details: (err as Error | undefined)?.message, - session_id, - objectId, - bucketName - }); - } + } catch (err) { + logger.error('Error downloading file:', { + error: err, + session_id, + objectId, + bucketName, + }); + return res.status(500).json({ + error: 'Error downloading file', + details: (err as Error | undefined)?.message, + session_id, + objectId, + bucketName, + }); + } }); /** * Decodes the original filename from metadata. * Handles both base64-encoded and plain text filenames for consistency. */ -function decodeOriginalFilename(metadata: Record | undefined, fallbackName: string): string { - if (!metadata) return fallbackName; +function decodeOriginalFilename( + metadata: Record | undefined, + fallbackName: string, +): string { + if (!metadata) return fallbackName; - const encodedFilename = metadata['original-filename']; - const encodingType = metadata['original-filename-encoded']; + const encodedFilename = metadata['original-filename']; + const encodingType = metadata['original-filename-encoded']; - if (encodedFilename && encodingType === 'base64') { - try { - return Buffer.from(encodedFilename, 'base64').toString('utf8'); - } catch { - return encodedFilename; + if (encodedFilename && encodingType === 'base64') { + try { + return Buffer.from(encodedFilename, 'base64').toString('utf8'); + } catch { + return encodedFilename; + } } - } - return encodedFilename || fallbackName; + return encodedFilename || fallbackName; } /** * Extracts session_id and file_id from object name (format: {session_id}/{file_id}.ext) */ -function parseObjectName(objectName: string | undefined): { session_id: string; file_id: string } | null { - if (objectName == null || objectName === '') return null; - const parts = objectName.split('/'); - if (parts.length < 2) return null; - const session_id = parts[0]; - const fileNameWithExt = parts[1]; - // Remove extension to get file_id - const file_id = fileNameWithExt.replace(/\.[^.]+$/, ''); - return { session_id, file_id }; +function parseObjectName( + objectName: string | undefined, +): { session_id: string; file_id: string } | null { + if (objectName == null || objectName === '') return null; + const parts = objectName.split('/'); + if (parts.length < 2) return null; + const session_id = parts[0]; + const fileNameWithExt = parts[1]; + // Remove extension to get file_id + const file_id = fileNameWithExt.replace(/\.[^.]+$/, ''); + return { session_id, file_id }; } -const detailLevels: Record Promise> | undefined> = { - simple: async (obj: BucketItem): Promise> => obj.name ?? '', - summary: async (obj: BucketItem): Promise> => ({ - name: obj.name, - size: obj.size, - lastModified: obj.lastModified, - etag: obj.etag - }), - full: async (obj: BucketItem): Promise> => { - const stat = await minioClient.statObject(bucketName, obj.name ?? ''); - // Decode original filename for consistent plain text response - const originalFilename = decodeOriginalFilename(stat.metaData, path.basename(obj.name ?? '')); - return { - name: obj.name, - size: obj.size, - lastModified: obj.lastModified, - etag: obj.etag, - metadata: { - ...stat.metaData, - // Provide decoded filename for client convenience (standardized) - 'original-filename': originalFilename, - 'original-filename-encoded': 'none' // Indicate it's already decoded - }, - versionId: stat.versionId, - contentType: stat.metaData['content-type'] ?? 'application/octet-stream' - }; - }, - // New normalized detail level - returns self-contained file references - // Ideal for clients that need to pass files to subsequent requests - normalized: async (obj: BucketItem): Promise> => { - const stat = await minioClient.statObject(bucketName, obj.name ?? ''); - const originalFilename = decodeOriginalFilename(stat.metaData, path.basename(obj.name ?? '')); - const parsed = parseObjectName(obj.name); - - const result: Record = { - id: parsed?.file_id ?? path.basename(obj.name ?? '').replace(/\.[^.]+$/, ''), - name: originalFilename, - storage_session_id: parsed?.session_id ?? '', - size: obj.size, - contentType: stat.metaData['content-type'] ?? 'application/octet-stream', - lastModified: obj.lastModified, - }; - if (stat.metaData['read-only'] === 'true') { - result.read_only = true; - } - return result; - } +const detailLevels: Record< + t.DetailLevel | string, + ( + obj: BucketItem, + ) => Promise> | undefined +> = { + simple: async (obj: BucketItem): Promise> => + obj.name ?? '', + summary: async (obj: BucketItem): Promise> => ({ + name: obj.name, + size: obj.size, + lastModified: obj.lastModified, + etag: obj.etag, + }), + full: async (obj: BucketItem): Promise> => { + const stat = await minioClient.statObject(bucketName, obj.name ?? ''); + // Decode original filename for consistent plain text response + const originalFilename = decodeOriginalFilename( + stat.metaData, + path.basename(obj.name ?? ''), + ); + return { + name: obj.name, + size: obj.size, + lastModified: obj.lastModified, + etag: obj.etag, + metadata: { + ...stat.metaData, + // Provide decoded filename for client convenience (standardized) + 'original-filename': originalFilename, + 'original-filename-encoded': 'none', // Indicate it's already decoded + }, + versionId: stat.versionId, + contentType: + stat.metaData['content-type'] ?? 'application/octet-stream', + }; + }, + // New normalized detail level - returns self-contained file references + // Ideal for clients that need to pass files to subsequent requests + normalized: async (obj: BucketItem): Promise> => { + const stat = await minioClient.statObject(bucketName, obj.name ?? ''); + const originalFilename = decodeOriginalFilename( + stat.metaData, + path.basename(obj.name ?? ''), + ); + const parsed = parseObjectName(obj.name); + + const result: Record = { + id: + parsed?.file_id ?? + path.basename(obj.name ?? '').replace(/\.[^.]+$/, ''), + name: originalFilename, + storage_session_id: parsed?.session_id ?? '', + size: obj.size, + contentType: + stat.metaData['content-type'] ?? 'application/octet-stream', + lastModified: obj.lastModified, + }; + if (stat.metaData['read-only'] === 'true') { + result.read_only = true; + } + return result; + }, }; app.get('/sessions/:session_id/objects', async (req, res) => { - const { session_id } = req.params; - const { detail = 'simple' } = req.query; + const { session_id } = req.params; + const { detail = 'simple' } = req.query; - try { - const stream = minioClient.listObjects(bucketName, session_id, true); - const objects: (t.ObjectTypes | Partial | undefined)[] = []; + try { + const stream = minioClient.listObjects(bucketName, session_id, true); + const objects: (t.ObjectTypes | Partial | undefined)[] = + []; - const getDetail = detailLevels[detail as string] ?? detailLevels.simple; + const getDetail = detailLevels[detail as string] ?? detailLevels.simple; - for await (const obj of stream) { - objects.push(await getDetail(obj)); - } + for await (const obj of stream) { + objects.push(await getDetail(obj)); + } - res.json(objects); - } catch (err) { - logger.error('Error listing objects:', { error: err, session_id }); - return res.status(500).send('Error listing objects'); - } + res.json(objects); + } catch (err) { + logger.error('Error listing objects:', { error: err, session_id }); + return res.status(500).send('Error listing objects'); + } }); app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { - const { session_id, fileId } = req.params; + const { session_id, fileId } = req.params; - try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${fileId}`, true); - let objectName = ''; + try { + const stream = minioClient.listObjects( + bucketName, + `${session_id}/${fileId}`, + true, + ); + let objectName = ''; + + for await (const obj of stream) { + if (obj.name.startsWith(`${session_id}/${fileId}`) === true) { + objectName = obj.name; + break; + } + } - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${fileId}`) === true) { - objectName = obj.name; - break; - } - } + if (!objectName) { + logger.warn('File not found for deletion', { + session_id, + fileId, + bucketName, + }); + return res.status(404).json({ + error: 'File not found', + details: 'No matching file found for deletion', + session_id, + fileId, + bucketName, + }); + } - if (!objectName) { - logger.warn('File not found for deletion', { session_id, fileId, bucketName }); - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found for deletion', - session_id, - fileId, - bucketName - }); + await minioClient.removeObject(bucketName, objectName); + logger.info( + `[${INSTANCE_ID}] File deleted successfully: ${objectName}`, + ); + return res.status(200).json({ + message: 'File deleted successfully', + session_id, + fileId, + }); + } catch (err) { + logger.error('Error deleting file:', err); + return res.status(500).json({ + error: 'Error deleting file', + }); } - - await minioClient.removeObject(bucketName, objectName); - logger.info(`[${INSTANCE_ID}] File deleted successfully: ${objectName}`); - return res.status(200).json({ - message: 'File deleted successfully', - session_id, - fileId - }); - - } catch (err) { - logger.error('Error deleting file:', err); - return res.status(500).json({ - error: 'Error deleting file', - }); - } }); const port = process.env.FILE_SERVER_PORT ?? 3000; @@ -724,49 +893,53 @@ let server: ReturnType | undefined; let shuttingDown = false; async function startServer(): Promise { - try { - await initializeStorage(); - server = app.listen(port, () => { - logger.info(`[${INSTANCE_ID}] Server running on port ${port}`); - }); - } catch (err) { - logger.error('Critical: Could not initialize storage', { error: err }); - process.exit(1); - } + try { + await initializeStorage(); + server = app.listen(port, () => { + logger.info(`[${INSTANCE_ID}] Server running on port ${port}`); + }); + } catch (err) { + logger.error('Critical: Could not initialize storage', { error: err }); + process.exit(1); + } } function closeHttpServer(): Promise { - if (!server) return Promise.resolve(); - return new Promise((resolve, reject) => { - server?.close((error) => { - if (error) reject(error); - else resolve(); + if (!server) return Promise.resolve(); + return new Promise((resolve, reject) => { + server?.close(error => { + if (error) reject(error); + else resolve(); + }); }); - }); } async function shutdown(): Promise { - if (shuttingDown) return; - shuttingDown = true; - logger.info(`[${INSTANCE_ID}] Shutting down file server...`); - try { - await closeHttpServer(); - await redisClient.quit(); - try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); - } - process.exit(0); - } catch (error) { - logger.error(`[${INSTANCE_ID}] File server shutdown failed`, { error }); + if (shuttingDown) return; + shuttingDown = true; + logger.info(`[${INSTANCE_ID}] Shutting down file server...`); try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); + await closeHttpServer(); + await redisClient.quit(); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(0); + } catch (error) { + logger.error(`[${INSTANCE_ID}] File server shutdown failed`, { error }); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(1); } - process.exit(1); - } } startServer(); @@ -774,10 +947,10 @@ startServer(); process.on('SIGTERM', () => void shutdown()); process.on('SIGINT', () => void shutdown()); -process.on('uncaughtException', (error) => { - logger.error('Uncaught Exception', { error }); +process.on('uncaughtException', error => { + logger.error('Uncaught Exception', { error }); }); process.on('unhandledRejection', (reason, promise) => { - logger.error('Unhandled Rejection', { reason, promise }); + logger.error('Unhandled Rejection', { reason, promise }); }); diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index b16ed19..92d5726 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -11,194 +11,224 @@ import { getExecutionIdentity } from '../execution-identity'; import logger from '../logger'; type RedisCommandTarget = { - call?: (command: string, ...args: (string | number | Buffer)[]) => Promise; + call?: ( + command: string, + ...args: (string | number | Buffer)[] + ) => Promise; }; let redisCommands: RedisCommandTarget | undefined; async function getRedisCommands(): Promise { - if (redisCommands) return redisCommands; - const { connection } = await import('../queue'); - return connection; + if (redisCommands) return redisCommands; + const { connection } = await import('../queue'); + return connection; } -const sendCommand: SendCommandFn = async (command: string, ...args: (string | number | Buffer)[]): Promise => { - const target = await getRedisCommands(); - if (typeof target.call === 'function') { - const result = await target.call(command, ...args); +const sendCommand: SendCommandFn = async ( + command: string, + ...args: (string | number | Buffer)[] +): Promise => { + const target = await getRedisCommands(); + const commandMethod = (target as Record)[ + command.toLowerCase() + ]; + if (typeof commandMethod !== 'function') { + if (typeof target.call === 'function') { + const result = await target.call(command, ...args); + return result as RedisReply; + } + throw new Error(`Redis command method unavailable: ${command}`); + } + const result = await commandMethod.apply(target, args); return result as RedisReply; - } - const commandMethod = (target as Record)[command.toLowerCase()]; - if (typeof commandMethod !== 'function') { - throw new Error(`Redis command method unavailable: ${command}`); - } - const result = await commandMethod.apply(target, args); - return result as RedisReply; }; type RequestWithRateLimit = Request & { - rateLimit?: { - limit: number; - used: number; - remaining: number; - resetTime?: Date; - }; + rateLimit?: { + limit: number; + used: number; + remaining: number; + resetTime?: Date; + }; }; type RateLimitResponseOptions = { - message: string; - structuredBody?: boolean; - logRejections?: boolean; + message: string; + structuredBody?: boolean; + logRejections?: boolean; }; const unknownPrincipal = 'unknown'; -const keySegment = (value: string | undefined, fallback = unknownPrincipal): string => { - const trimmed = value?.trim(); - return trimmed ? trimmed.replace(/:/g, '_') : fallback; +const keySegment = ( + value: string | undefined, + fallback = unknownPrincipal, +): string => { + const trimmed = value?.trim(); + return trimmed ? trimmed.replace(/:/g, '_') : fallback; }; const hashLabel = (value: string | undefined): string | undefined => { - const trimmed = value?.trim(); - if (!trimmed) return undefined; - return createHash('sha256').update(trimmed).digest('hex').slice(0, 12); + const trimmed = value?.trim(); + if (!trimmed) return undefined; + return createHash('sha256').update(trimmed).digest('hex').slice(0, 12); }; export function setRateLimitRedisForTests(client?: RedisCommandTarget): void { - redisCommands = client; + redisCommands = client; } export function retryAfterSeconds(req: Request, windowMs: number): number { - const resetTime = (req as RequestWithRateLimit).rateLimit?.resetTime; - if (resetTime instanceof Date) { - return Math.max(1, Math.ceil((resetTime.getTime() - Date.now()) / 1000)); - } - return Math.max(1, Math.ceil(windowMs / 1000)); + const resetTime = (req as RequestWithRateLimit).rateLimit?.resetTime; + if (resetTime instanceof Date) { + return Math.max( + 1, + Math.ceil((resetTime.getTime() - Date.now()) / 1000), + ); + } + return Math.max(1, Math.ceil(windowMs / 1000)); } -export function rateLimitResponseBody(message: string, retryAfter: number): { - error: 'rate_limited'; - message: string; - retry_after_seconds: number; +export function rateLimitResponseBody( + message: string, + retryAfter: number, +): { + error: 'rate_limited'; + message: string; + retry_after_seconds: number; } { - const retryAfterLabel = retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; - return { - error: 'rate_limited', - message: `${message} Please retry in ${retryAfterLabel}.`, - retry_after_seconds: retryAfter, - }; + const retryAfterLabel = + retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; + return { + error: 'rate_limited', + message: `${message} Please retry in ${retryAfterLabel}.`, + retry_after_seconds: retryAfter, + }; } export const keyGenerator = (req: Request): string => { - const authReq = req as AuthenticatedRequest; - const identity = getExecutionIdentity(authReq); - if (identity.canonicalUserId) { - return `${keySegment(identity.storageNamespace, 'legacy')}:user:${keySegment(identity.canonicalUserId)}`; - } - return `ip:${keySegment(req.ip)}`; + const authReq = req as AuthenticatedRequest; + const identity = getExecutionIdentity(authReq); + if (identity.canonicalUserId) { + return `${keySegment(identity.storageNamespace, 'legacy')}:user:${keySegment(identity.canonicalUserId)}`; + } + return `ip:${keySegment(req.ip)}`; }; const buildRateLimiter = ( - prefix: string, - windowMs: number, - max: number, - options: RateLimitResponseOptions + prefix: string, + windowMs: number, + max: number, + options: RateLimitResponseOptions, ): RateLimitRequestHandler => { - return rateLimitFactory({ - windowMs, - max, - standardHeaders: 'draft-6', - legacyHeaders: false, - store: new RateLimitRedisStore({ - sendCommand, - prefix: `${prefix}:` - }), - keyGenerator, - handler: (req: Request, res: Response) => { - const retryAfter = retryAfterSeconds(req, windowMs); - const rateLimit = (req as RequestWithRateLimit).rateLimit; - res.setHeader('Retry-After', retryAfter.toString()); - res.setHeader('RateLimit-Limit', (rateLimit?.limit ?? max).toString()); - res.setHeader('RateLimit-Remaining', '0'); - res.setHeader('RateLimit-Reset', retryAfter.toString()); - - if (options.logRejections) { - const authReq = req as AuthenticatedRequest; - const principal = authReq.codeApiPrincipal; - const identity = getExecutionIdentity(authReq); - const hasIdentity = Boolean(identity.canonicalUserId); - logger.warn('CodeAPI rate limit rejected', { - limiter: prefix, - path: req.originalUrl || req.path, - retryAfterSeconds: retryAfter, - limit: rateLimit?.limit ?? max, - windowMs, - principalSource: hasIdentity ? identity.principalSource : undefined, - tenantHash: hasIdentity ? hashLabel(identity.storageNamespace) : undefined, - userHash: hasIdentity ? hashLabel(identity.canonicalUserId) : undefined, - credentialHash: hashLabel(principal?.credentialId), - }); - } - - if (options.structuredBody) { - return res.status(429).json(rateLimitResponseBody(options.message, retryAfter)); - } - const retryAfterLabel = retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; - res.status(429).json({ - error: `${options.message} Please retry in ${retryAfterLabel}.`, - }); - }, - }); + return rateLimitFactory({ + windowMs, + max, + standardHeaders: 'draft-6', + legacyHeaders: false, + store: new RateLimitRedisStore({ + sendCommand, + prefix: `${prefix}:`, + }), + keyGenerator, + handler: (req: Request, res: Response) => { + const retryAfter = retryAfterSeconds(req, windowMs); + const rateLimit = (req as RequestWithRateLimit).rateLimit; + res.setHeader('Retry-After', retryAfter.toString()); + res.setHeader( + 'RateLimit-Limit', + (rateLimit?.limit ?? max).toString(), + ); + res.setHeader('RateLimit-Remaining', '0'); + res.setHeader('RateLimit-Reset', retryAfter.toString()); + + if (options.logRejections) { + const authReq = req as AuthenticatedRequest; + const principal = authReq.codeApiPrincipal; + const identity = getExecutionIdentity(authReq); + const hasIdentity = Boolean(identity.canonicalUserId); + logger.warn('CodeAPI rate limit rejected', { + limiter: prefix, + path: req.originalUrl || req.path, + retryAfterSeconds: retryAfter, + limit: rateLimit?.limit ?? max, + windowMs, + principalSource: hasIdentity + ? identity.principalSource + : undefined, + tenantHash: hasIdentity + ? hashLabel(identity.storageNamespace) + : undefined, + userHash: hasIdentity + ? hashLabel(identity.canonicalUserId) + : undefined, + credentialHash: hashLabel(principal?.credentialId), + }); + } + + if (options.structuredBody) { + return res + .status(429) + .json(rateLimitResponseBody(options.message, retryAfter)); + } + const retryAfterLabel = + retryAfter === 1 ? '1 second' : `${retryAfter} seconds`; + res.status(429).json({ + error: `${options.message} Please retry in ${retryAfterLabel}.`, + }); + }, + }); }; export const createRateLimiter = ( - prefix: string, - windowMs: number, - max: number, - options: RateLimitResponseOptions + prefix: string, + windowMs: number, + max: number, + options: RateLimitResponseOptions, ): RateLimitRequestHandler => { - let limiter: RateLimitRequestHandler | undefined; - const getLimiter = (): RateLimitRequestHandler => { - limiter ??= buildRateLimiter(prefix, windowMs, max, options); - return limiter; - }; - - const lazyLimiter = ((req: Request, res: Response, next: NextFunction) => { - return getLimiter()(req, res, next); - }) as RateLimitRequestHandler; - lazyLimiter.resetKey = (key: string) => getLimiter().resetKey(key); - lazyLimiter.getKey = (key: string) => getLimiter().getKey(key); - return lazyLimiter; + let limiter: RateLimitRequestHandler | undefined; + const getLimiter = (): RateLimitRequestHandler => { + limiter ??= buildRateLimiter(prefix, windowMs, max, options); + return limiter; + }; + + const lazyLimiter = ((req: Request, res: Response, next: NextFunction) => { + return getLimiter()(req, res, next); + }) as RateLimitRequestHandler; + lazyLimiter.resetKey = (key: string) => getLimiter().resetKey(key); + lazyLimiter.getKey = (key: string) => getLimiter().getKey(key); + return lazyLimiter; }; export const executionLimiter = createRateLimiter( - 'exec', - env.EXEC_LIMIT_WINDOW, - env.EXEC_MAX_REQUESTS, - { - message: 'Too many CodeAPI execution requests.', - structuredBody: true, - logRejections: true, - } + 'exec', + env.EXEC_LIMIT_WINDOW, + env.EXEC_MAX_REQUESTS, + { + message: 'Too many CodeAPI execution requests.', + structuredBody: true, + logRejections: true, + }, ); export const uploadLimiter = createRateLimiter( - 'upload', - env.UPLOAD_LIMIT_WINDOW, - env.UPLOAD_MAX_REQUESTS, - { message: 'Too many file uploads.' } + 'upload', + env.UPLOAD_LIMIT_WINDOW, + env.UPLOAD_MAX_REQUESTS, + { message: 'Too many file uploads.' }, ); export const downloadLimiter = createRateLimiter( - 'download', - env.DOWNLOAD_LIMIT_WINDOW, - env.DOWNLOAD_MAX_REQUESTS, - { message: 'Too many file downloads.' } + 'download', + env.DOWNLOAD_LIMIT_WINDOW, + env.DOWNLOAD_MAX_REQUESTS, + { message: 'Too many file downloads.' }, ); export const fetchLimiter = createRateLimiter( - 'fetch', - env.FETCH_LIMIT_WINDOW, - env.FETCH_MAX_REQUESTS, - { message: 'Too many file list requests.' } + 'fetch', + env.FETCH_LIMIT_WINDOW, + env.FETCH_MAX_REQUESTS, + { message: 'Too many file list requests.' }, ); diff --git a/service/src/queue.ts b/service/src/queue.ts index c6f3226..ea2b2ff 100644 --- a/service/src/queue.ts +++ b/service/src/queue.ts @@ -1,105 +1,114 @@ // src/queue.ts -import IORedis from 'ioredis'; import { Queue, QueueEvents } from 'bullmq'; import { setMaxListeners } from 'events'; import type { CommonRedisOptions } from 'ioredis'; -import type * as tls from 'tls'; import type * as t from './types'; import { Jobs, Queues } from './enum'; import { env } from './config'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; -import { bullmqQueueJobs, registerBullmqQueueMetricsCollector } from './metrics'; +import { + bullmqQueueJobs, + registerBullmqQueueMetricsCollector, +} from './metrics'; +import { createRedisConnection, bullmqPrefix } from './redis-connection'; const MAX_RECONNECT_ATTEMPTS = 5; const RECONNECT_DELAY = 2000; -const retryStrategy: CommonRedisOptions['retryStrategy'] = (times) => { - if (times > MAX_RECONNECT_ATTEMPTS) { - logger.error(`Failed to connect to Redis after ${times} attempts`); - return null; - } - logger.warn(`Retrying Redis connection attempt ${times}`); - return RECONNECT_DELAY; +const retryStrategy: CommonRedisOptions['retryStrategy'] = times => { + if (times > MAX_RECONNECT_ATTEMPTS) { + logger.error(`Failed to connect to Redis after ${times} attempts`); + return null; + } + logger.warn(`Retrying Redis connection attempt ${times}`); + return RECONNECT_DELAY; }; -const reconnectOnError: CommonRedisOptions['reconnectOnError'] = (err) => { - logger.error('Redis connection error:', err); - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; +const reconnectOnError: CommonRedisOptions['reconnectOnError'] = err => { + logger.error('Redis connection error:', err); + const targetError = 'READONLY'; + if (err.message.includes(targetError)) { + return true; + } + return false; }; -const connection = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - maxRetriesPerRequest: null, - retryStrategy, - reconnectOnError, - enableReadyCheck: true, - connectTimeout: 10000, - disconnectTimeout: 2000, - ...redisKeepAliveOptions(), - tls: process.env.REDIS_TLS === 'true' ? { - rejectUnauthorized: false - } as tls.ConnectionOptions : undefined, - // Alternative DNS lookup for AWS ElastiCache TLS connections - ...(env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}) +const connection = createRedisConnection({ + maxRetriesPerRequest: null, + retryStrategy, + reconnectOnError, + enableReadyCheck: true, + disconnectTimeout: 2000, }); +const prefix = bullmqPrefix(); + // Global queues - no INSTANCE_ID prefix // This enables horizontal scaling where any worker can process any job -const pyQueue = new Queue(Queues.python, { connection }); -const otherQueue = new Queue(Queues.other, { connection }); +const pyQueue = new Queue(Queues.python, { + connection, + prefix, +}); +const otherQueue = new Queue( + Queues.other, + { connection, prefix }, +); -const pyQueueEvents = new QueueEvents(Queues.python, { connection }); -const otherQueueEvents = new QueueEvents(Queues.other, { connection }); +const pyQueueEvents = new QueueEvents(Queues.python, { connection, prefix }); +const otherQueueEvents = new QueueEvents(Queues.other, { connection, prefix }); const queueMetricStates = ['waiting', 'active', 'delayed'] as const; const queueMetricSources = [ - { name: Queues.python, queue: pyQueue }, - { name: Queues.other, queue: otherQueue }, + { name: Queues.python, queue: pyQueue }, + { name: Queues.other, queue: otherQueue }, ] as const; const QUEUE_METRICS_TIMEOUT_MS = 1000; -async function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { - let timeout: ReturnType | undefined; - void promise.catch(() => undefined); - const timeoutPromise = new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error(message)), timeoutMs); - }); - try { - return await Promise.race([promise, timeoutPromise]); - } finally { - if (timeout !== undefined) { - clearTimeout(timeout); +async function withTimeout( + promise: Promise, + timeoutMs: number, + message: string, +): Promise { + let timeout: ReturnType | undefined; + void promise.catch(() => undefined); + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + }); + try { + return await Promise.race([promise, timeoutPromise]); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } } - } } registerBullmqQueueMetricsCollector(async () => { - await Promise.all(queueMetricSources.map(async ({ name, queue }) => { - try { - const counts = await withTimeout( - queue.getJobCounts(...queueMetricStates), - QUEUE_METRICS_TIMEOUT_MS, - `Timed out collecting BullMQ queue metrics for ${name}`, - ); - for (const state of queueMetricStates) { - bullmqQueueJobs.set({ queue: name, state }, counts[state] ?? 0); - } - } catch (error) { - logger.warn('Failed to collect BullMQ queue metrics', { queue: name, error }); - for (const state of queueMetricStates) { - bullmqQueueJobs.remove({ queue: name, state }); - } - } - })); + await Promise.all( + queueMetricSources.map(async ({ name, queue }) => { + try { + const counts = await withTimeout( + queue.getJobCounts(...queueMetricStates), + QUEUE_METRICS_TIMEOUT_MS, + `Timed out collecting BullMQ queue metrics for ${name}`, + ); + for (const state of queueMetricStates) { + bullmqQueueJobs.set( + { queue: name, state }, + counts[state] ?? 0, + ); + } + } catch (error) { + logger.warn('Failed to collect BullMQ queue metrics', { + queue: name, + error, + }); + for (const state of queueMetricStates) { + bullmqQueueJobs.remove({ queue: name, state }); + } + } + }), + ); }); /* job.waitUntilFinished() attaches a short-lived `closing` listener to the diff --git a/service/src/redis-connection.test.ts b/service/src/redis-connection.test.ts new file mode 100644 index 0000000..4632c53 --- /dev/null +++ b/service/src/redis-connection.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, test, afterEach } from 'bun:test'; +import { + parseRedisNodes, + isClusterMode, + buildTlsOptions, + bullmqPrefix, +} from './redis-connection'; + +// ────────────────────────────────────────────────────────────────────────────── +// We test the pure helper functions from redis-connection.ts without +// establishing any real network connection. ioredis constructors are NOT +// called in these tests. +// ────────────────────────────────────────────────────────────────────────────── + +// Preserve original env so each test can start clean +const ORIGINAL_ENV: Record = {}; +for (const key of Object.keys(process.env)) { + if (key.startsWith('REDIS_') || key === 'USE_REDIS_CLUSTER') { + ORIGINAL_ENV[key] = process.env[key]; + } +} + +function resetEnv(overrides: Record = {}): void { + for (const key of Object.keys(process.env)) { + if (key.startsWith('REDIS_') || key === 'USE_REDIS_CLUSTER') { + delete process.env[key]; + } + } + for (const [k, v] of Object.entries(overrides)) { + if (v !== undefined) process.env[k] = v; + } +} + +afterEach(() => { + resetEnv(ORIGINAL_ENV); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// parseRedisNodes +// ────────────────────────────────────────────────────────────────────────────── + +describe('parseRedisNodes', () => { + test('single host uses REDIS_PORT default 6379', () => { + resetEnv({ REDIS_HOST: 'myhost' }); + expect(parseRedisNodes()).toEqual([{ host: 'myhost', port: 6379 }]); + }); + + test('single host with embedded port', () => { + resetEnv({ REDIS_HOST: 'myhost:6380' }); + expect(parseRedisNodes()).toEqual([{ host: 'myhost', port: 6380 }]); + }); + + test('comma-separated hosts without ports', () => { + resetEnv({ REDIS_HOST: 'node1,node2,node3', REDIS_PORT: '6380' }); + expect(parseRedisNodes()).toEqual([ + { host: 'node1', port: 6380 }, + { host: 'node2', port: 6380 }, + { host: 'node3', port: 6380 }, + ]); + }); + + test('comma-separated hosts with embedded ports', () => { + resetEnv({ REDIS_HOST: 'node1:6379,node2:6380,node3:6381' }); + expect(parseRedisNodes()).toEqual([ + { host: 'node1', port: 6379 }, + { host: 'node2', port: 6380 }, + { host: 'node3', port: 6381 }, + ]); + }); + + test('trims whitespace around entries', () => { + resetEnv({ REDIS_HOST: ' node1 , node2 ' }); + expect(parseRedisNodes()).toEqual([ + { host: 'node1', port: 6379 }, + { host: 'node2', port: 6379 }, + ]); + }); + + test('defaults to "redis" when REDIS_HOST is unset', () => { + resetEnv({}); + expect(parseRedisNodes()).toEqual([{ host: 'redis', port: 6379 }]); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// isClusterMode +// ────────────────────────────────────────────────────────────────────────────── + +describe('isClusterMode', () => { + test('false by default', () => { + resetEnv({ REDIS_HOST: 'redis' }); + expect(isClusterMode()).toBe(false); + }); + + test('true when USE_REDIS_CLUSTER=true', () => { + resetEnv({ REDIS_HOST: 'redis', USE_REDIS_CLUSTER: 'true' }); + expect(isClusterMode()).toBe(true); + }); + + test('true when REDIS_HOST contains a comma', () => { + resetEnv({ REDIS_HOST: 'node1,node2' }); + expect(isClusterMode()).toBe(true); + }); + + test('false when REDIS_HOST is a single host', () => { + resetEnv({ + REDIS_HOST: 'single-host:6379', + USE_REDIS_CLUSTER: 'false', + }); + expect(isClusterMode()).toBe(false); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// buildTlsOptions +// ────────────────────────────────────────────────────────────────────────────── + +describe('buildTlsOptions', () => { + test('returns undefined when neither REDIS_TLS nor REDIS_CA is set', () => { + resetEnv({}); + expect(buildTlsOptions()).toBeUndefined(); + }); + + test('returns { rejectUnauthorized: false } when REDIS_TLS=true and no CA', () => { + resetEnv({ REDIS_TLS: 'true' }); + expect(buildTlsOptions()).toEqual({ rejectUnauthorized: false }); + }); + + test('returns { ca } when REDIS_CA points to an existing file', () => { + const tmpFile = '/tmp/test-redis-ca.pem'; + const pem = + '-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n'; + const { writeFileSync, unlinkSync } = + require('fs') as typeof import('fs'); + writeFileSync(tmpFile, pem, 'utf8'); + + resetEnv({ REDIS_CA: tmpFile }); + try { + expect(buildTlsOptions()).toEqual({ ca: pem }); + } finally { + unlinkSync(tmpFile); + } + }); + + test('CA takes precedence over REDIS_TLS=true', () => { + const tmpFile = '/tmp/test-redis-ca2.pem'; + const pem = + '-----BEGIN CERTIFICATE-----\ndata\n-----END CERTIFICATE-----\n'; + const { writeFileSync, unlinkSync } = + require('fs') as typeof import('fs'); + writeFileSync(tmpFile, pem, 'utf8'); + + resetEnv({ REDIS_CA: tmpFile, REDIS_TLS: 'true' }); + try { + expect(buildTlsOptions()).toEqual({ ca: pem }); + } finally { + unlinkSync(tmpFile); + } + }); + + test('returns undefined when REDIS_CA file does not exist', () => { + resetEnv({ REDIS_CA: '/tmp/does-not-exist-ca-xyz.pem' }); + expect(buildTlsOptions()).toBeUndefined(); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// bullmqPrefix +// ────────────────────────────────────────────────────────────────────────────── + +describe('bullmqPrefix', () => { + test('returns undefined in standalone mode', () => { + resetEnv({ REDIS_HOST: 'redis' }); + expect(bullmqPrefix()).toBeUndefined(); + }); + + test('returns "{codeapi}" in cluster mode via USE_REDIS_CLUSTER', () => { + resetEnv({ USE_REDIS_CLUSTER: 'true' }); + expect(bullmqPrefix()).toBe('{codeapi}'); + }); + + test('returns "{codeapi}" when REDIS_HOST contains multiple nodes', () => { + resetEnv({ REDIS_HOST: 'node1,node2,node3' }); + expect(bullmqPrefix()).toBe('{codeapi}'); + }); +}); diff --git a/service/src/redis-connection.ts b/service/src/redis-connection.ts new file mode 100644 index 0000000..17c75a0 --- /dev/null +++ b/service/src/redis-connection.ts @@ -0,0 +1,157 @@ +import { existsSync, readFileSync } from 'fs'; +import IORedis, { Cluster } from 'ioredis'; +import type { + Redis, + RedisOptions, + ClusterOptions, + CommonRedisOptions, +} from 'ioredis'; +import { redisKeepAliveOptions } from './redis-options'; +import logger from './logger'; + +export type RedisClient = Redis | Cluster; + +const MAX_CLUSTER_RECONNECT_ATTEMPTS = 5; + +/** + * Parse REDIS_HOST into an array of {host, port} startup nodes. + * Accepts comma-separated entries in "host:port" or plain "host" format. + * Falls back to REDIS_PORT (default 6379) when no port is embedded. + */ +export function parseRedisNodes(): Array<{ host: string; port: number }> { + const defaultPort = Number(process.env.REDIS_PORT) || 6379; + const raw = process.env.REDIS_HOST ?? 'redis'; + return raw.split(',').map(entry => { + const trimmed = entry.trim(); + const colonIdx = trimmed.lastIndexOf(':'); + if (colonIdx > 0) { + const potentialPort = Number(trimmed.slice(colonIdx + 1)); + if (Number.isInteger(potentialPort) && potentialPort > 0) { + return { + host: trimmed.slice(0, colonIdx), + port: potentialPort, + }; + } + } + return { host: trimmed, port: defaultPort }; + }); +} + +/** + * Returns true when Redis Cluster mode is active. + * Cluster mode is enabled when USE_REDIS_CLUSTER=true or REDIS_HOST contains + * a comma-separated list of nodes. + */ +export function isClusterMode(): boolean { + return ( + process.env.USE_REDIS_CLUSTER === 'true' || + (process.env.REDIS_HOST ?? '').includes(',') + ); +} + +/** + * Read and return the PEM-encoded CA certificate from the path given by the + * REDIS_CA environment variable. Returns null when the variable is unset or + * the file cannot be read. + */ +function readCACert(): string | null { + const caPath = process.env.REDIS_CA; + if (!caPath) return null; + try { + if (!existsSync(caPath)) { + logger.warn(`Redis CA certificate file not found: ${caPath}`); + return null; + } + return readFileSync(caPath, 'utf8'); + } catch (error) { + logger.error(`Failed to read Redis CA certificate: ${caPath}`, { + error, + }); + return null; + } +} + +/** + * Build the TLS options for an ioredis connection: + * - REDIS_CA points to a file → use that CA (certificate fully validated). + * - REDIS_TLS=true without a CA → disable certificate validation (backward-compatible). + * - Neither set → no TLS. + */ +export function buildTlsOptions(): Record | undefined { + const ca = readCACert(); + if (ca) return { ca }; + if (process.env.REDIS_TLS === 'true') return { rejectUnauthorized: false }; + return undefined; +} + +/** + * Returns the BullMQ `prefix` required in cluster mode so that all queue keys + * land in the same hash slot. Returns undefined in standalone mode (no prefix). + */ +export function bullmqPrefix(): string | undefined { + return isClusterMode() ? '{codeapi}' : undefined; +} + +type ConnectionOverrides = Partial< + Pick< + CommonRedisOptions, + | 'maxRetriesPerRequest' + | 'enableReadyCheck' + | 'retryStrategy' + | 'reconnectOnError' + | 'disconnectTimeout' + > +>; + +/** + * Create an ioredis client – standalone `Redis` or `Cluster` – based on the + * current environment configuration. + * + * Callers supply per-client `overrides` (retry strategy, maxRetriesPerRequest, + * enableReadyCheck, …) to preserve each component's existing connection semantics. + * + * In cluster mode the overrides are forwarded to `redisOptions` (applied + * per-node). The cluster-level reconnect loop is handled independently via + * `clusterRetryStrategy`. + */ +export function createRedisConnection( + overrides: ConnectionOverrides, +): RedisClient { + const tls = buildTlsOptions() as RedisOptions['tls']; + const dnsLookup: ClusterOptions['dnsLookup'] | undefined = + process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true' + ? (address, callback) => callback(null, address) + : undefined; + + const baseOptions: RedisOptions = { + password: process.env.REDIS_PASSWORD, + connectTimeout: 10000, + ...redisKeepAliveOptions(), + ...(tls !== undefined ? { tls } : {}), + ...(dnsLookup ? { dnsLookup } : {}), + ...overrides, + }; + + if (isClusterMode()) { + const nodes = parseRedisNodes(); + return new Cluster(nodes, { + ...(dnsLookup ? { dnsLookup } : {}), + redisOptions: baseOptions, + clusterRetryStrategy(times) { + if (times > MAX_CLUSTER_RECONNECT_ATTEMPTS) { + logger.error( + `Redis cluster giving up after ${times} reconnection attempts`, + ); + return null; + } + const base = Math.min(2 ** times * 100, 3000); + const jitter = Math.floor(Math.random() * Math.min(base, 1000)); + return Math.min(base + jitter, 3000); + }, + enableOfflineQueue: true, + }); + } + + const [{ host, port }] = parseRedisNodes(); + return new IORedis({ host, port, ...baseOptions }); +} diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 562e06d..06d232f 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -22,16 +22,18 @@ import axios from 'axios'; import { nanoid } from 'nanoid'; import type { Redis } from 'ioredis'; +import type { Cluster } from 'ioredis'; import type * as t from '../types'; import type { LCTool } from '../preamble'; import { connection } from '../queue'; import { env } from '../config'; +import { isClusterMode } from '../redis-connection'; import { internalServiceHeaders } from '../internal-service-auth'; import logger from '../logger'; import { - ptcReplayHistorySize, - ptcReplayHistoryEntries, - ptcReplayStaleCleanups, + ptcReplayHistorySize, + ptcReplayHistoryEntries, + ptcReplayStaleCleanups, } from '../metrics'; // --------------------------------------------------------------------------- @@ -54,7 +56,10 @@ export const MAX_EXECUTION_STATE_BYTES = 10_000_000; * expire while the holder is still executing and a second continuation * would be able to mutate `tool_history`/`exec_state` concurrently. Floor * at 10 minutes so short job timeouts don't produce a tiny lock window. */ -export const REPLAY_LOCK_TTL_MS = Math.max(10 * 60 * 1000, env.JOB_TIMEOUT * 2 + 30_000); +export const REPLAY_LOCK_TTL_MS = Math.max( + 10 * 60 * 1000, + env.JOB_TIMEOUT * 2 + 30_000, +); /** Per-entry cap for a single serialized tool result (JSON bytes). Keeps the * Redis hash and the `_ptc_history.json` injected into the sandbox bounded @@ -88,85 +93,85 @@ export const CALL_ID_RE = /^call_\d{3,6}$/; // --------------------------------------------------------------------------- export interface ExecutionState { - execution_id: string; - session_id: string; - sessionKey?: string; - userId: string; - tenantId?: string; - canonicalUserId?: string; - orgId?: string; - serviceId?: string; - externalUserId?: string; - principalSource?: string; - authContextHash?: string; - /** Optional even though current construction sites always set a string, - * because `ExecutionState` is JSON-deserialized from Redis and entries - * persisted by prior binaries (or future schema migrations) can omit - * it. The two continuation paths intentionally bypass the apiKeyId - * ownership check when the persisted value is missing — see - * `checkContinuationPreconditions` and the blocking continuation auth - * in `programmatic-router.ts`. Tighten on both paths in a follow-up - * after one `EXECUTION_STATE_TTL` window post a trusted-source - * apiKeyId invariant. */ - apiKeyId?: string; - startTime: number; - /** - * Wall-clock ms of the last interaction that advanced this execution (initial - * submit, continuation accept, or sandbox iteration finish). Used by the - * stale-execution sweeper so long-running but still-active replay flows - * aren't reaped purely based on total age. - */ - lastActivity?: number; - mode: 'blocking' | 'replay'; - /** Replay-only: persisted so continuations can re-enqueue without the client re-sending code. */ - userCode?: string; - tools?: LCTool[]; - files?: t.RequestFile[]; - isPyPlot?: boolean; - timeout?: number; - callCount?: number; - /** Running total of bytes persisted into `tool_history:`. - * Tracked so continuations can reject oversized histories without scanning - * the full hash on every request. */ - historyBytes?: number; - /** Cumulative set of call_ids ever emitted by the sandbox for this - * execution, serialized as an array. Used to reject incoming - * `tool_results` entries whose `call_id` was never actually requested - * — a buggy or malicious client could otherwise pre-seed arbitrary - * ids (e.g. `call_042`) that later replay iterations would consume - * as cache hits and skip real tool calls. Populated in `runAndRespond` - * right before emitting a `tool_call_required` response. */ - emittedCallIds?: string[]; - /** Replay-only: metadata for emitted tool calls, keyed by `id` at use sites. - * Stored so continuation commits can persist the original tool name and a - * compact input signature beside the result. Bash replay uses that metadata - * to safely resolve background tool calls whose completion order can differ - * between runs. */ - emittedToolCalls?: Array<{ - id: string; - name: string; - input_hash?: string; - call_site?: string; - }>; - /** Target language for PTC. Defaults to 'python'. Bash is replay-only. */ - language?: 'python' | 'bash'; - /** Blocking-only legacy fields. New blocking results live in - * `exec_result:` (see setBlockingResult); `jobResult` is kept on the - * interface only as a deploy-time fallback so executions whose state was - * persisted by an older binary still resolve correctly while in-flight. */ - jobCompleted?: boolean; - jobResult?: t.ExecuteResult; - jobError?: string; + execution_id: string; + session_id: string; + sessionKey?: string; + userId: string; + tenantId?: string; + canonicalUserId?: string; + orgId?: string; + serviceId?: string; + externalUserId?: string; + principalSource?: string; + authContextHash?: string; + /** Optional even though current construction sites always set a string, + * because `ExecutionState` is JSON-deserialized from Redis and entries + * persisted by prior binaries (or future schema migrations) can omit + * it. The two continuation paths intentionally bypass the apiKeyId + * ownership check when the persisted value is missing — see + * `checkContinuationPreconditions` and the blocking continuation auth + * in `programmatic-router.ts`. Tighten on both paths in a follow-up + * after one `EXECUTION_STATE_TTL` window post a trusted-source + * apiKeyId invariant. */ + apiKeyId?: string; + startTime: number; + /** + * Wall-clock ms of the last interaction that advanced this execution (initial + * submit, continuation accept, or sandbox iteration finish). Used by the + * stale-execution sweeper so long-running but still-active replay flows + * aren't reaped purely based on total age. + */ + lastActivity?: number; + mode: 'blocking' | 'replay'; + /** Replay-only: persisted so continuations can re-enqueue without the client re-sending code. */ + userCode?: string; + tools?: LCTool[]; + files?: t.RequestFile[]; + isPyPlot?: boolean; + timeout?: number; + callCount?: number; + /** Running total of bytes persisted into `tool_history:`. + * Tracked so continuations can reject oversized histories without scanning + * the full hash on every request. */ + historyBytes?: number; + /** Cumulative set of call_ids ever emitted by the sandbox for this + * execution, serialized as an array. Used to reject incoming + * `tool_results` entries whose `call_id` was never actually requested + * — a buggy or malicious client could otherwise pre-seed arbitrary + * ids (e.g. `call_042`) that later replay iterations would consume + * as cache hits and skip real tool calls. Populated in `runAndRespond` + * right before emitting a `tool_call_required` response. */ + emittedCallIds?: string[]; + /** Replay-only: metadata for emitted tool calls, keyed by `id` at use sites. + * Stored so continuation commits can persist the original tool name and a + * compact input signature beside the result. Bash replay uses that metadata + * to safely resolve background tool calls whose completion order can differ + * between runs. */ + emittedToolCalls?: Array<{ + id: string; + name: string; + input_hash?: string; + call_site?: string; + }>; + /** Target language for PTC. Defaults to 'python'. Bash is replay-only. */ + language?: 'python' | 'bash'; + /** Blocking-only legacy fields. New blocking results live in + * `exec_result:` (see setBlockingResult); `jobResult` is kept on the + * interface only as a deploy-time fallback so executions whose state was + * persisted by an older binary still resolve correctly while in-flight. */ + jobCompleted?: boolean; + jobResult?: t.ExecuteResult; + jobError?: string; } export interface HistoryEntry { - result: unknown; - is_error?: boolean; - error_message?: string; - received_at: number; - tool_name?: string; - input_hash?: string; - call_site?: string; + result: unknown; + is_error?: boolean; + error_message?: string; + received_at: number; + tool_name?: string; + input_hash?: string; + call_site?: string; } /** Describes how an incoming `tool_results` batch would interact with the @@ -174,15 +179,15 @@ export interface HistoryEntry { * Redis mutation so the caller can cheaply enforce caps without risking a * half-applied update. */ export interface ToolHistoryDelta { - /** Serialized entries keyed by call_id, ready to pass to `HSET`. */ - serializedByCallId: Map; - /** Call_ids that weren't previously persisted. */ - newCallIds: string[]; - /** Signed byte delta this batch would apply to the aggregate history - * size: `sum(newEntryBytes)` for new call_ids plus - * `sum(newEntryBytes - oldEntryBytes)` for overwrites. Negative when a - * retry shrinks a previously-persisted entry. */ - bytesDelta: number; + /** Serialized entries keyed by call_id, ready to pass to `HSET`. */ + serializedByCallId: Map; + /** Call_ids that weren't previously persisted. */ + newCallIds: string[]; + /** Signed byte delta this batch would apply to the aggregate history + * size: `sum(newEntryBytes)` for new call_ids plus + * `sum(newEntryBytes - oldEntryBytes)` for overwrites. Negative when a + * retry shrinks a previously-persisted entry. */ + bytesDelta: number; } /** Thrown by `setExecutionState` / `commitToolHistoryAndState` when the @@ -193,14 +198,16 @@ export interface ToolHistoryDelta { * valid request, which is a client/input sizing problem, not a server * failure. */ export class ExecutionStateTooLargeError extends Error { - readonly bytes: number; - readonly cap: number; - constructor(execution_id: string, bytes: number, cap: number) { - super(`ExecutionState for ${execution_id} is ${bytes} bytes, exceeds cap ${cap}`); - this.name = 'ExecutionStateTooLargeError'; - this.bytes = bytes; - this.cap = cap; - } + readonly bytes: number; + readonly cap: number; + constructor(execution_id: string, bytes: number, cap: number) { + super( + `ExecutionState for ${execution_id} is ${bytes} bytes, exceeds cap ${cap}`, + ); + this.name = 'ExecutionStateTooLargeError'; + this.bytes = bytes; + this.cap = cap; + } } // --------------------------------------------------------------------------- @@ -224,51 +231,51 @@ export class ExecutionStateTooLargeError extends Error { * partial `redis.call` arg parsing); the integration suite still runs * against a real Redis container so any divergence surfaces there. */ type RedisWithScripts = Redis & { - releaseExecutionLockScript(lockKey: string, token: string): Promise; - setExecutionResultScript( - stateKey: string, - resultKey: string, - stateJson: string, - resultJson: string, - ttlSeconds: string, - ): Promise; - setExecutionErrorScript( - stateKey: string, - stateJson: string, - ttlSeconds: string, - ): Promise; + releaseExecutionLockScript(lockKey: string, token: string): Promise; + setExecutionResultScript( + stateKey: string, + resultKey: string, + stateJson: string, + resultJson: string, + ttlSeconds: string, + ): Promise; + setExecutionErrorScript( + stateKey: string, + stateJson: string, + ttlSeconds: string, + ): Promise; }; const SCRIPTS_REGISTERED = Symbol.for('replay-state.scriptsRegistered'); function registerScripts(client: Redis): RedisWithScripts { - const tagged = client as Redis & { [SCRIPTS_REGISTERED]?: true }; - if (tagged[SCRIPTS_REGISTERED]) return client as RedisWithScripts; - client.defineCommand('releaseExecutionLockScript', { - numberOfKeys: 1, - lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", - }); - client.defineCommand('setExecutionResultScript', { - numberOfKeys: 2, - lua: `if redis.call('exists', KEYS[1]) == 1 then + const tagged = client as Redis & { [SCRIPTS_REGISTERED]?: true }; + if (tagged[SCRIPTS_REGISTERED]) return client as RedisWithScripts; + client.defineCommand('releaseExecutionLockScript', { + numberOfKeys: 1, + lua: "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", + }); + client.defineCommand('setExecutionResultScript', { + numberOfKeys: 2, + lua: `if redis.call('exists', KEYS[1]) == 1 then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[3]) redis.call('set', KEYS[2], ARGV[2], 'EX', ARGV[3]) return 1 else return 0 end`, - }); - client.defineCommand('setExecutionErrorScript', { - numberOfKeys: 1, - lua: `if redis.call('exists', KEYS[1]) == 1 then + }); + client.defineCommand('setExecutionErrorScript', { + numberOfKeys: 1, + lua: `if redis.call('exists', KEYS[1]) == 1 then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) return 1 else return 0 end`, - }); - tagged[SCRIPTS_REGISTERED] = true; - return client as RedisWithScripts; + }); + tagged[SCRIPTS_REGISTERED] = true; + return client as RedisWithScripts; } /** Module-level Redis handle. Defaults to the shared queue connection so @@ -282,13 +289,13 @@ let redis: RedisWithScripts = registerScripts(connection); * `setRedisForTests`/`resetRedisForTests` lets a test suite isolate per-test * keyspaces by handing in fresh mock instances. */ export function setRedisForTests(client: Redis): void { - redis = registerScripts(client); + redis = registerScripts(client); } /** Restore the production Redis handle. Useful after a test suite finishes * so accidental reuse doesn't leak a closed mock connection. */ export function resetRedisForTests(): void { - redis = registerScripts(connection); + redis = registerScripts(connection); } // --------------------------------------------------------------------------- @@ -299,38 +306,44 @@ export function resetRedisForTests(): void { * key; normalize on read. Safe to delete one EXECUTION_STATE_TTL window * after the external_user_id rollout. */ export function normalizeExecutionState(state: ExecutionState): ExecutionState { - const legacy = (state as Record)['chcUserId']; // leak-check:allow - if (state.externalUserId == null && typeof legacy === 'string') { - state.externalUserId = legacy; - } - return state; + const legacy = (state as Record)['chcUserId']; // leak-check:allow + if (state.externalUserId == null && typeof legacy === 'string') { + state.externalUserId = legacy; + } + return state; } -export async function getExecutionState(execution_id: string): Promise { - const data = await redis.get(`exec_state:${execution_id}`); - return data != null ? normalizeExecutionState(JSON.parse(data) as ExecutionState) : null; +export async function getExecutionState( + execution_id: string, +): Promise { + const data = await redis.get(`exec_state:${execution_id}`); + return data != null + ? normalizeExecutionState(JSON.parse(data) as ExecutionState) + : null; } export async function setExecutionState(state: ExecutionState): Promise { - const serialized = JSON.stringify(state); - const serializedBytes = Buffer.byteLength(serialized, 'utf8'); - if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { - throw new ExecutionStateTooLargeError( - state.execution_id, - serializedBytes, - MAX_EXECUTION_STATE_BYTES, + const serialized = JSON.stringify(state); + const serializedBytes = Buffer.byteLength(serialized, 'utf8'); + if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { + throw new ExecutionStateTooLargeError( + state.execution_id, + serializedBytes, + MAX_EXECUTION_STATE_BYTES, + ); + } + await redis.set( + `exec_state:${state.execution_id}`, + serialized, + 'EX', + EXECUTION_STATE_TTL, ); - } - await redis.set( - `exec_state:${state.execution_id}`, - serialized, - 'EX', - EXECUTION_STATE_TTL, - ); } -export async function deleteExecutionState(execution_id: string): Promise { - await redis.del(`exec_state:${execution_id}`); +export async function deleteExecutionState( + execution_id: string, +): Promise { + await redis.del(`exec_state:${execution_id}`); } // --------------------------------------------------------------------------- @@ -342,24 +355,32 @@ export async function deleteExecutionState(execution_id: string): Promise * lock token on success (used to release only the current holder) or `null` if * another request holds the lock. Prevents TOCTOU on concurrent continuations. */ -export async function acquireExecutionLock(execution_id: string): Promise { - const token = nanoid(); - const result = await redis.set( - `exec_lock:${execution_id}`, - token, - 'PX', - REPLAY_LOCK_TTL_MS, - 'NX', - ); - return result === 'OK' ? token : null; -} - -export async function releaseExecutionLock(execution_id: string, token: string): Promise { - try { - await redis.releaseExecutionLockScript(`exec_lock:${execution_id}`, token); - } catch (err) { - logger.warn('Failed to release exec lock', { execution_id, err }); - } +export async function acquireExecutionLock( + execution_id: string, +): Promise { + const token = nanoid(); + const result = await redis.set( + `exec_lock:${execution_id}`, + token, + 'PX', + REPLAY_LOCK_TTL_MS, + 'NX', + ); + return result === 'OK' ? token : null; +} + +export async function releaseExecutionLock( + execution_id: string, + token: string, +): Promise { + try { + await redis.releaseExecutionLockScript( + `exec_lock:${execution_id}`, + token, + ); + } catch (err) { + logger.warn('Failed to release exec lock', { execution_id, err }); + } } // --------------------------------------------------------------------------- @@ -368,27 +389,62 @@ export async function releaseExecutionLock(execution_id: string, token: string): /** Iterate matching Redis keys with SCAN instead of the blocking KEYS command. * Stops collecting once `limit` keys have been gathered to keep the array - * bounded on degenerate datasets. */ + * bounded on degenerate datasets. In cluster mode, SCAN is issued against + * every master node individually because `Cluster` has no top-level + * `scanStream`; each node owns a disjoint set of hash slots so there are + * no duplicates across nodes. */ export async function scanKeys( - match: string, - count = 200, - limit = SCAN_KEYS_DEFAULT_LIMIT, + match: string, + count = 200, + limit = SCAN_KEYS_DEFAULT_LIMIT, ): Promise { - const stream = redis.scanStream({ match, count }); - const out: string[] = []; - for await (const batch of stream as AsyncIterable) { - for (const key of batch) { - out.push(key); - if (out.length >= limit) { - stream.destroy(); - logger.warn('scanKeys hit limit; remaining keys deferred to next pass', { - match, limit, - }); + const out: string[] = []; + + if (isClusterMode()) { + // Fan out over every master shard; each owns a disjoint slice of the keyspace. + const nodes = (redis as unknown as Cluster).nodes('master'); + for (const node of nodes) { + if (out.length >= limit) break; + const stream = node.scanStream({ match, count }); + for await (const batch of stream as AsyncIterable) { + for (const key of batch) { + out.push(key); + if (out.length >= limit) { + stream.destroy(); + logger.warn( + 'scanKeys hit limit; remaining keys deferred to next pass', + { + match, + limit, + }, + ); + break; + } + } + if (out.length >= limit) break; + } + } return out; - } } - } - return out; + + const stream = redis.scanStream({ match, count }); + for await (const batch of stream as AsyncIterable) { + for (const key of batch) { + out.push(key); + if (out.length >= limit) { + stream.destroy(); + logger.warn( + 'scanKeys hit limit; remaining keys deferred to next pass', + { + match, + limit, + }, + ); + return out; + } + } + } + return out; } // --------------------------------------------------------------------------- @@ -406,25 +462,32 @@ export async function scanKeys( * on the lighter `exec_state:` blob the replay path mutates on every * continuation. */ function blockingResultKey(execution_id: string): string { - return `exec_result:${execution_id}`; + return `exec_result:${execution_id}`; } -export async function setBlockingResult(execution_id: string, result: t.ExecuteResult): Promise { - await redis.set( - blockingResultKey(execution_id), - JSON.stringify(result), - 'EX', - EXECUTION_STATE_TTL, - ); +export async function setBlockingResult( + execution_id: string, + result: t.ExecuteResult, +): Promise { + await redis.set( + blockingResultKey(execution_id), + JSON.stringify(result), + 'EX', + EXECUTION_STATE_TTL, + ); } -export async function getBlockingResult(execution_id: string): Promise { - const data = await redis.get(blockingResultKey(execution_id)); - return data != null ? (JSON.parse(data) as t.ExecuteResult) : null; +export async function getBlockingResult( + execution_id: string, +): Promise { + const data = await redis.get(blockingResultKey(execution_id)); + return data != null ? (JSON.parse(data) as t.ExecuteResult) : null; } -export async function deleteBlockingResult(execution_id: string): Promise { - await redis.del(blockingResultKey(execution_id)); +export async function deleteBlockingResult( + execution_id: string, +): Promise { + await redis.del(blockingResultKey(execution_id)); } /** Persist the terminal result of a blocking execution. @@ -442,51 +505,71 @@ export async function deleteBlockingResult(execution_id: string): Promise * and, only if so, writes BOTH the updated state (with `jobCompleted=true`) * and the result blob in a single hop. If cleanup has already removed the * state, the entire update is skipped. */ -export async function setExecutionResult(execution_id: string, result: t.ExecuteResult): Promise { - const stateKey = `exec_state:${execution_id}`; - const resultKey = `exec_result:${execution_id}`; - const existing = await getExecutionState(execution_id); - if (!existing) return; - const updated: ExecutionState = { ...existing, jobCompleted: true, jobResult: undefined }; - const updatedSerialized = JSON.stringify(updated); - if (Buffer.byteLength(updatedSerialized, 'utf8') > MAX_EXECUTION_STATE_BYTES) { - throw new ExecutionStateTooLargeError( - execution_id, - Buffer.byteLength(updatedSerialized, 'utf8'), - MAX_EXECUTION_STATE_BYTES, +export async function setExecutionResult( + execution_id: string, + result: t.ExecuteResult, +): Promise { + const stateKey = `exec_state:${execution_id}`; + const resultKey = `exec_result:${execution_id}`; + const existing = await getExecutionState(execution_id); + if (!existing) return; + const updated: ExecutionState = { + ...existing, + jobCompleted: true, + jobResult: undefined, + }; + const updatedSerialized = JSON.stringify(updated); + if ( + Buffer.byteLength(updatedSerialized, 'utf8') > MAX_EXECUTION_STATE_BYTES + ) { + throw new ExecutionStateTooLargeError( + execution_id, + Buffer.byteLength(updatedSerialized, 'utf8'), + MAX_EXECUTION_STATE_BYTES, + ); + } + /** EXISTS-then-SET both keys atomically so a concurrent + * `cleanupExecution` racing between our `getExecutionState` above and + * the writes below cannot leave an orphan blob behind. The Lua body + * is registered on the client via `defineCommand`. */ + await redis.setExecutionResultScript( + stateKey, + resultKey, + updatedSerialized, + JSON.stringify(result), + String(EXECUTION_STATE_TTL), ); - } - /** EXISTS-then-SET both keys atomically so a concurrent - * `cleanupExecution` racing between our `getExecutionState` above and - * the writes below cannot leave an orphan blob behind. The Lua body - * is registered on the client via `defineCommand`. */ - await redis.setExecutionResultScript( - stateKey, - resultKey, - updatedSerialized, - JSON.stringify(result), - String(EXECUTION_STATE_TTL), - ); -} - -export async function setExecutionError(execution_id: string, error: Error): Promise { - const stateKey = `exec_state:${execution_id}`; - const existing = await getExecutionState(execution_id); - if (!existing) return; - const updated: ExecutionState = { ...existing, jobCompleted: true, jobError: error.message }; - const serialized = JSON.stringify(updated); - if (Buffer.byteLength(serialized, 'utf8') > MAX_EXECUTION_STATE_BYTES) { - throw new ExecutionStateTooLargeError( - execution_id, - Buffer.byteLength(serialized, 'utf8'), - MAX_EXECUTION_STATE_BYTES, +} + +export async function setExecutionError( + execution_id: string, + error: Error, +): Promise { + const stateKey = `exec_state:${execution_id}`; + const existing = await getExecutionState(execution_id); + if (!existing) return; + const updated: ExecutionState = { + ...existing, + jobCompleted: true, + jobError: error.message, + }; + const serialized = JSON.stringify(updated); + if (Buffer.byteLength(serialized, 'utf8') > MAX_EXECUTION_STATE_BYTES) { + throw new ExecutionStateTooLargeError( + execution_id, + Buffer.byteLength(serialized, 'utf8'), + MAX_EXECUTION_STATE_BYTES, + ); + } + /** Same race window as `setExecutionResult`: a `cleanupExecution` + * winning between our `getExecutionState` and the write would + * otherwise resurrect torn-down state. The conditional SET makes + * the write a no-op once the execution has been cleaned up. */ + await redis.setExecutionErrorScript( + stateKey, + serialized, + String(EXECUTION_STATE_TTL), ); - } - /** Same race window as `setExecutionResult`: a `cleanupExecution` - * winning between our `getExecutionState` and the write would - * otherwise resurrect torn-down state. The conditional SET makes - * the write a no-op once the execution has been cleaned up. */ - await redis.setExecutionErrorScript(stateKey, serialized, String(EXECUTION_STATE_TTL)); } // --------------------------------------------------------------------------- @@ -494,7 +577,7 @@ export async function setExecutionError(execution_id: string, error: Error): Pro // --------------------------------------------------------------------------- export function historyKey(execution_id: string): string { - return `tool_history:${execution_id}`; + return `tool_history:${execution_id}`; } /** Canonical byte-comparable form of the "semantic" fields of a history @@ -504,15 +587,15 @@ export function historyKey(execution_id: string): string { * which is sufficient for checking equality of payloads produced from * the same client-supplied object. */ function canonicalEntryBody( - result: unknown, - isError: boolean | undefined, - errorMessage: string | undefined, + result: unknown, + isError: boolean | undefined, + errorMessage: string | undefined, ): string { - return JSON.stringify({ - result, - is_error: isError ?? false, - error_message: errorMessage, - }); + return JSON.stringify({ + result, + is_error: isError ?? false, + error_message: errorMessage, + }); } /** Compute the delta between an incoming result batch and the persisted @@ -535,82 +618,93 @@ function canonicalEntryBody( * 409-style mismatch error and no Redis mutation is attempted. */ export async function computeToolHistoryDelta( - execution_id: string, - results: Array<{ - call_id: string; - result: unknown; - is_error?: boolean; - error_message?: string; - tool_name?: string; - input_hash?: string; - call_site?: string; - }>, + execution_id: string, + results: Array<{ + call_id: string; + result: unknown; + is_error?: boolean; + error_message?: string; + tool_name?: string; + input_hash?: string; + call_site?: string; + }>, ): Promise { - const serializedByCallId = new Map(); - if (results.length === 0) { - return { serializedByCallId, newCallIds: [], bytesDelta: 0 }; - } - const key = historyKey(execution_id); - const callIds = results.map(r => r.call_id); - const existingRaw = callIds.length > 0 ? await redis.hmget(key, ...callIds) : []; - const oldByCallId = new Map(); - for (let i = 0; i < callIds.length; i++) { - const raw = existingRaw[i]; - if (typeof raw !== 'string') continue; - try { - const parsed = JSON.parse(raw) as HistoryEntry; - oldByCallId.set(callIds[i], { - entry: parsed, - bytes: Buffer.byteLength(raw, 'utf8'), - }); - } catch { - logger.warn('Malformed existing tool_history entry; treating as absent', { - execution_id, - call_id: callIds[i], - }); - } - } - const now = Date.now(); - const newCallIds: string[] = []; - let bytesDelta = 0; - for (const r of results) { - const entry: HistoryEntry = { - result: r.result, - is_error: r.is_error ?? false, - error_message: r.error_message, - received_at: now, - tool_name: r.tool_name, - input_hash: r.input_hash, - call_site: r.call_site, - }; - const serialized = JSON.stringify(entry); - const newBytes = Buffer.byteLength(serialized, 'utf8'); - if (newBytes > MAX_TOOL_RESULT_BYTES) { - return { - error: `tool_results[${r.call_id}] serialized entry is ${newBytes} bytes, exceeds per-entry cap ${MAX_TOOL_RESULT_BYTES}`, - }; - } - const existing = oldByCallId.get(r.call_id); - if (existing) { - const incomingBody = canonicalEntryBody(r.result, r.is_error, r.error_message); - const storedBody = canonicalEntryBody( - existing.entry.result, - existing.entry.is_error, - existing.entry.error_message, - ); - if (incomingBody !== storedBody) { - return { - status: 409, - error: `tool_results[${r.call_id}] has already been recorded with a different payload; replay history is immutable`, + const serializedByCallId = new Map(); + if (results.length === 0) { + return { serializedByCallId, newCallIds: [], bytesDelta: 0 }; + } + const key = historyKey(execution_id); + const callIds = results.map(r => r.call_id); + const existingRaw = + callIds.length > 0 ? await redis.hmget(key, ...callIds) : []; + const oldByCallId = new Map< + string, + { entry: HistoryEntry; bytes: number } + >(); + for (let i = 0; i < callIds.length; i++) { + const raw = existingRaw[i]; + if (typeof raw !== 'string') continue; + try { + const parsed = JSON.parse(raw) as HistoryEntry; + oldByCallId.set(callIds[i], { + entry: parsed, + bytes: Buffer.byteLength(raw, 'utf8'), + }); + } catch { + logger.warn( + 'Malformed existing tool_history entry; treating as absent', + { + execution_id, + call_id: callIds[i], + }, + ); + } + } + const now = Date.now(); + const newCallIds: string[] = []; + let bytesDelta = 0; + for (const r of results) { + const entry: HistoryEntry = { + result: r.result, + is_error: r.is_error ?? false, + error_message: r.error_message, + received_at: now, + tool_name: r.tool_name, + input_hash: r.input_hash, + call_site: r.call_site, }; - } - continue; + const serialized = JSON.stringify(entry); + const newBytes = Buffer.byteLength(serialized, 'utf8'); + if (newBytes > MAX_TOOL_RESULT_BYTES) { + return { + error: `tool_results[${r.call_id}] serialized entry is ${newBytes} bytes, exceeds per-entry cap ${MAX_TOOL_RESULT_BYTES}`, + }; + } + const existing = oldByCallId.get(r.call_id); + if (existing) { + const incomingBody = canonicalEntryBody( + r.result, + r.is_error, + r.error_message, + ); + const storedBody = canonicalEntryBody( + existing.entry.result, + existing.entry.is_error, + existing.entry.error_message, + ); + if (incomingBody !== storedBody) { + return { + status: 409, + error: `tool_results[${r.call_id}] has already been recorded with a different payload; replay history is immutable`, + }; + } + continue; + } + serializedByCallId.set(r.call_id, serialized); + newCallIds.push(r.call_id); + bytesDelta += newBytes; } - serializedByCallId.set(r.call_id, serialized); - newCallIds.push(r.call_id); - bytesDelta += newBytes; - } - return { serializedByCallId, newCallIds, bytesDelta }; + return { serializedByCallId, newCallIds, bytesDelta }; } /** Commit a previously-computed history delta AND the updated execution @@ -621,58 +715,60 @@ export async function computeToolHistoryDelta( * blob must already be validated against `MAX_EXECUTION_STATE_BYTES` * before this is called; failure is logged and propagated. */ export async function commitToolHistoryAndState( - state: ExecutionState, - delta: ToolHistoryDelta, + state: ExecutionState, + delta: ToolHistoryDelta, ): Promise { - const serialized = JSON.stringify(state); - const serializedBytes = Buffer.byteLength(serialized, 'utf8'); - if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { - /** Use the typed error so the continuation handler can distinguish - * a sizing problem (-> 413) from a generic Redis transaction failure - * (-> 503). A plain `Error` here would bubble through to the top-level - * router catch as an opaque 500. */ - throw new ExecutionStateTooLargeError( - state.execution_id, - serializedBytes, - MAX_EXECUTION_STATE_BYTES, - ); - } - const hKey = historyKey(state.execution_id); - const sKey = `exec_state:${state.execution_id}`; - const tx = redis.multi(); - if (delta.serializedByCallId.size > 0) { - const kvs: string[] = []; - for (const [callId, s] of delta.serializedByCallId) { - kvs.push(callId, s); - } - tx.hset(hKey, ...kvs); - } - /** Refresh the history-hash TTL on every commit, including idempotent - * retries where `delta.serializedByCallId` is empty. Without this, a - * retry near TTL expiry renews `exec_state` (via the `SET EX` below) - * but leaves `tool_history` on its original expiry; a slow next - * sandbox run could then span the history's original TTL and cause - * subsequent continuations to load an empty history and re-emit - * already-resolved tool calls. `EXPIRE` on a missing hash is a safe - * no-op. */ - tx.expire(hKey, EXECUTION_STATE_TTL); - tx.set(sKey, serialized, 'EX', EXECUTION_STATE_TTL); - const results = await tx.exec(); - if (!results) { - throw new Error(`Redis transaction aborted for execution ${state.execution_id}`); - } - for (const [err] of results) { - if (err) { - throw err; - } - } - /** Observe the size and entry-count of the persisted history *after* - * commit succeeds so failed commits don't pollute the histogram with - * sizes that never made it to Redis. `historyBytes` and `callCount` - * have already been advanced on `state` to reflect the post-commit - * truth, so they're the right values to record. */ - ptcReplayHistorySize.observe(state.historyBytes ?? 0); - ptcReplayHistoryEntries.observe(state.callCount ?? 0); + const serialized = JSON.stringify(state); + const serializedBytes = Buffer.byteLength(serialized, 'utf8'); + if (serializedBytes > MAX_EXECUTION_STATE_BYTES) { + /** Use the typed error so the continuation handler can distinguish + * a sizing problem (-> 413) from a generic Redis transaction failure + * (-> 503). A plain `Error` here would bubble through to the top-level + * router catch as an opaque 500. */ + throw new ExecutionStateTooLargeError( + state.execution_id, + serializedBytes, + MAX_EXECUTION_STATE_BYTES, + ); + } + const hKey = historyKey(state.execution_id); + const sKey = `exec_state:${state.execution_id}`; + const tx = redis.multi(); + if (delta.serializedByCallId.size > 0) { + const kvs: string[] = []; + for (const [callId, s] of delta.serializedByCallId) { + kvs.push(callId, s); + } + tx.hset(hKey, ...kvs); + } + /** Refresh the history-hash TTL on every commit, including idempotent + * retries where `delta.serializedByCallId` is empty. Without this, a + * retry near TTL expiry renews `exec_state` (via the `SET EX` below) + * but leaves `tool_history` on its original expiry; a slow next + * sandbox run could then span the history's original TTL and cause + * subsequent continuations to load an empty history and re-emit + * already-resolved tool calls. `EXPIRE` on a missing hash is a safe + * no-op. */ + tx.expire(hKey, EXECUTION_STATE_TTL); + tx.set(sKey, serialized, 'EX', EXECUTION_STATE_TTL); + const results = await tx.exec(); + if (!results) { + throw new Error( + `Redis transaction aborted for execution ${state.execution_id}`, + ); + } + for (const [err] of results) { + if (err) { + throw err; + } + } + /** Observe the size and entry-count of the persisted history *after* + * commit succeeds so failed commits don't pollute the histogram with + * sizes that never made it to Redis. `historyBytes` and `callCount` + * have already been advanced on `state` to reflect the post-commit + * truth, so they're the right values to record. */ + ptcReplayHistorySize.observe(state.historyBytes ?? 0); + ptcReplayHistoryEntries.observe(state.callCount ?? 0); } /** Extend the Redis TTL on both the execution-state blob and the tool-history @@ -680,27 +776,32 @@ export async function commitToolHistoryAndState( * still valid but prior tool results have already expired, which would force * the sandbox to re-emit earlier calls on replay. */ export async function refreshExecutionTtl(execution_id: string): Promise { - await Promise.all([ - redis.expire(`exec_state:${execution_id}`, EXECUTION_STATE_TTL), - redis.expire(historyKey(execution_id), EXECUTION_STATE_TTL), - ]); + await Promise.all([ + redis.expire(`exec_state:${execution_id}`, EXECUTION_STATE_TTL), + redis.expire(historyKey(execution_id), EXECUTION_STATE_TTL), + ]); } -export async function loadToolHistory(execution_id: string): Promise> { - const raw = await redis.hgetall(historyKey(execution_id)); - const out: Record = {}; - for (const [k, v] of Object.entries(raw)) { - try { - out[k] = JSON.parse(v) as HistoryEntry; - } catch { - logger.warn('Dropping malformed tool_history entry', { execution_id, call_id: k }); +export async function loadToolHistory( + execution_id: string, +): Promise> { + const raw = await redis.hgetall(historyKey(execution_id)); + const out: Record = {}; + for (const [k, v] of Object.entries(raw)) { + try { + out[k] = JSON.parse(v) as HistoryEntry; + } catch { + logger.warn('Dropping malformed tool_history entry', { + execution_id, + call_id: k, + }); + } } - } - return out; + return out; } export async function deleteToolHistory(execution_id: string): Promise { - await redis.del(historyKey(execution_id)); + await redis.del(historyKey(execution_id)); } // --------------------------------------------------------------------------- @@ -714,117 +815,148 @@ export async function deleteToolHistory(execution_id: string): Promise { const CLEANUP_BATCH_SIZE = 200; export async function cleanupStaleExecutions(): Promise { - try { - const keys = await scanKeys('exec_state:*'); - if (keys.length === 0) return 0; - const now = Date.now(); - let cleaned = 0; - - /** Batch the per-key state reads via `MGET` so a sweep over many - * exec_state keys completes in O(keys / batch) Redis round-trips - * rather than O(keys) sequential `GET`s. The decision-and-delete - * step still iterates per-execution because the deletes (history, - * blocking result, tool_call_server session, exec_state key) are - * heterogeneous and each touches different Redis keyspaces. */ - for (let offset = 0; offset < keys.length; offset += CLEANUP_BATCH_SIZE) { - const batchKeys = keys.slice(offset, offset + CLEANUP_BATCH_SIZE); - const values = await redis.mget(...batchKeys); - for (let i = 0; i < batchKeys.length; i++) { - const data = values[i]; - if (data == null) continue; - - let state: ExecutionState; - try { - state = JSON.parse(data) as ExecutionState; - } catch { - /** Malformed JSON in `exec_state:` (corrupt write or partial - * legacy migration) — drop it AND its sibling `tool_history` / - * `exec_result` keys instead of breaking the whole sweep. - * Earlier we only dropped `exec_state` here, leaving siblings - * orphaned until their independent Redis TTL fired (up to - * `EXECUTION_STATE_TTL` of wasted memory per malformed key). - * The execution id is encoded in the key suffix; we can't - * recover the typed state, but we don't need to in order to - * scope the targeted deletes. - * - * Counted toward `cleaned` and logged at warn level so a burst - * of malformed keys (corrupt writes, mid-deploy migrations) is - * visible to operators via both `ptc_replay_stale_cleanups_total` - * and structured logs, instead of silently disappearing. */ - const orphanId = batchKeys[i].slice('exec_state:'.length); - logger.warn('Reaping malformed exec_state and sibling keys', { - execution_id: orphanId, - }); - await Promise.all([ - redis.del(batchKeys[i]), - deleteToolHistory(orphanId), - deleteBlockingResult(orphanId), - ]).catch(err => { - logger.warn('Sibling delete failed during malformed-key cleanup', { - execution_id: orphanId, - error: err instanceof Error ? err.message : String(err), - }); - }); - cleaned++; - continue; - } - const lastActivity = state.lastActivity ?? state.startTime; - const idle = now - lastActivity; - - if (idle > EXECUTION_STATE_TTL * 1000 && state.jobCompleted !== true) { - logger.warn('Cleaning up stale execution', { - execution_id: state.execution_id, - idleSeconds: idle / 1000, - totalAgeSeconds: (now - state.startTime) / 1000, - mode: state.mode, - }); - - if (state.mode === 'blocking') { - await axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${state.execution_id}`, { - headers: internalServiceHeaders(), - }) - .catch(() => {}); - } - await Promise.all([ - deleteToolHistory(state.execution_id), - deleteBlockingResult(state.execution_id), - redis.del(batchKeys[i]), - ]); - cleaned++; + try { + const keys = await scanKeys('exec_state:*'); + if (keys.length === 0) return 0; + const now = Date.now(); + let cleaned = 0; + + /** Batch the per-key state reads via `MGET` so a sweep over many + * exec_state keys completes in O(keys / batch) Redis round-trips + * rather than O(keys) sequential `GET`s. The decision-and-delete + * step still iterates per-execution because the deletes (history, + * blocking result, tool_call_server session, exec_state key) are + * heterogeneous and each touches different Redis keyspaces. */ + for ( + let offset = 0; + offset < keys.length; + offset += CLEANUP_BATCH_SIZE + ) { + const batchKeys = keys.slice(offset, offset + CLEANUP_BATCH_SIZE); + const values = await redis.mget(...batchKeys); + for (let i = 0; i < batchKeys.length; i++) { + const data = values[i]; + if (data == null) continue; + + let state: ExecutionState; + try { + state = JSON.parse(data) as ExecutionState; + } catch { + /** Malformed JSON in `exec_state:` (corrupt write or partial + * legacy migration) — drop it AND its sibling `tool_history` / + * `exec_result` keys instead of breaking the whole sweep. + * Earlier we only dropped `exec_state` here, leaving siblings + * orphaned until their independent Redis TTL fired (up to + * `EXECUTION_STATE_TTL` of wasted memory per malformed key). + * The execution id is encoded in the key suffix; we can't + * recover the typed state, but we don't need to in order to + * scope the targeted deletes. + * + * Counted toward `cleaned` and logged at warn level so a burst + * of malformed keys (corrupt writes, mid-deploy migrations) is + * visible to operators via both `ptc_replay_stale_cleanups_total` + * and structured logs, instead of silently disappearing. */ + const orphanId = batchKeys[i].slice('exec_state:'.length); + logger.warn( + 'Reaping malformed exec_state and sibling keys', + { + execution_id: orphanId, + }, + ); + await Promise.all([ + redis.del(batchKeys[i]), + deleteToolHistory(orphanId), + deleteBlockingResult(orphanId), + ]).catch(err => { + logger.warn( + 'Sibling delete failed during malformed-key cleanup', + { + execution_id: orphanId, + error: + err instanceof Error + ? err.message + : String(err), + }, + ); + }); + cleaned++; + continue; + } + const lastActivity = state.lastActivity ?? state.startTime; + const idle = now - lastActivity; + + if ( + idle > EXECUTION_STATE_TTL * 1000 && + state.jobCompleted !== true + ) { + logger.warn('Cleaning up stale execution', { + execution_id: state.execution_id, + idleSeconds: idle / 1000, + totalAgeSeconds: (now - state.startTime) / 1000, + mode: state.mode, + }); + + if (state.mode === 'blocking') { + await axios + .delete( + `${env.TOOL_CALL_SERVER_URL}/sessions/${state.execution_id}`, + { + headers: internalServiceHeaders(), + }, + ) + .catch(() => {}); + } + await Promise.all([ + deleteToolHistory(state.execution_id), + deleteBlockingResult(state.execution_id), + redis.del(batchKeys[i]), + ]); + cleaned++; + } + } } - } - } - if (cleaned > 0) { - logger.info(`Cleaned up ${cleaned} stale executions`); - ptcReplayStaleCleanups.inc(cleaned); + if (cleaned > 0) { + logger.info(`Cleaned up ${cleaned} stale executions`); + ptcReplayStaleCleanups.inc(cleaned); + } + return cleaned; + } catch (error) { + logger.error('Error cleaning up stale executions:', error); + return 0; } - return cleaned; - } catch (error) { - logger.error('Error cleaning up stale executions:', error); - return 0; - } } -export async function cleanupExecution(execution_id: string, mode: 'blocking' | 'replay'): Promise { - try { - const ops: Array> = [ - deleteExecutionState(execution_id), - deleteToolHistory(execution_id), - deleteBlockingResult(execution_id), - ]; - if (mode === 'blocking') { - ops.push( - axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}`, { - headers: internalServiceHeaders(), - }).catch(() => {}), - ); +export async function cleanupExecution( + execution_id: string, + mode: 'blocking' | 'replay', +): Promise { + try { + const ops: Array> = [ + deleteExecutionState(execution_id), + deleteToolHistory(execution_id), + deleteBlockingResult(execution_id), + ]; + if (mode === 'blocking') { + ops.push( + axios + .delete( + `${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}`, + { + headers: internalServiceHeaders(), + }, + ) + .catch(() => {}), + ); + } + await Promise.all(ops); + logger.info('Execution cleanup completed', { execution_id, mode }); + } catch (error) { + logger.error('Error during execution cleanup:', { + execution_id, + error, + }); } - await Promise.all(ops); - logger.info('Execution cleanup completed', { execution_id, mode }); - } catch (error) { - logger.error('Error during execution cleanup:', { execution_id, error }); - } } // --------------------------------------------------------------------------- @@ -836,57 +968,73 @@ export async function cleanupExecution(execution_id: string, mode: 'blocking' | * (null, wrong types, bad call_ids) from being persisted into Redis history. */ export function validateToolResult( - r: unknown, + r: unknown, ): - | { call_id: string; result: unknown; is_error?: boolean; error_message?: string } - | { error: string } { - if (r == null || typeof r !== 'object' || Array.isArray(r)) { - return { error: 'tool_results entries must be objects' }; - } - const obj = r as Record; - const callId = obj.call_id; - if (typeof callId !== 'string' || !CALL_ID_RE.test(callId)) { - return { error: `tool_results[].call_id must match /^call_\\d{3,6}$/, got: ${JSON.stringify(callId)}` }; - } - if (!('result' in obj)) { - return { error: `tool_results[${callId}].result is required` }; - } - if (obj.is_error !== undefined && typeof obj.is_error !== 'boolean') { - return { error: `tool_results[${callId}].is_error must be a boolean if present` }; - } - if (obj.error_message !== undefined && typeof obj.error_message !== 'string') { - return { error: `tool_results[${callId}].error_message must be a string if present` }; - } - let resultSerialized: string | undefined; - try { - resultSerialized = JSON.stringify(obj.result); - } catch (err) { - return { error: `tool_results[${callId}].result is not JSON-serializable` }; - } - /** `JSON.stringify` only throws on circular references; for `undefined`, - * functions, and `Symbol` it returns the JS value `undefined` rather - * than the string `'undefined'`. The earlier `?? 'null'` fallback - * silently coerced these to a 4-byte placeholder and accepted them - * as valid history entries — but the entry would then deserialize - * to `null` on replay, drifting from what the client originally - * sent. Reject explicitly so callers get a clear error. */ - if (resultSerialized === undefined) { - return { - error: `tool_results[${callId}].result is not JSON-serializable (got undefined / function / symbol)`, - }; - } - const resultBytes = Buffer.byteLength(resultSerialized, 'utf8'); - if (resultBytes > MAX_TOOL_RESULT_BYTES) { + | { + call_id: string; + result: unknown; + is_error?: boolean; + error_message?: string; + } + | { error: string } { + if (r == null || typeof r !== 'object' || Array.isArray(r)) { + return { error: 'tool_results entries must be objects' }; + } + const obj = r as Record; + const callId = obj.call_id; + if (typeof callId !== 'string' || !CALL_ID_RE.test(callId)) { + return { + error: `tool_results[].call_id must match /^call_\\d{3,6}$/, got: ${JSON.stringify(callId)}`, + }; + } + if (!('result' in obj)) { + return { error: `tool_results[${callId}].result is required` }; + } + if (obj.is_error !== undefined && typeof obj.is_error !== 'boolean') { + return { + error: `tool_results[${callId}].is_error must be a boolean if present`, + }; + } + if ( + obj.error_message !== undefined && + typeof obj.error_message !== 'string' + ) { + return { + error: `tool_results[${callId}].error_message must be a string if present`, + }; + } + let resultSerialized: string | undefined; + try { + resultSerialized = JSON.stringify(obj.result); + } catch (err) { + return { + error: `tool_results[${callId}].result is not JSON-serializable`, + }; + } + /** `JSON.stringify` only throws on circular references; for `undefined`, + * functions, and `Symbol` it returns the JS value `undefined` rather + * than the string `'undefined'`. The earlier `?? 'null'` fallback + * silently coerced these to a 4-byte placeholder and accepted them + * as valid history entries — but the entry would then deserialize + * to `null` on replay, drifting from what the client originally + * sent. Reject explicitly so callers get a clear error. */ + if (resultSerialized === undefined) { + return { + error: `tool_results[${callId}].result is not JSON-serializable (got undefined / function / symbol)`, + }; + } + const resultBytes = Buffer.byteLength(resultSerialized, 'utf8'); + if (resultBytes > MAX_TOOL_RESULT_BYTES) { + return { + error: `tool_results[${callId}].result is ${resultBytes} bytes, exceeds per-result cap ${MAX_TOOL_RESULT_BYTES}`, + }; + } return { - error: `tool_results[${callId}].result is ${resultBytes} bytes, exceeds per-result cap ${MAX_TOOL_RESULT_BYTES}`, + call_id: callId, + result: obj.result, + is_error: obj.is_error as boolean | undefined, + error_message: obj.error_message as string | undefined, }; - } - return { - call_id: callId, - result: obj.result, - is_error: obj.is_error as boolean | undefined, - error_message: obj.error_message as string | undefined, - }; } // --------------------------------------------------------------------------- @@ -894,13 +1042,13 @@ export function validateToolResult( // --------------------------------------------------------------------------- export interface ValidatedContinuationResult { - call_id: string; - result: unknown; - is_error?: boolean; - error_message?: string; - tool_name?: string; - input_hash?: string; - call_site?: string; + call_id: string; + result: unknown; + is_error?: boolean; + error_message?: string; + tool_name?: string; + input_hash?: string; + call_site?: string; } /** Validate the raw `tool_results` array a continuation request submitted: @@ -908,39 +1056,41 @@ export interface ValidatedContinuationResult { * uniqueness within the batch. Pure; no Redis. Extracted from the router * so the branch coverage is unit-testable. */ -export function validateContinuationBatch(tool_results: unknown[]): - | { ok: true; results: ValidatedContinuationResult[] } - | { ok: false; status: number; error: string } { - if (tool_results.length > MAX_REPLAY_CALLS) { - return { - ok: false, - status: 400, - error: `tool_results has ${tool_results.length} entries; the per-batch cap is ${MAX_REPLAY_CALLS}`, - }; - } - const results: ValidatedContinuationResult[] = []; - const seen = new Set(); - for (const raw of tool_results) { - const v = validateToolResult(raw); - if ('error' in v) { - return { ok: false, status: 400, error: v.error }; - } - if (seen.has(v.call_id)) { - return { - ok: false, - status: 400, - error: `Duplicate call_id in tool_results: ${v.call_id}`, - }; - } - seen.add(v.call_id); - results.push({ - call_id: v.call_id, - result: v.result, - is_error: v.is_error, - error_message: v.error_message, - }); - } - return { ok: true, results }; +export function validateContinuationBatch( + tool_results: unknown[], +): + | { ok: true; results: ValidatedContinuationResult[] } + | { ok: false; status: number; error: string } { + if (tool_results.length > MAX_REPLAY_CALLS) { + return { + ok: false, + status: 400, + error: `tool_results has ${tool_results.length} entries; the per-batch cap is ${MAX_REPLAY_CALLS}`, + }; + } + const results: ValidatedContinuationResult[] = []; + const seen = new Set(); + for (const raw of tool_results) { + const v = validateToolResult(raw); + if ('error' in v) { + return { ok: false, status: 400, error: v.error }; + } + if (seen.has(v.call_id)) { + return { + ok: false, + status: 400, + error: `Duplicate call_id in tool_results: ${v.call_id}`, + }; + } + seen.add(v.call_id); + results.push({ + call_id: v.call_id, + result: v.result, + is_error: v.is_error, + error_message: v.error_message, + }); + } + return { ok: true, results }; } /** Apply the post-state-load pre-checks to a continuation request: @@ -952,57 +1102,66 @@ export function validateContinuationBatch(tool_results: unknown[]): * Redis client. */ export function checkContinuationPreconditions(params: { - state: ExecutionState; - results: ValidatedContinuationResult[]; - userId: string; - apiKeyId?: string; - tenantId?: string; - authContextHash?: string; - delta: ToolHistoryDelta; + state: ExecutionState; + results: ValidatedContinuationResult[]; + userId: string; + apiKeyId?: string; + tenantId?: string; + authContextHash?: string; + delta: ToolHistoryDelta; }): - | { ok: true } - | { ok: false; status: number; error: string; cleanupOnReject?: boolean } { - const { state, results, userId, apiKeyId, tenantId, authContextHash, delta } = params; - if (state.mode !== 'replay') { - return { - ok: false, - status: 400, - error: 'Execution was started in blocking mode; continuation mode mismatch', - }; - } - if ( - state.userId !== userId || - (state.apiKeyId != null && state.apiKeyId !== apiKeyId) || - (state.tenantId != null && state.tenantId !== tenantId) || - (state.authContextHash != null && state.authContextHash !== authContextHash) - ) { - return { ok: false, status: 403, error: 'Forbidden' }; - } - const issued = new Set(state.emittedCallIds ?? []); - for (const r of results) { - if (!issued.has(r.call_id)) { - return { - ok: false, - status: 400, - error: `tool_results[${r.call_id}] was not issued by the sandbox for this execution`, - }; - } - } - if ((state.callCount ?? 0) + delta.newCallIds.length > MAX_REPLAY_CALLS) { - return { - ok: false, - status: 400, - error: `Execution exceeded maximum tool calls (${MAX_REPLAY_CALLS})`, - cleanupOnReject: true, - }; - } - const projectedHistoryBytes = (state.historyBytes ?? 0) + delta.bytesDelta; - if (projectedHistoryBytes > MAX_TOOL_HISTORY_TOTAL_BYTES) { - return { - ok: false, - status: 400, - error: `Aggregate tool_history for this execution would be ${projectedHistoryBytes} bytes, exceeds cap ${MAX_TOOL_HISTORY_TOTAL_BYTES}`, - }; - } - return { ok: true }; + | { ok: true } + | { ok: false; status: number; error: string; cleanupOnReject?: boolean } { + const { + state, + results, + userId, + apiKeyId, + tenantId, + authContextHash, + delta, + } = params; + if (state.mode !== 'replay') { + return { + ok: false, + status: 400, + error: 'Execution was started in blocking mode; continuation mode mismatch', + }; + } + if ( + state.userId !== userId || + (state.apiKeyId != null && state.apiKeyId !== apiKeyId) || + (state.tenantId != null && state.tenantId !== tenantId) || + (state.authContextHash != null && + state.authContextHash !== authContextHash) + ) { + return { ok: false, status: 403, error: 'Forbidden' }; + } + const issued = new Set(state.emittedCallIds ?? []); + for (const r of results) { + if (!issued.has(r.call_id)) { + return { + ok: false, + status: 400, + error: `tool_results[${r.call_id}] was not issued by the sandbox for this execution`, + }; + } + } + if ((state.callCount ?? 0) + delta.newCallIds.length > MAX_REPLAY_CALLS) { + return { + ok: false, + status: 400, + error: `Execution exceeded maximum tool calls (${MAX_REPLAY_CALLS})`, + cleanupOnReject: true, + }; + } + const projectedHistoryBytes = (state.historyBytes ?? 0) + delta.bytesDelta; + if (projectedHistoryBytes > MAX_TOOL_HISTORY_TOTAL_BYTES) { + return { + ok: false, + status: 400, + error: `Aggregate tool_history for this execution would be ${projectedHistoryBytes} bytes, exceeds cap ${MAX_TOOL_HISTORY_TOTAL_BYTES}`, + }; + } + return { ok: true }; } diff --git a/service/src/tool-call-server.ts b/service/src/tool-call-server.ts index 866de62..a0d2ddf 100644 --- a/service/src/tool-call-server.ts +++ b/service/src/tool-call-server.ts @@ -1,20 +1,26 @@ -import IORedis from 'ioredis'; import { nanoid } from 'nanoid'; -import type * as tls from 'tls'; import { - httpLatencyElapsedSeconds, - httpLatencyStartMs, - metricsResponse, - recordHttpRequest, - toolCalls, - toolCallTimeouts, - toolCallActiveSessions, + httpLatencyElapsedSeconds, + httpLatencyStartMs, + metricsResponse, + recordHttpRequest, + toolCalls, + toolCallTimeouts, + toolCallActiveSessions, } from './metrics'; -import { internalServiceAuthEnabled, isAuthorizedInternalServiceRequest } from './internal-service-auth'; +import { + internalServiceAuthEnabled, + isAuthorizedInternalServiceRequest, +} from './internal-service-auth'; import { isRegisteredToolName } from './tool-scope'; -import { normalizeTracePath, shutdownTelemetry, withSpan, withTraceContext } from './telemetry'; +import { + normalizeTracePath, + shutdownTelemetry, + withSpan, + withTraceContext, +} from './telemetry'; import logger from './toolCallServerLogger'; -import { redisKeepAliveOptions } from './redis-options'; +import { createRedisConnection } from './redis-connection'; const INSTANCE_ID = process.env.INSTANCE_ID ?? nanoid(); const PORT = Number(process.env.TOOL_CALL_SERVER_PORT) || 3033; @@ -22,604 +28,703 @@ const REQUEST_TIMEOUT = Number(process.env.TOOL_CALL_REQUEST_TIMEOUT) || 300000; const SESSION_EXPIRY = Number(process.env.TOOL_CALL_SESSION_EXPIRY) || 600; // 10 minutes in seconds if (!internalServiceAuthEnabled()) { - logger.warn('CODEAPI_INTERNAL_SERVICE_TOKEN is not set; tool-call session management routes are unauthenticated'); + logger.warn( + 'CODEAPI_INTERNAL_SERVICE_TOKEN is not set; tool-call session management routes are unauthenticated', + ); } // Redis connection -const useAltDnsLookup = process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true'; - -const redis = new IORedis({ - host: process.env.REDIS_HOST ?? 'redis', - port: Number(process.env.REDIS_PORT) || 6379, - password: process.env.REDIS_PASSWORD, - enableReadyCheck: false, - tls: process.env.REDIS_TLS === 'true' ? { - rejectUnauthorized: false - } as tls.ConnectionOptions : undefined, - connectTimeout: 10000, - ...redisKeepAliveOptions(), - maxRetriesPerRequest: 3, - retryStrategy(times: number): number { - const delay = Math.min(times * 500, 2000); - return delay; - }, - reconnectOnError(err: Error): boolean { - const targetError = 'READONLY'; - if (err.message.includes(targetError)) { - return true; - } - return false; - }, - // Alternative DNS lookup for AWS ElastiCache TLS connections - ...(useAltDnsLookup - ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } - : {}) +const redis = createRedisConnection({ + enableReadyCheck: false, + maxRetriesPerRequest: 3, + retryStrategy(times: number): number { + return Math.min(times * 500, 2000); + }, + reconnectOnError(err: Error): boolean { + return err.message.includes('READONLY'); + }, }); -redis.on('error', (err) => { - logger.error('Redis Client Error', { error: err }); +redis.on('error', err => { + logger.error('Redis Client Error', { error: err }); }); redis.on('connect', () => { - logger.info('Redis Client Connected', { - host: process.env.REDIS_HOST, - port: process.env.REDIS_PORT - }); + logger.info('Redis Client Connected', { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + }); }); redis.on('ready', () => { - logger.info('Redis Client Ready'); + logger.info('Redis Client Ready'); }); // Types interface ToolCallSession { - execution_id: string; - session_id: string; - callback_token: string; - status: 'running' | 'waiting' | 'completed' | 'error'; - tools: Array<{ name: string; description?: string; parameters?: Record }>; - created_at: number; - updated_at: number; - timeout: number; + execution_id: string; + session_id: string; + callback_token: string; + status: 'running' | 'waiting' | 'completed' | 'error'; + tools: Array<{ + name: string; + description?: string; + parameters?: Record; + }>; + created_at: number; + updated_at: number; + timeout: number; } interface ToolCallRequest { - call_id: string; - tool_name: string; - tool_input: Record; - timestamp: number; + call_id: string; + tool_name: string; + tool_input: Record; + timestamp: number; } interface ToolCallResult { - call_id: string; - result: unknown; - is_error: boolean; - error_message?: string; - received_at: number; + call_id: string; + result: unknown; + is_error: boolean; + error_message?: string; + received_at: number; } // Helper functions function jsonResponse(data: unknown, status = 200): Response { - return new Response(JSON.stringify(data), { - status, - headers: { 'Content-Type': 'application/json' } - }); + return new Response(JSON.stringify(data), { + status, + headers: { 'Content-Type': 'application/json' }, + }); } function errorResponse(error: string, status = 400): Response { - return jsonResponse({ error }, status); + return jsonResponse({ error }, status); } -async function getSession(executionId: string): Promise { - const data = await redis.get(`tool_call:session:${executionId}`); - return data != null && data !== '' ? JSON.parse(data) : null; +async function getSession( + executionId: string, +): Promise { + const data = await redis.get(`tool_call:session:${executionId}`); + return data != null && data !== '' ? JSON.parse(data) : null; } async function setSession(session: ToolCallSession): Promise { - await redis.set( - `tool_call:session:${session.execution_id}`, - JSON.stringify(session), - 'EX', - SESSION_EXPIRY - ); + await redis.set( + `tool_call:session:${session.execution_id}`, + JSON.stringify(session), + 'EX', + SESSION_EXPIRY, + ); } // Route handlers async function handleCreateSession(req: Request): Promise { - try { - const body = await req.json() as { - execution_id: string; - session_id: string; - timeout?: number; - tools?: ToolCallSession['tools']; - }; - - const { execution_id, session_id, timeout = REQUEST_TIMEOUT, tools = [] } = body; - - if (!execution_id || !session_id) { - return errorResponse('Missing required fields: execution_id, session_id', 400); - } - - const callback_token = nanoid(32); - - const session: ToolCallSession = { - execution_id, - session_id, - callback_token, - status: 'running', - tools, - created_at: Date.now(), - updated_at: Date.now(), - timeout - }; - - await setSession(session); - toolCallActiveSessions.inc(); - - logger.info(`[${INSTANCE_ID}] Session created: ${execution_id}`); - - return jsonResponse({ - success: true, - execution_id, - callback_url: `http://localhost:${PORT}`, - callback_token - }); - } catch (error) { - logger.error('Error creating session:', { error }); - return errorResponse('Internal server error', 500); - } -} + try { + const body = (await req.json()) as { + execution_id: string; + session_id: string; + timeout?: number; + tools?: ToolCallSession['tools']; + }; + + const { + execution_id, + session_id, + timeout = REQUEST_TIMEOUT, + tools = [], + } = body; + + if (!execution_id || !session_id) { + return errorResponse( + 'Missing required fields: execution_id, session_id', + 400, + ); + } -async function handleGetPending(executionId: string): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); + const callback_token = nanoid(32); + + const session: ToolCallSession = { + execution_id, + session_id, + callback_token, + status: 'running', + tools, + created_at: Date.now(), + updated_at: Date.now(), + timeout, + }; + + await setSession(session); + toolCallActiveSessions.inc(); + + logger.info(`[${INSTANCE_ID}] Session created: ${execution_id}`); + + return jsonResponse({ + success: true, + execution_id, + callback_url: `http://localhost:${PORT}`, + callback_token, + }); + } catch (error) { + logger.error('Error creating session:', { error }); + return errorResponse('Internal server error', 500); } - - // Get pending calls from Redis list - const pendingData = await redis.lrange(`tool_call:pending:${executionId}`, 0, -1); - const pendingCalls: ToolCallRequest[] = pendingData.map(d => JSON.parse(d)); - - return jsonResponse({ - status: session.status, - pending_calls: pendingCalls, - partial_stdout: '', - partial_stderr: '' - }); - } catch (error) { - logger.error('Error getting pending calls:', { error }); - return errorResponse('Internal server error', 500); - } } -async function handleSubmitResults(executionId: string, req: Request): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } - - const body = await req.json() as { results: ToolCallResult[] }; - const { results } = body; +async function handleGetPending(executionId: string): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - if (!Array.isArray(results)) { - return errorResponse('Invalid results format', 400); + // Get pending calls from Redis list + const pendingData = await redis.lrange( + `tool_call:pending:${executionId}`, + 0, + -1, + ); + const pendingCalls: ToolCallRequest[] = pendingData.map(d => + JSON.parse(d), + ); + + return jsonResponse({ + status: session.status, + pending_calls: pendingCalls, + partial_stdout: '', + partial_stderr: '', + }); + } catch (error) { + logger.error('Error getting pending calls:', { error }); + return errorResponse('Internal server error', 500); } +} - let processed = 0; - - // Get all pending calls once - const pendingData = await redis.lrange(`tool_call:pending:${executionId}`, 0, -1); - - for (const result of results) { - const resultData: ToolCallResult = { - call_id: result.call_id, - result: result.result, - is_error: result.is_error || false, - error_message: result.error_message, - received_at: Date.now() - }; +async function handleSubmitResults( + executionId: string, + req: Request, +): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - // Store result - await redis.set( - `tool_call:result:${executionId}:${result.call_id}`, - JSON.stringify(resultData), - 'EX', - SESSION_EXPIRY - ); + const body = (await req.json()) as { results: ToolCallResult[] }; + const { results } = body; - // Publish for waiting subscribers - await redis.publish( - `tool_call:result:${executionId}:${result.call_id}`, - JSON.stringify(resultData) - ); + if (!Array.isArray(results)) { + return errorResponse('Invalid results format', 400); + } - // Remove from pending list - find the exact string that was stored - const pendingToRemove = pendingData.find(p => { - try { - return JSON.parse(p).call_id === result.call_id; - } catch { - return false; + let processed = 0; + + // Get all pending calls once + const pendingData = await redis.lrange( + `tool_call:pending:${executionId}`, + 0, + -1, + ); + + for (const result of results) { + const resultData: ToolCallResult = { + call_id: result.call_id, + result: result.result, + is_error: result.is_error || false, + error_message: result.error_message, + received_at: Date.now(), + }; + + // Store result + await redis.set( + `tool_call:result:${executionId}:${result.call_id}`, + JSON.stringify(resultData), + 'EX', + SESSION_EXPIRY, + ); + + // Publish for waiting subscribers + await redis.publish( + `tool_call:result:${executionId}:${result.call_id}`, + JSON.stringify(resultData), + ); + + // Remove from pending list - find the exact string that was stored + const pendingToRemove = pendingData.find(p => { + try { + return JSON.parse(p).call_id === result.call_id; + } catch { + return false; + } + }); + + if (pendingToRemove != null && pendingToRemove !== '') { + await redis.lrem( + `tool_call:pending:${executionId}`, + 1, + pendingToRemove, + ); + logger.info( + `[${INSTANCE_ID}] Removed pending call ${result.call_id} from ${executionId}`, + ); + } + + processed++; } - }); - if (pendingToRemove != null && pendingToRemove !== '') { - await redis.lrem(`tool_call:pending:${executionId}`, 1, pendingToRemove); - logger.info(`[${INSTANCE_ID}] Removed pending call ${result.call_id} from ${executionId}`); - } + // Update session status + const remainingPending = await redis.llen( + `tool_call:pending:${executionId}`, + ); + if (remainingPending === 0) { + session.status = 'running'; + session.updated_at = Date.now(); + await setSession(session); + } - processed++; - } + logger.info( + `[${INSTANCE_ID}] Results submitted for ${executionId}: ${processed} processed`, + ); - // Update session status - const remainingPending = await redis.llen(`tool_call:pending:${executionId}`); - if (remainingPending === 0) { - session.status = 'running'; - session.updated_at = Date.now(); - await setSession(session); + return jsonResponse({ success: true, processed }); + } catch (error) { + logger.error('Error submitting results:', { error }); + return errorResponse('Internal server error', 500); } - - logger.info(`[${INSTANCE_ID}] Results submitted for ${executionId}: ${processed} processed`); - - return jsonResponse({ success: true, processed }); - } catch (error) { - logger.error('Error submitting results:', { error }); - return errorResponse('Internal server error', 500); - } } async function handleGetStatus(executionId: string): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - return jsonResponse({ - status: session.status, - execution_id: session.execution_id, - session_id: session.session_id, - created_at: session.created_at, - updated_at: session.updated_at - }); - } catch (error) { - logger.error('Error getting status:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ + status: session.status, + execution_id: session.execution_id, + session_id: session.session_id, + created_at: session.created_at, + updated_at: session.updated_at, + }); + } catch (error) { + logger.error('Error getting status:', { error }); + return errorResponse('Internal server error', 500); + } } -async function handleComplete(executionId: string, req: Request): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } +async function handleComplete( + executionId: string, + req: Request, +): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - const body = await req.json() as { - stdout?: string; - stderr?: string; - exit_code?: number; - }; - - const wasActive = session.status !== 'completed' && session.status !== 'error'; - session.status = 'completed'; - session.updated_at = Date.now(); - await setSession(session); - if (wasActive) { - toolCallActiveSessions.dec(); - } + const body = (await req.json()) as { + stdout?: string; + stderr?: string; + exit_code?: number; + }; + + const wasActive = + session.status !== 'completed' && session.status !== 'error'; + session.status = 'completed'; + session.updated_at = Date.now(); + await setSession(session); + if (wasActive) { + toolCallActiveSessions.dec(); + } - // Store completion data - await redis.set( - `tool_call:complete:${executionId}`, - JSON.stringify({ - ...body, - completed_at: Date.now() - }), - 'EX', - SESSION_EXPIRY - ); + // Store completion data + await redis.set( + `tool_call:complete:${executionId}`, + JSON.stringify({ + ...body, + completed_at: Date.now(), + }), + 'EX', + SESSION_EXPIRY, + ); - logger.info(`[${INSTANCE_ID}] Execution completed: ${executionId}`); + logger.info(`[${INSTANCE_ID}] Execution completed: ${executionId}`); - return jsonResponse({ success: true }); - } catch (error) { - logger.error('Error marking complete:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ success: true }); + } catch (error) { + logger.error('Error marking complete:', { error }); + return errorResponse('Internal server error', 500); + } } -async function handleError(executionId: string, req: Request): Promise { - try { - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } +async function handleError( + executionId: string, + req: Request, +): Promise { + try { + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - const body = await req.json() as { - error: string; - error_type?: string; - stderr?: string; - }; - - const wasActive = session.status !== 'completed' && session.status !== 'error'; - session.status = 'error'; - session.updated_at = Date.now(); - await setSession(session); - if (wasActive) { - toolCallActiveSessions.dec(); - } + const body = (await req.json()) as { + error: string; + error_type?: string; + stderr?: string; + }; + + const wasActive = + session.status !== 'completed' && session.status !== 'error'; + session.status = 'error'; + session.updated_at = Date.now(); + await setSession(session); + if (wasActive) { + toolCallActiveSessions.dec(); + } - // Store error data - await redis.set( - `tool_call:error:${executionId}`, - JSON.stringify({ - ...body, - error_at: Date.now() - }), - 'EX', - SESSION_EXPIRY - ); + // Store error data + await redis.set( + `tool_call:error:${executionId}`, + JSON.stringify({ + ...body, + error_at: Date.now(), + }), + 'EX', + SESSION_EXPIRY, + ); - logger.info(`[${INSTANCE_ID}] Execution error: ${executionId}`); + logger.info(`[${INSTANCE_ID}] Execution error: ${executionId}`); - return jsonResponse({ success: true }); - } catch (error) { - logger.error('Error marking error:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ success: true }); + } catch (error) { + logger.error('Error marking error:', { error }); + return errorResponse('Internal server error', 500); + } } async function handleDeleteSession(executionId: string): Promise { - try { - // Check session status before deleting — only decrement if still active - const session = await getSession(executionId); - const wasActive = session != null && session.status !== 'completed' && session.status !== 'error'; - - const keys = await redis.keys(`tool_call:*:${executionId}*`); - if (keys.length > 0) { - await redis.del(...keys); - } + try { + // Check session status before deleting — only decrement if still active + const session = await getSession(executionId); + const wasActive = + session != null && + session.status !== 'completed' && + session.status !== 'error'; + + const keys = await redis.keys(`tool_call:*:${executionId}*`); + if (keys.length > 0) { + await redis.del(...keys); + } - if (wasActive) { - toolCallActiveSessions.dec(); - } - logger.info(`[${INSTANCE_ID}] Session deleted: ${executionId}, cleaned ${keys.length} keys`); + if (wasActive) { + toolCallActiveSessions.dec(); + } + logger.info( + `[${INSTANCE_ID}] Session deleted: ${executionId}, cleaned ${keys.length} keys`, + ); - return jsonResponse({ success: true, cleaned_keys: keys.length }); - } catch (error) { - logger.error('Error deleting session:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ success: true, cleaned_keys: keys.length }); + } catch (error) { + logger.error('Error deleting session:', { error }); + return errorResponse('Internal server error', 500); + } } async function handleToolCall(req: Request): Promise { - try { - const executionId = req.headers.get('X-Execution-ID') ?? ''; - const callbackToken = req.headers.get('X-Callback-Token') ?? ''; - const callId = req.headers.get('X-Tool-Call-ID') ?? ''; + try { + const executionId = req.headers.get('X-Execution-ID') ?? ''; + const callbackToken = req.headers.get('X-Callback-Token') ?? ''; + const callId = req.headers.get('X-Tool-Call-ID') ?? ''; - if (!executionId || !callbackToken || !callId) { - return errorResponse('Missing required headers', 400); - } + if (!executionId || !callbackToken || !callId) { + return errorResponse('Missing required headers', 400); + } - const session = await getSession(executionId); - if (!session) { - return errorResponse('Session not found', 404); - } + const session = await getSession(executionId); + if (!session) { + return errorResponse('Session not found', 404); + } - if (session.callback_token !== callbackToken) { - return errorResponse('Invalid callback token', 401); - } + if (session.callback_token !== callbackToken) { + return errorResponse('Invalid callback token', 401); + } - const body = await req.json() as { - tool_name: string; - input: Record; - }; + const body = (await req.json()) as { + tool_name: string; + input: Record; + }; - const { tool_name, input } = body; - if (typeof tool_name !== 'string' || tool_name === '') { - return errorResponse('Invalid tool name', 400); - } - if (!isRegisteredToolName(tool_name, session.tools)) { - logger.warn(`[${INSTANCE_ID}] Rejected unregistered tool call: ${executionId}/${callId} - ${tool_name}`); - return errorResponse('Tool is not registered for this execution', 403); - } + const { tool_name, input } = body; + if (typeof tool_name !== 'string' || tool_name === '') { + return errorResponse('Invalid tool name', 400); + } + if (!isRegisteredToolName(tool_name, session.tools)) { + logger.warn( + `[${INSTANCE_ID}] Rejected unregistered tool call: ${executionId}/${callId} - ${tool_name}`, + ); + return errorResponse( + 'Tool is not registered for this execution', + 403, + ); + } - // Create pending call record - const toolCall: ToolCallRequest = { - call_id: callId, - tool_name, - tool_input: input, - timestamp: Date.now() - }; - - // Add to pending list - await redis.rpush(`tool_call:pending:${executionId}`, JSON.stringify(toolCall)); - - // Update session status - session.status = 'waiting'; - session.updated_at = Date.now(); - await setSession(session); - - toolCalls.inc(); - logger.info(`[${INSTANCE_ID}] Tool call received: ${executionId}/${callId} - ${tool_name}`); - - // Wait for result (blocking) - const result = await waitForResult(executionId, callId, session.timeout); - - if (result === null) { - toolCallTimeouts.inc(); - return jsonResponse({ - success: false, - error: 'timeout', - message: 'Tool call timed out waiting for result' - }, 408); - } + // Create pending call record + const toolCall: ToolCallRequest = { + call_id: callId, + tool_name, + tool_input: input, + timestamp: Date.now(), + }; + + // Add to pending list + await redis.rpush( + `tool_call:pending:${executionId}`, + JSON.stringify(toolCall), + ); + + // Update session status + session.status = 'waiting'; + session.updated_at = Date.now(); + await setSession(session); + + toolCalls.inc(); + logger.info( + `[${INSTANCE_ID}] Tool call received: ${executionId}/${callId} - ${tool_name}`, + ); + + // Wait for result (blocking) + const result = await waitForResult( + executionId, + callId, + session.timeout, + ); + + if (result === null) { + toolCallTimeouts.inc(); + return jsonResponse( + { + success: false, + error: 'timeout', + message: 'Tool call timed out waiting for result', + }, + 408, + ); + } - return jsonResponse({ - success: true, - result: result.result, - is_error: result.is_error, - error_message: result.error_message - }); - } catch (error) { - logger.error('Error handling tool call:', { error }); - return errorResponse('Internal server error', 500); - } + return jsonResponse({ + success: true, + result: result.result, + is_error: result.is_error, + error_message: result.error_message, + }); + } catch (error) { + logger.error('Error handling tool call:', { error }); + return errorResponse('Internal server error', 500); + } } async function waitForResult( - executionId: string, - callId: string, - timeout: number + executionId: string, + callId: string, + timeout: number, ): Promise { - const resultKey = `tool_call:result:${executionId}:${callId}`; - const startTime = Date.now(); - const pollInterval = 100; // 100ms - - while (Date.now() - startTime < timeout) { - const result = await redis.get(resultKey); - if (result != null && result !== '') { - await redis.del(resultKey); // Consume the result - return JSON.parse(result); + const resultKey = `tool_call:result:${executionId}:${callId}`; + const startTime = Date.now(); + const pollInterval = 100; // 100ms + + while (Date.now() - startTime < timeout) { + const result = await redis.get(resultKey); + if (result != null && result !== '') { + await redis.del(resultKey); // Consume the result + return JSON.parse(result); + } + await new Promise(resolve => setTimeout(resolve, pollInterval)); } - await new Promise(resolve => setTimeout(resolve, pollInterval)); - } - return null; + return null; } // Health check async function handleHealth(): Promise { - try { - await redis.ping(); - return jsonResponse({ - status: 'healthy', - redis: true, - uptime: process.uptime(), - instance_id: INSTANCE_ID - }); - } catch (error) { - return jsonResponse({ - status: 'unhealthy', - redis: false, - error: (error as Error).message - }, 503); - } + try { + await redis.ping(); + return jsonResponse({ + status: 'healthy', + redis: true, + uptime: process.uptime(), + instance_id: INSTANCE_ID, + }); + } catch (error) { + return jsonResponse( + { + status: 'unhealthy', + redis: false, + error: (error as Error).message, + }, + 503, + ); + } } // Router -async function routeToolCallRequest(req: Request): Promise<{ response: Response; route: string }> { - const url = new URL(req.url); - const path = url.pathname; - const method = req.method; - - // Health check - if (path === '/health' && method === 'GET') { - return { response: await handleHealth(), route: '/health' }; - } - - // Prometheus metrics - if (path === '/metrics' && method === 'GET') { - const { body, contentType } = await metricsResponse(); - return { response: new Response(body, { status: 200, headers: { 'Content-Type': contentType } }), route: '/metrics' }; - } - - if ((path === '/sessions' || path.startsWith('/sessions/')) && !isAuthorizedInternalServiceRequest(req.headers)) { - return { response: errorResponse('Unauthorized', 401), route: '/sessions/*' }; - } - - // POST /sessions - Create session - if (path === '/sessions' && method === 'POST') { - return { response: await handleCreateSession(req), route: '/sessions' }; - } - - // POST /tool-call - Tool call (blocking) - if (path === '/tool-call' && method === 'POST') { - return { response: await handleToolCall(req), route: '/tool-call' }; - } - - // Session-specific routes - const sessionMatch = path.match(/^\/sessions\/([^/]+)(\/(.+))?$/); - if (sessionMatch) { - const executionId = sessionMatch[1]; - const subPath = sessionMatch[3]; - - // GET /sessions/:id - if (!subPath && method === 'GET') { - return { response: await handleGetStatus(executionId), route: '/sessions/:executionId' }; +async function routeToolCallRequest( + req: Request, +): Promise<{ response: Response; route: string }> { + const url = new URL(req.url); + const path = url.pathname; + const method = req.method; + + // Health check + if (path === '/health' && method === 'GET') { + return { response: await handleHealth(), route: '/health' }; } - // DELETE /sessions/:id - if (!subPath && method === 'DELETE') { - return { response: await handleDeleteSession(executionId), route: '/sessions/:executionId' }; + // Prometheus metrics + if (path === '/metrics' && method === 'GET') { + const { body, contentType } = await metricsResponse(); + return { + response: new Response(body, { + status: 200, + headers: { 'Content-Type': contentType }, + }), + route: '/metrics', + }; } - if (subPath === 'pending' && method === 'GET') { - return { response: await handleGetPending(executionId), route: '/sessions/:executionId/pending' }; + if ( + (path === '/sessions' || path.startsWith('/sessions/')) && + !isAuthorizedInternalServiceRequest(req.headers) + ) { + return { + response: errorResponse('Unauthorized', 401), + route: '/sessions/*', + }; } - // POST /sessions/:id/results - if (subPath === 'results' && method === 'POST') { - return { response: await handleSubmitResults(executionId, req), route: '/sessions/:executionId/results' }; + // POST /sessions - Create session + if (path === '/sessions' && method === 'POST') { + return { response: await handleCreateSession(req), route: '/sessions' }; } - // GET /sessions/:id/status - if (subPath === 'status' && method === 'GET') { - return { response: await handleGetStatus(executionId), route: '/sessions/:executionId/status' }; + // POST /tool-call - Tool call (blocking) + if (path === '/tool-call' && method === 'POST') { + return { response: await handleToolCall(req), route: '/tool-call' }; } - // POST /sessions/:id/complete - if (subPath === 'complete' && method === 'POST') { - return { response: await handleComplete(executionId, req), route: '/sessions/:executionId/complete' }; - } + // Session-specific routes + const sessionMatch = path.match(/^\/sessions\/([^/]+)(\/(.+))?$/); + if (sessionMatch) { + const executionId = sessionMatch[1]; + const subPath = sessionMatch[3]; + + // GET /sessions/:id + if (!subPath && method === 'GET') { + return { + response: await handleGetStatus(executionId), + route: '/sessions/:executionId', + }; + } + + // DELETE /sessions/:id + if (!subPath && method === 'DELETE') { + return { + response: await handleDeleteSession(executionId), + route: '/sessions/:executionId', + }; + } + + if (subPath === 'pending' && method === 'GET') { + return { + response: await handleGetPending(executionId), + route: '/sessions/:executionId/pending', + }; + } + + // POST /sessions/:id/results + if (subPath === 'results' && method === 'POST') { + return { + response: await handleSubmitResults(executionId, req), + route: '/sessions/:executionId/results', + }; + } + + // GET /sessions/:id/status + if (subPath === 'status' && method === 'GET') { + return { + response: await handleGetStatus(executionId), + route: '/sessions/:executionId/status', + }; + } - // POST /sessions/:id/error - if (subPath === 'error' && method === 'POST') { - return { response: await handleError(executionId, req), route: '/sessions/:executionId/error' }; + // POST /sessions/:id/complete + if (subPath === 'complete' && method === 'POST') { + return { + response: await handleComplete(executionId, req), + route: '/sessions/:executionId/complete', + }; + } + + // POST /sessions/:id/error + if (subPath === 'error' && method === 'POST') { + return { + response: await handleError(executionId, req), + route: '/sessions/:executionId/error', + }; + } } - } - return { response: errorResponse('Not found', 404), route: 'unmatched' }; + return { response: errorResponse('Not found', 404), route: 'unmatched' }; } async function handleRequest(req: Request): Promise { - return withTraceContext(Object.fromEntries(req.headers.entries()), () => withSpan('codeapi.tool_call_server.request', { - 'http.request.method': req.method, - 'url.path': normalizeTracePath(new URL(req.url).pathname), - }, async (span) => { - const start = httpLatencyStartMs(); - let route = 'unmatched'; - try { - const { response, route: matchedRoute } = await routeToolCallRequest(req); - route = matchedRoute; - span.setAttribute('http.response.status_code', response.status); - span.setAttribute('http.route', route); - recordHttpRequest({ - method: req.method, - route, - statusCode: response.status, - durationSeconds: httpLatencyElapsedSeconds(start), - }); - return response; - } catch (error) { - recordHttpRequest({ - method: req.method, - route, - statusCode: 500, - durationSeconds: httpLatencyElapsedSeconds(start), - }); - throw error; - } - }, 'SERVER')); + return withTraceContext(Object.fromEntries(req.headers.entries()), () => + withSpan( + 'codeapi.tool_call_server.request', + { + 'http.request.method': req.method, + 'url.path': normalizeTracePath(new URL(req.url).pathname), + }, + async span => { + const start = httpLatencyStartMs(); + let route = 'unmatched'; + try { + const { response, route: matchedRoute } = + await routeToolCallRequest(req); + route = matchedRoute; + span.setAttribute( + 'http.response.status_code', + response.status, + ); + span.setAttribute('http.route', route); + recordHttpRequest({ + method: req.method, + route, + statusCode: response.status, + durationSeconds: httpLatencyElapsedSeconds(start), + }); + return response; + } catch (error) { + recordHttpRequest({ + method: req.method, + route, + statusCode: 500, + durationSeconds: httpLatencyElapsedSeconds(start), + }); + throw error; + } + }, + 'SERVER', + ), + ); } // Start server const server = Bun.serve({ - port: PORT, - fetch: handleRequest, + port: PORT, + fetch: handleRequest, }); logger.info(`[${INSTANCE_ID}] Tool Call Server running on port ${PORT}`); @@ -628,38 +733,42 @@ logger.info(`[${INSTANCE_ID}] Tool Call Server running on port ${PORT}`); let shuttingDown = false; async function shutdown(): Promise { - if (shuttingDown) return; - shuttingDown = true; - logger.info(`[${INSTANCE_ID}] Shutting down...`); - try { - server.stop(); - await redis.quit(); + if (shuttingDown) return; + shuttingDown = true; + logger.info(`[${INSTANCE_ID}] Shutting down...`); try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); - } - process.exit(0); - } catch (error) { - logger.error(`[${INSTANCE_ID}] Shutdown failed`, { error }); - try { - await shutdownTelemetry(); - } catch (telemetryError) { - logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { error: telemetryError }); + server.stop(); + await redis.quit(); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(0); + } catch (error) { + logger.error(`[${INSTANCE_ID}] Shutdown failed`, { error }); + try { + await shutdownTelemetry(); + } catch (telemetryError) { + logger.warn(`[${INSTANCE_ID}] OpenTelemetry shutdown failed`, { + error: telemetryError, + }); + } + process.exit(1); } - process.exit(1); - } } process.on('SIGTERM', () => void shutdown()); process.on('SIGINT', () => void shutdown()); -process.on('uncaughtException', (error) => { - logger.error('Uncaught Exception', { error }); +process.on('uncaughtException', error => { + logger.error('Uncaught Exception', { error }); }); process.on('unhandledRejection', (reason, promise) => { - logger.error('Unhandled Rejection', { reason, promise }); + logger.error('Unhandled Rejection', { reason, promise }); }); export { server }; diff --git a/service/src/workers.ts b/service/src/workers.ts index 177e8fd..b01c1d1 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -1,247 +1,299 @@ import axios from 'axios'; import { Worker } from 'bullmq'; import type * as t from './types'; -import { filterSystemLogs, applySystemReplacements, getAxiosErrorDetails, sandboxErrorMessageFromAxios } from './utils'; -import { jobProcessingDuration, jobsCompleted, jobsFailed, activeJobs, workerRunning } from './metrics'; +import { + filterSystemLogs, + applySystemReplacements, + getAxiosErrorDetails, + sandboxErrorMessageFromAxios, +} from './utils'; +import { + jobProcessingDuration, + jobsCompleted, + jobsFailed, + activeJobs, + workerRunning, +} from './metrics'; import { Jobs, Queues } from './enum'; import { connection } from './queue'; import { env } from './config'; import { summarizeSandboxResponse, summarizeText } from './execution-log'; -import { createGatewayEgressGrant, restoreGatewaySandboxResult, revokeGatewayEgressGrant } from './egress-gateway-client'; +import { + createGatewayEgressGrant, + restoreGatewaySandboxResult, + revokeGatewayEgressGrant, +} from './egress-gateway-client'; import { refreshEgressGrantClaims } from './sandbox-egress'; import { buildSandboxExecuteRequest } from './sandbox-dispatch'; import { isSyntheticPrincipalSource } from './auth/synthetic'; import { injectTraceHeaders, withSpan, withTraceContext } from './telemetry'; import logger from './logger'; +import { bullmqPrefix } from './redis-connection'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; type SandboxLogResponse = t.ExecuteResponse & { - session_id: string; - files?: t.FileRefs; - run?: t.ExecuteResponse['run']; + session_id: string; + files?: t.FileRefs; + run?: t.ExecuteResponse['run']; }; function isAbortError(error: unknown): boolean { - return axios.isAxiosError(error) && (error.name === 'AbortError' || error.code === 'ERR_CANCELED'); + return ( + axios.isAxiosError(error) && + (error.name === 'AbortError' || error.code === 'ERR_CANCELED') + ); } async function processJob(job: t.ExecuteJob): Promise { - return withTraceContext(job.data._otel, () => withSpan('codeapi.job.process', { - 'messaging.system': 'bullmq', - 'messaging.operation.name': 'process', - 'messaging.message.id': typeof job.id === 'string' ? job.id : String(job.id ?? ''), - 'codeapi.language': job.data.payload?.language ?? 'unknown', - }, () => processJobInner(job), 'CONSUMER')); + return withTraceContext(job.data._otel, () => + withSpan( + 'codeapi.job.process', + { + 'messaging.system': 'bullmq', + 'messaging.operation.name': 'process', + 'messaging.message.id': + typeof job.id === 'string' ? job.id : String(job.id ?? ''), + 'codeapi.language': job.data.payload?.language ?? 'unknown', + }, + () => processJobInner(job), + 'CONSUMER', + ), + ); } async function processJobInner(job: t.ExecuteJob): Promise { - const { code, payload, isPyPlot } = job.data; - const isSyntheticJob = job.data.isSynthetic === true || isSyntheticPrincipalSource(job.data.principalSource); - const language = payload?.language ?? 'unknown'; - const endTimer = jobProcessingDuration.startTimer({ language }); - activeJobs.inc({ language }); - - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), env.JOB_TIMEOUT); - let egressGrantId: string | undefined; - let egressGrantTokenForRestore: string | undefined; - let revokeReason = 'completed'; - - try { - let sandboxPayload = payload; - let executionManifestClaims = job.data.executionManifestClaims; - let egressGrantToken = job.data.egressGrantToken; - - if (job.data.egressGrantClaims) { - const nowSeconds = Math.floor(Date.now() / 1000); - const prepared = await createGatewayEgressGrant({ - payload, - claims: refreshEgressGrantClaims(job.data.egressGrantClaims, nowSeconds), - isSynthetic: isSyntheticJob, - signal: controller.signal, - }); - egressGrantId = prepared.grant_id; - sandboxPayload = prepared.payload; - egressGrantToken = prepared.egressGrantToken; - egressGrantTokenForRestore = prepared.egressGrantToken; - executionManifestClaims = (env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET) - ? prepared.executionManifestClaims - : undefined; - } + const { code, payload, isPyPlot } = job.data; + const isSyntheticJob = + job.data.isSynthetic === true || + isSyntheticPrincipalSource(job.data.principalSource); + const language = payload?.language ?? 'unknown'; + const endTimer = jobProcessingDuration.startTimer({ language }); + activeJobs.inc({ language }); + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), env.JOB_TIMEOUT); + let egressGrantId: string | undefined; + let egressGrantTokenForRestore: string | undefined; + let revokeReason = 'completed'; + + try { + let sandboxPayload = payload; + let executionManifestClaims = job.data.executionManifestClaims; + let egressGrantToken = job.data.egressGrantToken; - const sandboxRequest = buildSandboxExecuteRequest({ - payload: sandboxPayload, - egressGrantToken, - executionManifestClaims, - executionManifestPrivateKey: env.EXECUTION_MANIFEST_PRIVATE_KEY, - executionManifestSecret: env.EXECUTION_MANIFEST_SECRET, - executionManifestTtlSeconds: env.EXECUTION_MANIFEST_TTL_SECONDS, - }); - egressGrantTokenForRestore = egressGrantToken; - - const response = await withSpan('codeapi.sandbox.execute', { - 'http.request.method': 'POST', - 'url.path': `/${Jobs.execute}`, - 'codeapi.language': language, - }, () => axios.post( - `${env.SANDBOX_ENDPOINT}/${Jobs.execute}`, - sandboxRequest.body, - { - headers: injectTraceHeaders(sandboxRequest.headers), - signal: controller.signal, + if (job.data.egressGrantClaims) { + const nowSeconds = Math.floor(Date.now() / 1000); + const prepared = await createGatewayEgressGrant({ + payload, + claims: refreshEgressGrantClaims( + job.data.egressGrantClaims, + nowSeconds, + ), + isSynthetic: isSyntheticJob, + signal: controller.signal, + }); + egressGrantId = prepared.grant_id; + sandboxPayload = prepared.payload; + egressGrantToken = prepared.egressGrantToken; + egressGrantTokenForRestore = prepared.egressGrantToken; + executionManifestClaims = + env.EXECUTION_MANIFEST_PRIVATE_KEY || + env.EXECUTION_MANIFEST_SECRET + ? prepared.executionManifestClaims + : undefined; } - ), 'CLIENT'); - if (response.status !== 200) { - throw new Error('Error from sandbox'); - } + const sandboxRequest = buildSandboxExecuteRequest({ + payload: sandboxPayload, + egressGrantToken, + executionManifestClaims, + executionManifestPrivateKey: env.EXECUTION_MANIFEST_PRIVATE_KEY, + executionManifestSecret: env.EXECUTION_MANIFEST_SECRET, + executionManifestTtlSeconds: env.EXECUTION_MANIFEST_TTL_SECONDS, + }); + egressGrantTokenForRestore = egressGrantToken; - const responseData = egressGrantTokenForRestore - ? await restoreGatewaySandboxResult({ - grantId: egressGrantId, - egressGrantToken: egressGrantTokenForRestore, - result: response.data, - isSynthetic: isSyntheticJob, - signal: controller.signal, - }) - : response.data; - - if (!isSyntheticJob) { - logger.info('Sandbox response', summarizeSandboxResponse(responseData)); - } + const response = await withSpan( + 'codeapi.sandbox.execute', + { + 'http.request.method': 'POST', + 'url.path': `/${Jobs.execute}`, + 'codeapi.language': language, + }, + () => + axios.post( + `${env.SANDBOX_ENDPOINT}/${Jobs.execute}`, + sandboxRequest.body, + { + headers: injectTraceHeaders(sandboxRequest.headers), + signal: controller.signal, + }, + ), + 'CLIENT', + ); - const { files } = responseData; - const run = responseData.run; - const stdout = applySystemReplacements(run?.stdout ?? ''); - const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); - - const result: t.ExecuteResult = { - session_id: responseData.session_id, - /* `files` is optional on the sandbox response (e.g. dry-run - * execute with no outputs); the public `ExecuteResult.files` is - * required and downstream callers always iterate it. Default to - * `[]` so the strictened response type from Phase B doesn't - * surface a regression that wasn't there before. */ - files: files ?? [], - stdout, - stderr, - }; - - if (run) { - result.code = run.code ?? null; - result.signal = run.signal != null ? String(run.signal) : null; - result.message = run.message ?? null; - result.status = run.status ?? null; - result.wall_time = (run as Record).wall_time as number | null ?? null; - } + if (response.status !== 200) { + throw new Error('Error from sandbox'); + } - if (result.message || result.signal) { - logger.warn('Sandbox execution error metadata', { - session_id: responseData.session_id, - code: result.code, - signal: result.signal, - message: summarizeText(result.message), - status: result.status, - wall_time: result.wall_time, - }); - } + const responseData = egressGrantTokenForRestore + ? await restoreGatewaySandboxResult({ + grantId: egressGrantId, + egressGrantToken: egressGrantTokenForRestore, + result: response.data, + isSynthetic: isSyntheticJob, + signal: controller.signal, + }) + : response.data; - return result; - } catch (error) { - revokeReason = isAbortError(error) ? 'timeout' : 'failed'; - const errorDetails = getAxiosErrorDetails(error); - logger.error('Error processing job', errorDetails); - - if (isAbortError(error)) { - throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); - } else if (axios.isAxiosError(error)) { - /** Preserve error message from sandbox */ - const sandboxError = sandboxErrorMessageFromAxios(error); - throw new Error(`Error from sandbox: ${sandboxError}`); - } - throw error; - } finally { - if (egressGrantId || egressGrantTokenForRestore) { - await revokeGatewayEgressGrant({ - grantId: egressGrantId, - egressGrantToken: egressGrantId ? undefined : egressGrantTokenForRestore, - isSynthetic: isSyntheticJob, - reason: revokeReason, - timeoutMs: env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, - }).catch(error => { - logger.error('Failed to revoke egress grant', { grantId: egressGrantId, error: getAxiosErrorDetails(error) }); - }); + if (!isSyntheticJob) { + logger.info( + 'Sandbox response', + summarizeSandboxResponse(responseData), + ); + } + + const { files } = responseData; + const run = responseData.run; + const stdout = applySystemReplacements(run?.stdout ?? ''); + const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); + + const result: t.ExecuteResult = { + session_id: responseData.session_id, + /* `files` is optional on the sandbox response (e.g. dry-run + * execute with no outputs); the public `ExecuteResult.files` is + * required and downstream callers always iterate it. Default to + * `[]` so the strictened response type from Phase B doesn't + * surface a regression that wasn't there before. */ + files: files ?? [], + stdout, + stderr, + }; + + if (run) { + result.code = run.code ?? null; + result.signal = run.signal != null ? String(run.signal) : null; + result.message = run.message ?? null; + result.status = run.status ?? null; + result.wall_time = + ((run as Record).wall_time as number | null) ?? + null; + } + + if (result.message || result.signal) { + logger.warn('Sandbox execution error metadata', { + session_id: responseData.session_id, + code: result.code, + signal: result.signal, + message: summarizeText(result.message), + status: result.status, + wall_time: result.wall_time, + }); + } + + return result; + } catch (error) { + revokeReason = isAbortError(error) ? 'timeout' : 'failed'; + const errorDetails = getAxiosErrorDetails(error); + logger.error('Error processing job', errorDetails); + + if (isAbortError(error)) { + throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); + } else if (axios.isAxiosError(error)) { + /** Preserve error message from sandbox */ + const sandboxError = sandboxErrorMessageFromAxios(error); + throw new Error(`Error from sandbox: ${sandboxError}`); + } + throw error; + } finally { + if (egressGrantId || egressGrantTokenForRestore) { + await revokeGatewayEgressGrant({ + grantId: egressGrantId, + egressGrantToken: egressGrantId + ? undefined + : egressGrantTokenForRestore, + isSynthetic: isSyntheticJob, + reason: revokeReason, + timeoutMs: env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, + }).catch(error => { + logger.error('Failed to revoke egress grant', { + grantId: egressGrantId, + error: getAxiosErrorDetails(error), + }); + }); + } + clearTimeout(timer); + endTimer(); + activeJobs.dec({ language }); } - clearTimeout(timer); - endTimer(); - activeJobs.dec({ language }); - } } // Global workers - no INSTANCE_ID prefix // This enables horizontal scaling where any worker can process any job from the shared queue // Each worker respects its own concurrency limit based on its co-located sandbox capacity export const pyWorker = new Worker(Queues.python, processJob, { - connection, - concurrency: env.PYTHON_CONCURRENCY, - limiter: { - max: env.PYTHON_CONCURRENCY, - duration: env.JOB_WINDOW, - }, + connection, + prefix: bullmqPrefix(), + concurrency: env.PYTHON_CONCURRENCY, + limiter: { + max: env.PYTHON_CONCURRENCY, + duration: env.JOB_WINDOW, + }, }); export const otherWorker = new Worker(Queues.other, processJob, { - connection, - concurrency: env.OTHER_CONCURRENCY, - limiter: { - max: env.OTHER_CONCURRENCY, - duration: env.JOB_WINDOW, - }, + connection, + prefix: bullmqPrefix(), + concurrency: env.OTHER_CONCURRENCY, + limiter: { + max: env.OTHER_CONCURRENCY, + duration: env.JOB_WINDOW, + }, }); workerRunning.set({ worker_type: 'python' }, 1); workerRunning.set({ worker_type: 'other' }, 1); pyWorker.on('completed', job => { - if (job.data.isSynthetic !== true) { - logger.info(`[${WORKER_ID}] Python job completed ${job.id}`); - } - jobsCompleted.inc({ language: 'python' }); + if (job.data.isSynthetic !== true) { + logger.info(`[${WORKER_ID}] Python job completed ${job.id}`); + } + jobsCompleted.inc({ language: 'python' }); }); otherWorker.on('completed', job => { - if (job.data.isSynthetic !== true) { - logger.info(`[${WORKER_ID}] Other job completed ${job.id}`); - } - jobsCompleted.inc({ language: 'other' }); + if (job.data.isSynthetic !== true) { + logger.info(`[${WORKER_ID}] Other job completed ${job.id}`); + } + jobsCompleted.inc({ language: 'other' }); }); pyWorker.on('failed', (job, err) => { - logger.error(`[${WORKER_ID}] Python job ${job?.id} failed`, err); - jobsFailed.inc({ language: 'python' }); + logger.error(`[${WORKER_ID}] Python job ${job?.id} failed`, err); + jobsFailed.inc({ language: 'python' }); }); otherWorker.on('failed', (job, err) => { - logger.error(`[${WORKER_ID}] Other job ${job?.id} failed`, err); - jobsFailed.inc({ language: 'other' }); + logger.error(`[${WORKER_ID}] Other job ${job?.id} failed`, err); + jobsFailed.inc({ language: 'other' }); }); -pyWorker.on('error', (err) => { - logger.error(`[${WORKER_ID}] Python worker error`, err); - workerRunning.set({ worker_type: 'python' }, 0); +pyWorker.on('error', err => { + logger.error(`[${WORKER_ID}] Python worker error`, err); + workerRunning.set({ worker_type: 'python' }, 0); }); -otherWorker.on('error', (err) => { - logger.error(`[${WORKER_ID}] Other worker error`, err); - workerRunning.set({ worker_type: 'other' }, 0); +otherWorker.on('error', err => { + logger.error(`[${WORKER_ID}] Other worker error`, err); + workerRunning.set({ worker_type: 'other' }, 0); }); pyWorker.on('closed', () => { - workerRunning.set({ worker_type: 'python' }, 0); + workerRunning.set({ worker_type: 'python' }, 0); }); otherWorker.on('closed', () => { - workerRunning.set({ worker_type: 'other' }, 0); + workerRunning.set({ worker_type: 'other' }, 0); });