diff --git a/BUILD.bazel b/BUILD.bazel index 2774adc9..5254c80e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -13,8 +13,7 @@ load("@gazelle//:def.bzl", "gazelle") # gazelle:resolve go github.com/uber/submitqueue/api/runway/orchestrator/protopb //api/runway/orchestrator/protopb # gazelle:resolve go github.com/uber/submitqueue/api/submitqueue/gateway/protopb //api/submitqueue/gateway/protopb # gazelle:resolve go github.com/uber/submitqueue/api/submitqueue/orchestrator/protopb //api/submitqueue/orchestrator/protopb -# gazelle:resolve go github.com/uber/submitqueue/api/stovepipe/gateway/protopb //api/stovepipe/gateway/protopb -# gazelle:resolve go github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb //api/stovepipe/orchestrator/protopb +# gazelle:resolve go github.com/uber/submitqueue/api/stovepipe/protopb //api/stovepipe/protopb # Export marker files for test data dependencies (used by FindRepoRoot in tests) exports_files( diff --git a/CLAUDE.md b/CLAUDE.md index 11a3b070..c928c9d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,7 +38,7 @@ request.Version = newVersion submitqueue/ # repo root (Go module github.com/uber/submitqueue) ├── api/ # Published wire contracts (cross-domain/external) │ ├── submitqueue/{gateway,orchestrator}/{proto,protopb}/ # RPC (proto) -│ ├── stovepipe/{gateway,orchestrator}/{proto,protopb}/ +│ ├── stovepipe/{proto,protopb}/ # single-service RPC (proto) — no service segment yet │ └── runway/messagequeue/ # external queue contracts (proto + protojson) ├── platform/ # SHARED cross-domain packages — no domain deps │ ├── errs/, metrics/, consumer/, http/ @@ -50,12 +50,8 @@ submitqueue/ # repo root (Go module github.com/uber/submi │ ├── entity/ # SubmitQueue-specific domain entities │ ├── extension/ # SubmitQueue-specific extension impls (storage, counter, mergechecker, …) │ └── core/ # SubmitQueue-internal shared infra (consumer wiring, request, topickey, …) -├── stovepipe/ # Stovepipe domain -│ ├── gateway/ # Gateway service: commit deployment verification entry point -│ ├── orchestrator/ # Orchestrator service: commit verification pipeline -│ ├── entity/ # Stovepipe-specific domain entities -│ ├── extension/ # Stovepipe-specific extension impls -│ └── core/ # Stovepipe-internal shared infra (placeholder; mirrors submitqueue/core) +├── stovepipe/ # Stovepipe domain (single Ping-only service for now) +│ └── controller/ # Business logic (currently just Ping); entity/extension/core added as it grows ├── tool/ # Development and CI tooling ├── example/ │ ├── submitqueue/ # Runnable SubmitQueue servers/clients + Docker Compose @@ -67,7 +63,7 @@ submitqueue/ # repo root (Go module github.com/uber/submi └── doc/ # Documentation ``` -The `platform/` tree holds code reused across domains (infrastructure, shared entities, shared extension contracts). Each **domain** (`submitqueue/`, `stovepipe/`, …) keeps the same internal layout (`gateway/`, `orchestrator/`, `entity/`, `extension/`, `core/`); a domain's own `core/` (e.g. `submitqueue/core/`) holds infra shared only between that domain's services. +The `platform/` tree holds code reused across domains (infrastructure, shared entities, shared extension contracts). Each **domain** (`submitqueue/`, `stovepipe/`, …) grows into the same internal layout (`gateway/`, `orchestrator/`, `entity/`, `extension/`, `core/`); a domain's own `core/` (e.g. `submitqueue/core/`) holds infra shared only between that domain's services. A domain may start smaller — Stovepipe is currently a single Ping-only service with just `controller/` (and a service-segment-free `api/stovepipe/`), adding the other layers as it gains real behavior. The `api/` tree holds **published** wire contracts — those depended on from outside the owning domain. RPC contracts live at `api/{domain}/{service}/` (`proto/` for `.proto` sources, `protopb/` for committed generated Go); a service package may hold multiple `.proto` files, all generating into the same `protopb/`. External message-queue contracts live at `api/{domain}/messagequeue/` (see Message Queue Contracts below). Internal queue contracts do **not** go here — they live under `{domain}/core/messagequeue/`. diff --git a/Makefile b/Makefile index 00a6655c..847a526f 100644 --- a/Makefile +++ b/Makefile @@ -12,10 +12,8 @@ ORCHESTRATOR_COMPOSE_FILE = example/submitqueue/orchestrator/server/docker-compo # Fixed project name for local manual testing (tests use unique random names) SUBMITQUEUE_LOCAL_PROJECT = submitqueue -# Stovepipe compose files -STOVEPIPE_GATEWAY_COMPOSE_FILE = example/stovepipe/gateway/server/docker-compose.yml -STOVEPIPE_ORCHESTRATOR_COMPOSE_FILE = example/stovepipe/orchestrator/server/docker-compose.yml -STOVEPIPE_STACK_COMPOSE_FILE = example/stovepipe/docker-compose.yml +# Stovepipe compose file (single Ping-only service) +STOVEPIPE_COMPOSE_FILE = example/stovepipe/docker-compose.yml # Fixed project name for local manual testing (tests use unique random names) STOVEPIPE_LOCAL_PROJECT = stovepipe @@ -37,7 +35,7 @@ GOIMPORTS_VERSION ?= v0.33.0 # (the out_dir convention in tool/proto/BUILD.bazel) and copied back here. A # package may hold multiple .proto files (e.g. an RPC contract plus messagequeue # contracts); all generated stubs land in the same protopb/ dir. -PROTO_PACKAGES = api/base/change api/base/mergestrategy api/base/messagequeue api/runway/messagequeue api/runway/orchestrator api/submitqueue/gateway api/submitqueue/orchestrator api/stovepipe/gateway api/stovepipe/orchestrator +PROTO_PACKAGES = api/base/change api/base/mergestrategy api/base/messagequeue api/runway/messagequeue api/runway/orchestrator api/submitqueue/gateway api/submitqueue/orchestrator api/stovepipe # Set REPO_ROOT for docker-compose export REPO_ROOT := $(shell pwd) @@ -53,7 +51,7 @@ define assert_clean fi endef -.PHONY: build build-all-linux build-runway-orchestrator-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-init-runway-queue-schema local-runway-orchestrator-start local-runway-orchestrator-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-init-stovepipe-queue-schema local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-runway-orchestrator run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help +.PHONY: build build-all-linux build-runway-orchestrator-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-init-runway-queue-schema local-runway-orchestrator-start local-runway-orchestrator-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway-orchestrator run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help build: ## Build all services and examples @@ -62,7 +60,7 @@ build: ## Build all services and examples @echo "Build complete!" # Build Linux binaries required for Docker containers -build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux build-runway-orchestrator-linux ## Build all Linux binaries for Docker +build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-runway-orchestrator-linux ## Build all Linux binaries for Docker @echo "All Linux binaries ready for Docker" build-runway-orchestrator-linux: ## Build Runway orchestrator Linux binary for Docker @@ -89,21 +87,13 @@ build-submitqueue-orchestrator-linux: ## Build Orchestrator Linux binary for Doc cp -f bazel-bin/example/submitqueue/orchestrator/server/orchestrator .docker-bin/orchestrator @echo "Orchestrator Linux binary ready at .docker-bin/orchestrator" -build-stovepipe-gateway-linux: ## Build Stovepipe gateway Linux binary for Docker - @echo "Building Stovepipe gateway Linux binary for Docker..." - @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/stovepipe/gateway/server:gateway +build-stovepipe-linux: ## Build Stovepipe Linux binary for Docker + @echo "Building Stovepipe Linux binary for Docker..." + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/stovepipe/server:stovepipe @mkdir -p .docker-bin - @cp -f bazel-bin/example/stovepipe/gateway/server/gateway_/gateway .docker-bin/stovepipe-gateway 2>/dev/null || \ - cp -f bazel-bin/example/stovepipe/gateway/server/gateway .docker-bin/stovepipe-gateway - @echo "Stovepipe gateway Linux binary ready at .docker-bin/stovepipe-gateway" - -build-stovepipe-orchestrator-linux: ## Build Stovepipe orchestrator Linux binary for Docker - @echo "Building Stovepipe orchestrator Linux binary for Docker..." - @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/stovepipe/orchestrator/server:orchestrator - @mkdir -p .docker-bin - @cp -f bazel-bin/example/stovepipe/orchestrator/server/orchestrator_/orchestrator .docker-bin/stovepipe-orchestrator 2>/dev/null || \ - cp -f bazel-bin/example/stovepipe/orchestrator/server/orchestrator .docker-bin/stovepipe-orchestrator - @echo "Stovepipe orchestrator Linux binary ready at .docker-bin/stovepipe-orchestrator" + @cp -f bazel-bin/example/stovepipe/server/stovepipe_/stovepipe .docker-bin/stovepipe 2>/dev/null || \ + cp -f bazel-bin/example/stovepipe/server/stovepipe .docker-bin/stovepipe + @echo "Stovepipe Linux binary ready at .docker-bin/stovepipe" check-gazelle: ## Check BUILD.bazel files are up to date @echo "Running Gazelle to check BUILD files..." @@ -223,14 +213,6 @@ local-init-submitqueue-schemas: ## Manually apply all database schemas done @echo "✅ All schemas applied successfully" -local-init-stovepipe-queue-schema: ## Apply queue schema only (mysql-queue) for Stovepipe compose stacks - @echo "Applying queue schema to mysql-queue (Stovepipe; no app storage/counter schema yet)..." - @for file in platform/extension/messagequeue/mysql/schema/*.sql; do \ - echo " - Applying $$(basename $$file)..."; \ - docker exec -i $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \ - done - @echo "✅ Stovepipe queue schema applied successfully" - local-init-runway-queue-schema: ## Apply queue schema only (mysql-queue) for Runway compose stacks @echo "Applying queue schema to mysql-queue (Runway; consumes the merge queues)..." @for file in platform/extension/messagequeue/mysql/schema/*.sql; do \ @@ -321,61 +303,27 @@ local-submitqueue-start: build-all-linux ## Start full stack (Gateway + Orchestr local-stop: ## Stop all services (keep data) @echo "Stopping all services..." @$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down - @$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down + @$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down @$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down @echo "Services stopped. Data volumes preserved." -local-stovepipe-logs: ## View logs from all running Stovepipe services - @$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) logs -f +local-stovepipe-logs: ## View logs from the running Stovepipe service + @$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) logs -f -local-stovepipe-start: build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux ## Start full Stovepipe stack (gateway + orchestrator + MySQL) - @echo "Starting full Stovepipe stack with compose..." - @$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) up -d --build --wait - @echo "Applying queue schema to mysql-queue (no Stovepipe app schema yet)..." - @$(MAKE) -s local-init-stovepipe-queue-schema - @echo "" - @echo "✅ Full Stovepipe stack is running!" - @echo "" - @echo "Expected container NAMEs (project=$(STOVEPIPE_LOCAL_PROJECT), one replica each):" - @echo " $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1" - @echo " $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1" - @echo " $(STOVEPIPE_LOCAL_PROJECT)-gateway-service-1" - @echo " $(STOVEPIPE_LOCAL_PROJECT)-orchestrator-service-1" +local-stovepipe-start: build-stovepipe-linux ## Start Stovepipe service (single Ping-only gRPC service) + @echo "Starting Stovepipe service with compose..." + @$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) up -d --build --wait @echo "" - @$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) ps + @echo "✅ Stovepipe service is running!" @echo "" - @echo "Stovepipe gateway gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" - @echo "Stovepipe orchestrator gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" - @echo "MySQL App port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" - @echo "MySQL Queue port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" - -local-stovepipe-orchestrator-start: build-stovepipe-orchestrator-linux ## Start Stovepipe orchestrator locally (orchestrator + 2 MySQL databases) - @echo "Starting Stovepipe orchestrator with compose..." - @$(COMPOSE) -f $(STOVEPIPE_ORCHESTRATOR_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) up -d --build --wait - @echo "Applying queue schema to mysql-queue (no Stovepipe app schema yet)..." - @$(MAKE) -s local-init-stovepipe-queue-schema + @$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) ps @echo "" - @echo "✅ Stovepipe orchestrator is running!" - @echo "" - @$(COMPOSE) -f $(STOVEPIPE_ORCHESTRATOR_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) ps - @echo "" - @echo "Stovepipe orchestrator gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-orchestrator-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" - @echo "MySQL App port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" - @echo "MySQL Queue port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" - -local-stovepipe-gateway-start: build-stovepipe-gateway-linux ## Start Stovepipe gateway locally (gateway + 2 MySQL databases) - @echo "Starting Stovepipe gateway with compose..." - @$(COMPOSE) -f $(STOVEPIPE_GATEWAY_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) up -d --build --wait - @echo "Applying queue schema to mysql-queue (no Stovepipe app schema yet)..." - @$(MAKE) -s local-init-stovepipe-queue-schema - @echo "" - @echo "✅ Stovepipe gateway is running!" - @echo "" - @$(COMPOSE) -f $(STOVEPIPE_GATEWAY_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) ps - @echo "" - @echo "Stovepipe gateway gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" - @echo "MySQL App port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" - @echo "MySQL Queue port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')" + @echo "Stovepipe gRPC port: $$(docker port $(STOVEPIPE_LOCAL_PROJECT)-stovepipe-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')" + +local-stovepipe-stop: ## Stop the Stovepipe service + @echo "Stopping Stovepipe service..." + @$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down + @echo "Stovepipe service stopped." mocks: ## Generate mock files using mockgen @echo "Generating mocks..." @@ -411,13 +359,9 @@ run-client-submitqueue-gateway: run-client-submitqueue-orchestrator: @$(BAZEL) run //example/submitqueue/orchestrator/client:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8082) -message "$(or $(MESSAGE),ping)" -# Run stovepipe gateway client (connects to any running stovepipe gateway service) -run-client-stovepipe-gateway: - @$(BAZEL) run //example/stovepipe/gateway/client:gateway -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)" - -# Run stovepipe orchestrator client (connects to any running stovepipe orchestrator service) -run-client-stovepipe-orchestrator: - @$(BAZEL) run //example/stovepipe/orchestrator/client:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8084) -message "$(or $(MESSAGE),ping)" +# Run stovepipe client (connects to any running stovepipe service) +run-client-stovepipe: + @$(BAZEL) run //example/stovepipe/client:stovepipe -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)" # Run runway orchestrator client (connects to any running runway orchestrator service) run-client-runway-orchestrator: diff --git a/README.md b/README.md index 20c45816..dbc81ab6 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Designed for large monorepos and fast-moving teams where concurrent changes can ## Repository layout -Cross-domain Go code (errors, metrics, consumer framework, HTTP helpers, shared entities, shared extension contracts) lives under [`platform/`](platform/README.md). Each product domain has its own tree (`submitqueue/`, `stovepipe/`, …) with `gateway/`, `orchestrator/`, `entity/`, `extension/`, and domain-local `core/`. See [CLAUDE.md](CLAUDE.md) for conventions and import paths. +Cross-domain Go code (errors, metrics, consumer framework, HTTP helpers, shared entities, shared extension contracts) lives under [`platform/`](platform/README.md). Each product domain has its own tree (`submitqueue/`, `stovepipe/`, …) and grows into `gateway/`, `orchestrator/`, `entity/`, `extension/`, and domain-local `core/` — though a domain may start smaller (Stovepipe is currently a single Ping-only service with just `controller/`). See [CLAUDE.md](CLAUDE.md) for conventions and import paths. ## Quick Start diff --git a/api/stovepipe/gateway/proto/BUILD.bazel b/api/stovepipe/gateway/proto/BUILD.bazel deleted file mode 100644 index 36a87e6d..00000000 --- a/api/stovepipe/gateway/proto/BUILD.bazel +++ /dev/null @@ -1,43 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") -load("@rules_go//proto:def.bzl", "go_proto_library") -load("@rules_proto//proto:defs.bzl", "proto_library") - -exports_files( - ["gateway.proto"], - visibility = ["//tool/proto:__pkg__"], -) - -proto_library( - name = "gatewaypb_proto", - srcs = ["gateway.proto"], - visibility = ["//visibility:public"], - deps = ["//api/base/change/proto:changepb_proto"], -) - -# keep -go_proto_library( - name = "gatewaypb_go_proto", - compilers = [ - "@rules_go//proto:go_proto", - "@rules_go//proto:go_grpc_v2", - ], - importpath = "github.com/uber/submitqueue/api/stovepipe/gateway/proto", - proto = ":gatewaypb_proto", - visibility = ["//visibility:public"], - # keep - deps = ["//api/base/change/proto:changepb_go_proto"], -) - -go_library( - name = "proto", - embed = [":gatewaypb_go_proto"], - importpath = "github.com/uber/submitqueue/api/stovepipe/gateway/proto", - visibility = ["//visibility:public"], -) - -go_library( - name = "protopb", - embed = [":gatewaypb_go_proto"], - importpath = "github.com/uber/submitqueue/api/stovepipe/gateway/protopb", - visibility = ["//visibility:public"], -) diff --git a/api/stovepipe/gateway/proto/gateway.proto b/api/stovepipe/gateway/proto/gateway.proto deleted file mode 100644 index 4463e026..00000000 --- a/api/stovepipe/gateway/proto/gateway.proto +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package uber.submitqueue.stovepipe; - -option go_package = "github.com/uber/submitqueue/api/stovepipe/gateway/protopb"; -option java_multiple_files = true; -option java_outer_classname = "GatewayProto"; -option java_package = "com.uber.submitqueue.stovepipe"; - -import "api/base/change/proto/change.proto"; - -// PingRequest is the request for the Ping method -message PingRequest { - // Optional message to include in the ping - string message = 1; -} - -// PingResponse is the response for the Ping method -message PingResponse { - // The response message - string message = 1; - // The service name that handled the request - string service_name = 2; - // Timestamp of when the ping was received - int64 timestamp = 3; - // Hostname of the server that handled the request - string hostname = 4; -} - -// IngestRequest defines a request to ingest a trunk commit into Stovepipe, triggering its verification (build and test) post-merge/post-submit. -message IngestRequest { - // Name of the queue processing the ingest request. The queue should be defined in the configuration, otherwise an error is returned. - string queue = 1; - // Change (trunk commit) to ingest, identified by URI. The target trunk and build/test configuration are defined by the queue configuration. - // Stovepipe ingests commits that are already on trunk; for git, supply a "git://///" URI. - uber.base.change.Change change = 2; -} - -// IngestResponse defines the response to an ingest request. -message IngestResponse { - // Globally unique identifier for the ingested commit's verification request (the "stovepipe id", spid). Used to track the request lifecycle. - string spid = 1; -} - -// StovepipeGateway provides the Stovepipe gateway API. -service StovepipeGateway { - // Ping returns a response indicating the service is alive - rpc Ping(PingRequest) returns (PingResponse) {} - - // Ingest accepts a trunk commit and triggers its verification (post-merge/post-submit build and test). - // The processing is asynchronous and returns an IngestResponse immediately; the verification runs in the background. - rpc Ingest(IngestRequest) returns (IngestResponse) {} -} diff --git a/api/stovepipe/gateway/protopb/BUILD.bazel b/api/stovepipe/gateway/protopb/BUILD.bazel deleted file mode 100644 index 9117601b..00000000 --- a/api/stovepipe/gateway/protopb/BUILD.bazel +++ /dev/null @@ -1,27 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "protopb", - srcs = [ - "gateway.pb.go", - "gateway.pb.yarpc.go", - "gateway_grpc.pb.go", - ], - importpath = "github.com/uber/submitqueue/api/stovepipe/gateway/protopb", - visibility = ["//visibility:public"], - deps = [ - "//api/base/change/protopb", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//codes", - "@org_golang_google_grpc//status", - "@org_golang_google_protobuf//proto", - "@org_golang_google_protobuf//reflect/protoreflect", - "@org_golang_google_protobuf//runtime/protoimpl", - "@org_uber_go_fx//:fx", - "@org_uber_go_yarpc//:yarpc", - "@org_uber_go_yarpc//api/transport", - "@org_uber_go_yarpc//api/x/restriction", - "@org_uber_go_yarpc//encoding/protobuf/reflection", - "@org_uber_go_yarpc//encoding/protobuf/v2:protobuf", - ], -) diff --git a/api/stovepipe/gateway/protopb/gateway.pb.go b/api/stovepipe/gateway/protopb/gateway.pb.go deleted file mode 100644 index 5d75950c..00000000 --- a/api/stovepipe/gateway/protopb/gateway.pb.go +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.36.10 -// protoc v5.29.3 -// source: gateway.proto - -package protopb - -import ( - reflect "reflect" - sync "sync" - unsafe "unsafe" - - protopb "github.com/uber/submitqueue/api/base/change/protopb" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// PingRequest is the request for the Ping method -type PingRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - // Optional message to include in the ping - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *PingRequest) Reset() { - *x = PingRequest{} - mi := &file_gateway_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *PingRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PingRequest) ProtoMessage() {} - -func (x *PingRequest) ProtoReflect() protoreflect.Message { - mi := &file_gateway_proto_msgTypes[0] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. -func (*PingRequest) Descriptor() ([]byte, []int) { - return file_gateway_proto_rawDescGZIP(), []int{0} -} - -func (x *PingRequest) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -// PingResponse is the response for the Ping method -type PingResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - // The response message - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` - // The service name that handled the request - ServiceName string `protobuf:"bytes,2,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"` - // Timestamp of when the ping was received - Timestamp int64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - // Hostname of the server that handled the request - Hostname string `protobuf:"bytes,4,opt,name=hostname,proto3" json:"hostname,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *PingResponse) Reset() { - *x = PingResponse{} - mi := &file_gateway_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *PingResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PingResponse) ProtoMessage() {} - -func (x *PingResponse) ProtoReflect() protoreflect.Message { - mi := &file_gateway_proto_msgTypes[1] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. -func (*PingResponse) Descriptor() ([]byte, []int) { - return file_gateway_proto_rawDescGZIP(), []int{1} -} - -func (x *PingResponse) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -func (x *PingResponse) GetServiceName() string { - if x != nil { - return x.ServiceName - } - return "" -} - -func (x *PingResponse) GetTimestamp() int64 { - if x != nil { - return x.Timestamp - } - return 0 -} - -func (x *PingResponse) GetHostname() string { - if x != nil { - return x.Hostname - } - return "" -} - -// IngestRequest defines a request to ingest a trunk commit into Stovepipe, triggering its verification (build and test) post-merge/post-submit. -type IngestRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - // Name of the queue processing the ingest request. The queue should be defined in the configuration, otherwise an error is returned. - Queue string `protobuf:"bytes,1,opt,name=queue,proto3" json:"queue,omitempty"` - // Change (trunk commit) to ingest, identified by URI. The target trunk and build/test configuration are defined by the queue configuration. - // Stovepipe ingests commits that are already on trunk; for git, supply a "git://///" URI. - Change *protopb.Change `protobuf:"bytes,2,opt,name=change,proto3" json:"change,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *IngestRequest) Reset() { - *x = IngestRequest{} - mi := &file_gateway_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *IngestRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*IngestRequest) ProtoMessage() {} - -func (x *IngestRequest) ProtoReflect() protoreflect.Message { - mi := &file_gateway_proto_msgTypes[2] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use IngestRequest.ProtoReflect.Descriptor instead. -func (*IngestRequest) Descriptor() ([]byte, []int) { - return file_gateway_proto_rawDescGZIP(), []int{2} -} - -func (x *IngestRequest) GetQueue() string { - if x != nil { - return x.Queue - } - return "" -} - -func (x *IngestRequest) GetChange() *protopb.Change { - if x != nil { - return x.Change - } - return nil -} - -// IngestResponse defines the response to an ingest request. -type IngestResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique identifier for the ingested commit's verification request (the "stovepipe id", spid). Used to track the request lifecycle. - Spid string `protobuf:"bytes,1,opt,name=spid,proto3" json:"spid,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *IngestResponse) Reset() { - *x = IngestResponse{} - mi := &file_gateway_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *IngestResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*IngestResponse) ProtoMessage() {} - -func (x *IngestResponse) ProtoReflect() protoreflect.Message { - mi := &file_gateway_proto_msgTypes[3] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use IngestResponse.ProtoReflect.Descriptor instead. -func (*IngestResponse) Descriptor() ([]byte, []int) { - return file_gateway_proto_rawDescGZIP(), []int{3} -} - -func (x *IngestResponse) GetSpid() string { - if x != nil { - return x.Spid - } - return "" -} - -var File_gateway_proto protoreflect.FileDescriptor - -const file_gateway_proto_rawDesc = "" + - "\n" + - "\rgateway.proto\x12\x1auber.submitqueue.stovepipe\x1a\"api/base/change/proto/change.proto\"'\n" + - "\vPingRequest\x12\x18\n" + - "\amessage\x18\x01 \x01(\tR\amessage\"\x85\x01\n" + - "\fPingResponse\x12\x18\n" + - "\amessage\x18\x01 \x01(\tR\amessage\x12!\n" + - "\fservice_name\x18\x02 \x01(\tR\vserviceName\x12\x1c\n" + - "\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\x12\x1a\n" + - "\bhostname\x18\x04 \x01(\tR\bhostname\"W\n" + - "\rIngestRequest\x12\x14\n" + - "\x05queue\x18\x01 \x01(\tR\x05queue\x120\n" + - "\x06change\x18\x02 \x01(\v2\x18.uber.base.change.ChangeR\x06change\"$\n" + - "\x0eIngestResponse\x12\x12\n" + - "\x04spid\x18\x01 \x01(\tR\x04spid2\xd2\x01\n" + - "\x10StovepipeGateway\x12[\n" + - "\x04Ping\x12'.uber.submitqueue.stovepipe.PingRequest\x1a(.uber.submitqueue.stovepipe.PingResponse\"\x00\x12a\n" + - "\x06Ingest\x12).uber.submitqueue.stovepipe.IngestRequest\x1a*.uber.submitqueue.stovepipe.IngestResponse\"\x00Bk\n" + - "\x1ecom.uber.submitqueue.stovepipeB\fGatewayProtoP\x01Z9github.com/uber/submitqueue/api/stovepipe/gateway/protopbb\x06proto3" - -var ( - file_gateway_proto_rawDescOnce sync.Once - file_gateway_proto_rawDescData []byte -) - -func file_gateway_proto_rawDescGZIP() []byte { - file_gateway_proto_rawDescOnce.Do(func() { - file_gateway_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_gateway_proto_rawDesc), len(file_gateway_proto_rawDesc))) - }) - return file_gateway_proto_rawDescData -} - -var file_gateway_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_gateway_proto_goTypes = []any{ - (*PingRequest)(nil), // 0: uber.submitqueue.stovepipe.PingRequest - (*PingResponse)(nil), // 1: uber.submitqueue.stovepipe.PingResponse - (*IngestRequest)(nil), // 2: uber.submitqueue.stovepipe.IngestRequest - (*IngestResponse)(nil), // 3: uber.submitqueue.stovepipe.IngestResponse - (*protopb.Change)(nil), // 4: uber.base.change.Change -} -var file_gateway_proto_depIdxs = []int32{ - 4, // 0: uber.submitqueue.stovepipe.IngestRequest.change:type_name -> uber.base.change.Change - 0, // 1: uber.submitqueue.stovepipe.StovepipeGateway.Ping:input_type -> uber.submitqueue.stovepipe.PingRequest - 2, // 2: uber.submitqueue.stovepipe.StovepipeGateway.Ingest:input_type -> uber.submitqueue.stovepipe.IngestRequest - 1, // 3: uber.submitqueue.stovepipe.StovepipeGateway.Ping:output_type -> uber.submitqueue.stovepipe.PingResponse - 3, // 4: uber.submitqueue.stovepipe.StovepipeGateway.Ingest:output_type -> uber.submitqueue.stovepipe.IngestResponse - 3, // [3:5] is the sub-list for method output_type - 1, // [1:3] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name -} - -func init() { file_gateway_proto_init() } -func file_gateway_proto_init() { - if File_gateway_proto != nil { - return - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_gateway_proto_rawDesc), len(file_gateway_proto_rawDesc)), - NumEnums: 0, - NumMessages: 4, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_gateway_proto_goTypes, - DependencyIndexes: file_gateway_proto_depIdxs, - MessageInfos: file_gateway_proto_msgTypes, - }.Build() - File_gateway_proto = out.File - file_gateway_proto_goTypes = nil - file_gateway_proto_depIdxs = nil -} diff --git a/api/stovepipe/gateway/protopb/gateway.pb.yarpc.go b/api/stovepipe/gateway/protopb/gateway.pb.yarpc.go deleted file mode 100644 index 89f0c61d..00000000 --- a/api/stovepipe/gateway/protopb/gateway.pb.yarpc.go +++ /dev/null @@ -1,324 +0,0 @@ -// Code generated by protoc-gen-yarpc-go. DO NOT EDIT. -// source: gateway.proto - -package protopb - -import ( - "context" - "io/ioutil" - "reflect" - - "go.uber.org/fx" - "go.uber.org/yarpc" - "go.uber.org/yarpc/api/transport" - "go.uber.org/yarpc/api/x/restriction" - "go.uber.org/yarpc/encoding/protobuf/reflection" - v2 "go.uber.org/yarpc/encoding/protobuf/v2" - "google.golang.org/protobuf/proto" -) - -var _ = ioutil.NopCloser - -// StovepipeGatewayYARPCClient is the YARPC client-side interface for the StovepipeGateway service. -type StovepipeGatewayYARPCClient interface { - Ping(context.Context, *PingRequest, ...yarpc.CallOption) (*PingResponse, error) - Ingest(context.Context, *IngestRequest, ...yarpc.CallOption) (*IngestResponse, error) -} - -func newStovepipeGatewayYARPCClient(clientConfig transport.ClientConfig, anyResolver v2.AnyResolver, options ...v2.ClientOption) StovepipeGatewayYARPCClient { - return &_StovepipeGatewayYARPCCaller{v2.NewStreamClient( - v2.ClientParams{ - ServiceName: "uber.submitqueue.stovepipe.StovepipeGateway", - ClientConfig: clientConfig, - AnyResolver: anyResolver, - Options: options, - }, - )} -} - -// NewStovepipeGatewayYARPCClient builds a new YARPC client for the StovepipeGateway service. -func NewStovepipeGatewayYARPCClient(clientConfig transport.ClientConfig, options ...v2.ClientOption) StovepipeGatewayYARPCClient { - return newStovepipeGatewayYARPCClient(clientConfig, nil, options...) -} - -// StovepipeGatewayYARPCServer is the YARPC server-side interface for the StovepipeGateway service. -type StovepipeGatewayYARPCServer interface { - Ping(context.Context, *PingRequest) (*PingResponse, error) - Ingest(context.Context, *IngestRequest) (*IngestResponse, error) -} - -type buildStovepipeGatewayYARPCProceduresParams struct { - Server StovepipeGatewayYARPCServer - AnyResolver v2.AnyResolver -} - -func buildStovepipeGatewayYARPCProcedures(params buildStovepipeGatewayYARPCProceduresParams) []transport.Procedure { - handler := &_StovepipeGatewayYARPCHandler{params.Server} - return v2.BuildProcedures( - v2.BuildProceduresParams{ - ServiceName: "uber.submitqueue.stovepipe.StovepipeGateway", - UnaryHandlerParams: []v2.BuildProceduresUnaryHandlerParams{ - { - MethodName: "Ping", - Handler: v2.NewUnaryHandler( - v2.UnaryHandlerParams{ - Handle: handler.Ping, - NewRequest: newStovepipeGatewayServicePingYARPCRequest, - AnyResolver: params.AnyResolver, - }, - ), - }, - { - MethodName: "Ingest", - Handler: v2.NewUnaryHandler( - v2.UnaryHandlerParams{ - Handle: handler.Ingest, - NewRequest: newStovepipeGatewayServiceIngestYARPCRequest, - AnyResolver: params.AnyResolver, - }, - ), - }, - }, - OnewayHandlerParams: []v2.BuildProceduresOnewayHandlerParams{}, - StreamHandlerParams: []v2.BuildProceduresStreamHandlerParams{}, - }, - ) -} - -// BuildStovepipeGatewayYARPCProcedures prepares an implementation of the StovepipeGateway service for YARPC registration. -func BuildStovepipeGatewayYARPCProcedures(server StovepipeGatewayYARPCServer) []transport.Procedure { - return buildStovepipeGatewayYARPCProcedures(buildStovepipeGatewayYARPCProceduresParams{Server: server}) -} - -// FxStovepipeGatewayYARPCClientParams defines the input -// for NewFxStovepipeGatewayYARPCClient. It provides the -// paramaters to get a StovepipeGatewayYARPCClient in an -// Fx application. -type FxStovepipeGatewayYARPCClientParams struct { - fx.In - - Provider yarpc.ClientConfig - AnyResolver v2.AnyResolver `name:"yarpcfx" optional:"true"` - Restriction restriction.Checker `optional:"true"` -} - -// FxStovepipeGatewayYARPCClientResult defines the output -// of NewFxStovepipeGatewayYARPCClient. It provides a -// StovepipeGatewayYARPCClient to an Fx application. -type FxStovepipeGatewayYARPCClientResult struct { - fx.Out - - Client StovepipeGatewayYARPCClient - - // We are using an fx.Out struct here instead of just returning a client - // so that we can add more values or add named versions of the client in - // the future without breaking any existing code. -} - -// NewFxStovepipeGatewayYARPCClient provides a StovepipeGatewayYARPCClient -// to an Fx application using the given name for routing. -// -// fx.Provide( -// protopb.NewFxStovepipeGatewayYARPCClient("service-name"), -// ... -// ) -func NewFxStovepipeGatewayYARPCClient(name string, options ...v2.ClientOption) interface{} { - return func(params FxStovepipeGatewayYARPCClientParams) FxStovepipeGatewayYARPCClientResult { - cc := params.Provider.ClientConfig(name) - - if params.Restriction != nil { - if namer, ok := cc.GetUnaryOutbound().(transport.Namer); ok { - if err := params.Restriction.Check(v2.Encoding, namer.TransportName()); err != nil { - panic(err.Error()) - } - } - } - - return FxStovepipeGatewayYARPCClientResult{ - Client: newStovepipeGatewayYARPCClient(cc, params.AnyResolver, options...), - } - } -} - -// FxStovepipeGatewayYARPCProceduresParams defines the input -// for NewFxStovepipeGatewayYARPCProcedures. It provides the -// paramaters to get StovepipeGatewayYARPCServer procedures in an -// Fx application. -type FxStovepipeGatewayYARPCProceduresParams struct { - fx.In - - Server StovepipeGatewayYARPCServer - AnyResolver v2.AnyResolver `name:"yarpcfx" optional:"true"` -} - -// FxStovepipeGatewayYARPCProceduresResult defines the output -// of NewFxStovepipeGatewayYARPCProcedures. It provides -// StovepipeGatewayYARPCServer procedures to an Fx application. -// -// The procedures are provided to the "yarpcfx" value group. -// Dig 1.2 or newer must be used for this feature to work. -type FxStovepipeGatewayYARPCProceduresResult struct { - fx.Out - - Procedures []transport.Procedure `group:"yarpcfx"` - ReflectionMeta reflection.ServerMeta `group:"yarpcfx"` -} - -// NewFxStovepipeGatewayYARPCProcedures provides StovepipeGatewayYARPCServer procedures to an Fx application. -// It expects a StovepipeGatewayYARPCServer to be present in the container. -// -// fx.Provide( -// protopb.NewFxStovepipeGatewayYARPCProcedures(), -// ... -// ) -func NewFxStovepipeGatewayYARPCProcedures() interface{} { - return func(params FxStovepipeGatewayYARPCProceduresParams) FxStovepipeGatewayYARPCProceduresResult { - return FxStovepipeGatewayYARPCProceduresResult{ - Procedures: buildStovepipeGatewayYARPCProcedures(buildStovepipeGatewayYARPCProceduresParams{ - Server: params.Server, - AnyResolver: params.AnyResolver, - }), - ReflectionMeta: reflection.ServerMeta{ - ServiceName: "uber.submitqueue.stovepipe.StovepipeGateway", - FileDescriptors: yarpcFileDescriptorClosuref1a937782ebbded5, - }, - } - } -} - -type _StovepipeGatewayYARPCCaller struct { - streamClient v2.StreamClient -} - -func (c *_StovepipeGatewayYARPCCaller) Ping(ctx context.Context, request *PingRequest, options ...yarpc.CallOption) (*PingResponse, error) { - responseMessage, err := c.streamClient.Call(ctx, "Ping", request, newStovepipeGatewayServicePingYARPCResponse, options...) - if responseMessage == nil { - return nil, err - } - response, ok := responseMessage.(*PingResponse) - if !ok { - return nil, v2.CastError(emptyStovepipeGatewayServicePingYARPCResponse, responseMessage) - } - return response, err -} - -func (c *_StovepipeGatewayYARPCCaller) Ingest(ctx context.Context, request *IngestRequest, options ...yarpc.CallOption) (*IngestResponse, error) { - responseMessage, err := c.streamClient.Call(ctx, "Ingest", request, newStovepipeGatewayServiceIngestYARPCResponse, options...) - if responseMessage == nil { - return nil, err - } - response, ok := responseMessage.(*IngestResponse) - if !ok { - return nil, v2.CastError(emptyStovepipeGatewayServiceIngestYARPCResponse, responseMessage) - } - return response, err -} - -type _StovepipeGatewayYARPCHandler struct { - server StovepipeGatewayYARPCServer -} - -func (h *_StovepipeGatewayYARPCHandler) Ping(ctx context.Context, requestMessage proto.Message) (proto.Message, error) { - var request *PingRequest - var ok bool - if requestMessage != nil { - request, ok = requestMessage.(*PingRequest) - if !ok { - return nil, v2.CastError(emptyStovepipeGatewayServicePingYARPCRequest, requestMessage) - } - } - response, err := h.server.Ping(ctx, request) - if response == nil { - return nil, err - } - return response, err -} - -func (h *_StovepipeGatewayYARPCHandler) Ingest(ctx context.Context, requestMessage proto.Message) (proto.Message, error) { - var request *IngestRequest - var ok bool - if requestMessage != nil { - request, ok = requestMessage.(*IngestRequest) - if !ok { - return nil, v2.CastError(emptyStovepipeGatewayServiceIngestYARPCRequest, requestMessage) - } - } - response, err := h.server.Ingest(ctx, request) - if response == nil { - return nil, err - } - return response, err -} - -func newStovepipeGatewayServicePingYARPCRequest() proto.Message { - return &PingRequest{} -} - -func newStovepipeGatewayServicePingYARPCResponse() proto.Message { - return &PingResponse{} -} - -func newStovepipeGatewayServiceIngestYARPCRequest() proto.Message { - return &IngestRequest{} -} - -func newStovepipeGatewayServiceIngestYARPCResponse() proto.Message { - return &IngestResponse{} -} - -var ( - emptyStovepipeGatewayServicePingYARPCRequest = &PingRequest{} - emptyStovepipeGatewayServicePingYARPCResponse = &PingResponse{} - emptyStovepipeGatewayServiceIngestYARPCRequest = &IngestRequest{} - emptyStovepipeGatewayServiceIngestYARPCResponse = &IngestResponse{} -) - -var yarpcFileDescriptorClosuref1a937782ebbded5 = [][]byte{ - // gateway.proto - []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xc1, 0x4e, 0xe3, 0x30, - 0x10, 0x86, 0x37, 0xdb, 0x6e, 0x77, 0x3b, 0x6d, 0x57, 0xc8, 0xe2, 0x10, 0x45, 0x08, 0x95, 0x08, - 0xa9, 0x85, 0x83, 0x83, 0xca, 0x89, 0x6b, 0x39, 0x20, 0x2e, 0xa8, 0x0a, 0x07, 0x24, 0x38, 0x20, - 0x27, 0x8c, 0x52, 0x0b, 0x39, 0x76, 0x6b, 0xa7, 0x88, 0x07, 0xe0, 0x05, 0x79, 0x22, 0x14, 0xdb, - 0x29, 0xe5, 0xd0, 0xc2, 0x29, 0x1e, 0xe7, 0xff, 0xec, 0x99, 0xdf, 0x3f, 0x0c, 0x0a, 0x66, 0xf0, - 0x85, 0xbd, 0x52, 0xb5, 0x94, 0x46, 0x92, 0xa8, 0xca, 0x70, 0x49, 0x75, 0x95, 0x09, 0x6e, 0x16, - 0x15, 0x56, 0x48, 0xb5, 0x91, 0x2b, 0x54, 0x5c, 0x61, 0x14, 0x33, 0xc5, 0x93, 0x8c, 0x69, 0x4c, - 0xf2, 0x39, 0x2b, 0x0b, 0x4c, 0x2c, 0xe2, 0x0b, 0xc7, 0xc7, 0x23, 0xe8, 0xcd, 0x78, 0x59, 0xa4, - 0xb8, 0xa8, 0x50, 0x1b, 0x12, 0xc2, 0x5f, 0x81, 0x5a, 0xb3, 0x02, 0xc3, 0x60, 0x18, 0x8c, 0xbb, - 0x69, 0x53, 0xc6, 0x6f, 0x01, 0xf4, 0x9d, 0x52, 0x2b, 0x59, 0x6a, 0xdc, 0x2e, 0x25, 0x47, 0xd0, - 0xd7, 0xb8, 0x5c, 0xf1, 0x1c, 0x1f, 0x4b, 0x26, 0x30, 0xfc, 0x6d, 0x7f, 0xf7, 0xfc, 0xde, 0x0d, - 0x13, 0x48, 0x0e, 0xa0, 0x6b, 0xb8, 0x40, 0x6d, 0x98, 0x50, 0x61, 0x6b, 0x18, 0x8c, 0x5b, 0xe9, - 0xe7, 0x06, 0x89, 0xe0, 0xdf, 0x5c, 0x6a, 0x63, 0xe1, 0xb6, 0x85, 0xd7, 0x75, 0x7c, 0x07, 0x83, - 0xeb, 0xb2, 0x40, 0x6d, 0x9a, 0x96, 0xf7, 0xe1, 0x8f, 0x1d, 0xdc, 0x77, 0xe1, 0x0a, 0x72, 0x06, - 0x1d, 0x37, 0xa7, 0xbd, 0xbd, 0x37, 0x09, 0xa9, 0x35, 0xaa, 0x76, 0x83, 0x7a, 0x03, 0x2e, 0xed, - 0x27, 0xf5, 0xba, 0xf8, 0x18, 0xfe, 0x37, 0x07, 0xfb, 0x09, 0x09, 0xb4, 0xb5, 0xe2, 0x4f, 0xfe, - 0x60, 0xbb, 0x9e, 0xbc, 0x07, 0xb0, 0x77, 0xdb, 0x38, 0x7c, 0xe5, 0x9e, 0x82, 0x3c, 0x40, 0xbb, - 0xb6, 0x86, 0x8c, 0xe8, 0xf6, 0xd7, 0xa0, 0x1b, 0x36, 0x47, 0xe3, 0xef, 0x85, 0xae, 0x87, 0xf8, - 0x17, 0x61, 0xd0, 0x71, 0x7d, 0x91, 0x93, 0x5d, 0xd4, 0x17, 0x53, 0xa2, 0xd3, 0x9f, 0x48, 0x9b, - 0x2b, 0xa6, 0xcf, 0x70, 0x98, 0x4b, 0xb1, 0x03, 0x99, 0xf6, 0xfd, 0xa8, 0xb3, 0x3a, 0x34, 0xb3, - 0xe0, 0xfe, 0xa2, 0xe0, 0x66, 0x5e, 0x65, 0x34, 0x97, 0x22, 0xa9, 0xb1, 0x64, 0x03, 0x4b, 0xea, - 0xd8, 0xad, 0xd1, 0xc4, 0xe7, 0xd5, 0x85, 0x4f, 0x65, 0x59, 0xc7, 0x2e, 0xce, 0x3f, 0x02, 0x00, - 0x00, 0xff, 0xff, 0xd1, 0x69, 0x3d, 0x05, 0xc9, 0x02, 0x00, 0x00, - }, - // api/base/change/proto/change.proto - []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4a, 0x2c, 0xc8, 0xd4, - 0x4f, 0x4a, 0x2c, 0x4e, 0xd5, 0x4f, 0xce, 0x48, 0xcc, 0x4b, 0x4f, 0xd5, 0x2f, 0x28, 0xca, 0x2f, - 0xc9, 0x87, 0x72, 0xf4, 0xc0, 0x1c, 0x21, 0x81, 0xd2, 0xa4, 0xd4, 0x22, 0x3d, 0x90, 0x22, 0x3d, - 0x88, 0xb8, 0x92, 0x0c, 0x17, 0x9b, 0x33, 0x98, 0x25, 0x24, 0xc4, 0xc5, 0x52, 0x5a, 0x94, 0x59, - 0x2c, 0xc1, 0xa8, 0xc0, 0xac, 0xc1, 0x19, 0x04, 0x66, 0x3b, 0xa5, 0x71, 0x29, 0x24, 0xe7, 0xe7, - 0xea, 0x81, 0x75, 0x15, 0x97, 0x26, 0xe5, 0x66, 0x96, 0x14, 0x96, 0xa6, 0x96, 0xa6, 0x22, 0x9b, - 0xe0, 0xc4, 0x0d, 0xd1, 0x1f, 0x00, 0xb2, 0x20, 0x80, 0x31, 0xca, 0x38, 0x3d, 0xb3, 0x24, 0xa3, - 0x34, 0x49, 0x2f, 0x39, 0x3f, 0x57, 0x1f, 0xa4, 0x4f, 0x1f, 0x49, 0x9f, 0x3e, 0x56, 0x27, 0x16, - 0x24, 0x25, 0xb1, 0x81, 0x19, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbf, 0x91, 0x62, 0xef, - 0xc4, 0x00, 0x00, 0x00, - }, -} - -func init() { - yarpc.RegisterClientBuilder( - func(clientConfig transport.ClientConfig, structField reflect.StructField) StovepipeGatewayYARPCClient { - return NewStovepipeGatewayYARPCClient(clientConfig, v2.ClientBuilderOptions(clientConfig, structField)...) - }, - ) -} diff --git a/api/stovepipe/gateway/protopb/gateway_grpc.pb.go b/api/stovepipe/gateway/protopb/gateway_grpc.pb.go deleted file mode 100644 index b1056451..00000000 --- a/api/stovepipe/gateway/protopb/gateway_grpc.pb.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.5.1 -// - protoc v5.29.3 -// source: gateway.proto - -package protopb - -import ( - context "context" - - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.64.0 or later. -const _ = grpc.SupportPackageIsVersion9 - -const ( - StovepipeGateway_Ping_FullMethodName = "/uber.submitqueue.stovepipe.StovepipeGateway/Ping" - StovepipeGateway_Ingest_FullMethodName = "/uber.submitqueue.stovepipe.StovepipeGateway/Ingest" -) - -// StovepipeGatewayClient is the client API for StovepipeGateway service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -// -// StovepipeGateway provides the Stovepipe gateway API. -type StovepipeGatewayClient interface { - // Ping returns a response indicating the service is alive - Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) - // Ingest accepts a trunk commit and triggers its verification (post-merge/post-submit build and test). - // The processing is asynchronous and returns an IngestResponse immediately; the verification runs in the background. - Ingest(ctx context.Context, in *IngestRequest, opts ...grpc.CallOption) (*IngestResponse, error) -} - -type stovepipeGatewayClient struct { - cc grpc.ClientConnInterface -} - -func NewStovepipeGatewayClient(cc grpc.ClientConnInterface) StovepipeGatewayClient { - return &stovepipeGatewayClient{cc} -} - -func (c *stovepipeGatewayClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(PingResponse) - err := c.cc.Invoke(ctx, StovepipeGateway_Ping_FullMethodName, in, out, cOpts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *stovepipeGatewayClient) Ingest(ctx context.Context, in *IngestRequest, opts ...grpc.CallOption) (*IngestResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(IngestResponse) - err := c.cc.Invoke(ctx, StovepipeGateway_Ingest_FullMethodName, in, out, cOpts...) - if err != nil { - return nil, err - } - return out, nil -} - -// StovepipeGatewayServer is the server API for StovepipeGateway service. -// All implementations must embed UnimplementedStovepipeGatewayServer -// for forward compatibility. -// -// StovepipeGateway provides the Stovepipe gateway API. -type StovepipeGatewayServer interface { - // Ping returns a response indicating the service is alive - Ping(context.Context, *PingRequest) (*PingResponse, error) - // Ingest accepts a trunk commit and triggers its verification (post-merge/post-submit build and test). - // The processing is asynchronous and returns an IngestResponse immediately; the verification runs in the background. - Ingest(context.Context, *IngestRequest) (*IngestResponse, error) - mustEmbedUnimplementedStovepipeGatewayServer() -} - -// UnimplementedStovepipeGatewayServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedStovepipeGatewayServer struct{} - -func (UnimplementedStovepipeGatewayServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") -} -func (UnimplementedStovepipeGatewayServer) Ingest(context.Context, *IngestRequest) (*IngestResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Ingest not implemented") -} -func (UnimplementedStovepipeGatewayServer) mustEmbedUnimplementedStovepipeGatewayServer() {} -func (UnimplementedStovepipeGatewayServer) testEmbeddedByValue() {} - -// UnsafeStovepipeGatewayServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to StovepipeGatewayServer will -// result in compilation errors. -type UnsafeStovepipeGatewayServer interface { - mustEmbedUnimplementedStovepipeGatewayServer() -} - -func RegisterStovepipeGatewayServer(s grpc.ServiceRegistrar, srv StovepipeGatewayServer) { - // If the following call pancis, it indicates UnimplementedStovepipeGatewayServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } - s.RegisterService(&StovepipeGateway_ServiceDesc, srv) -} - -func _StovepipeGateway_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(PingRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(StovepipeGatewayServer).Ping(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: StovepipeGateway_Ping_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(StovepipeGatewayServer).Ping(ctx, req.(*PingRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _StovepipeGateway_Ingest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(IngestRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(StovepipeGatewayServer).Ingest(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: StovepipeGateway_Ingest_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(StovepipeGatewayServer).Ingest(ctx, req.(*IngestRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// StovepipeGateway_ServiceDesc is the grpc.ServiceDesc for StovepipeGateway service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var StovepipeGateway_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "uber.submitqueue.stovepipe.StovepipeGateway", - HandlerType: (*StovepipeGatewayServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Ping", - Handler: _StovepipeGateway_Ping_Handler, - }, - { - MethodName: "Ingest", - Handler: _StovepipeGateway_Ingest_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "gateway.proto", -} diff --git a/api/stovepipe/orchestrator/protopb/orchestrator.pb.yarpc.go b/api/stovepipe/orchestrator/protopb/orchestrator.pb.yarpc.go deleted file mode 100644 index efa204a8..00000000 --- a/api/stovepipe/orchestrator/protopb/orchestrator.pb.yarpc.go +++ /dev/null @@ -1,256 +0,0 @@ -// Code generated by protoc-gen-yarpc-go. DO NOT EDIT. -// source: orchestrator.proto - -package protopb - -import ( - "context" - "io/ioutil" - "reflect" - - "go.uber.org/fx" - "go.uber.org/yarpc" - "go.uber.org/yarpc/api/transport" - "go.uber.org/yarpc/api/x/restriction" - "go.uber.org/yarpc/encoding/protobuf/reflection" - v2 "go.uber.org/yarpc/encoding/protobuf/v2" - "google.golang.org/protobuf/proto" -) - -var _ = ioutil.NopCloser - -// StovepipeOrchestratorYARPCClient is the YARPC client-side interface for the StovepipeOrchestrator service. -type StovepipeOrchestratorYARPCClient interface { - Ping(context.Context, *PingRequest, ...yarpc.CallOption) (*PingResponse, error) -} - -func newStovepipeOrchestratorYARPCClient(clientConfig transport.ClientConfig, anyResolver v2.AnyResolver, options ...v2.ClientOption) StovepipeOrchestratorYARPCClient { - return &_StovepipeOrchestratorYARPCCaller{v2.NewStreamClient( - v2.ClientParams{ - ServiceName: "uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator", - ClientConfig: clientConfig, - AnyResolver: anyResolver, - Options: options, - }, - )} -} - -// NewStovepipeOrchestratorYARPCClient builds a new YARPC client for the StovepipeOrchestrator service. -func NewStovepipeOrchestratorYARPCClient(clientConfig transport.ClientConfig, options ...v2.ClientOption) StovepipeOrchestratorYARPCClient { - return newStovepipeOrchestratorYARPCClient(clientConfig, nil, options...) -} - -// StovepipeOrchestratorYARPCServer is the YARPC server-side interface for the StovepipeOrchestrator service. -type StovepipeOrchestratorYARPCServer interface { - Ping(context.Context, *PingRequest) (*PingResponse, error) -} - -type buildStovepipeOrchestratorYARPCProceduresParams struct { - Server StovepipeOrchestratorYARPCServer - AnyResolver v2.AnyResolver -} - -func buildStovepipeOrchestratorYARPCProcedures(params buildStovepipeOrchestratorYARPCProceduresParams) []transport.Procedure { - handler := &_StovepipeOrchestratorYARPCHandler{params.Server} - return v2.BuildProcedures( - v2.BuildProceduresParams{ - ServiceName: "uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator", - UnaryHandlerParams: []v2.BuildProceduresUnaryHandlerParams{ - { - MethodName: "Ping", - Handler: v2.NewUnaryHandler( - v2.UnaryHandlerParams{ - Handle: handler.Ping, - NewRequest: newStovepipeOrchestratorServicePingYARPCRequest, - AnyResolver: params.AnyResolver, - }, - ), - }, - }, - OnewayHandlerParams: []v2.BuildProceduresOnewayHandlerParams{}, - StreamHandlerParams: []v2.BuildProceduresStreamHandlerParams{}, - }, - ) -} - -// BuildStovepipeOrchestratorYARPCProcedures prepares an implementation of the StovepipeOrchestrator service for YARPC registration. -func BuildStovepipeOrchestratorYARPCProcedures(server StovepipeOrchestratorYARPCServer) []transport.Procedure { - return buildStovepipeOrchestratorYARPCProcedures(buildStovepipeOrchestratorYARPCProceduresParams{Server: server}) -} - -// FxStovepipeOrchestratorYARPCClientParams defines the input -// for NewFxStovepipeOrchestratorYARPCClient. It provides the -// paramaters to get a StovepipeOrchestratorYARPCClient in an -// Fx application. -type FxStovepipeOrchestratorYARPCClientParams struct { - fx.In - - Provider yarpc.ClientConfig - AnyResolver v2.AnyResolver `name:"yarpcfx" optional:"true"` - Restriction restriction.Checker `optional:"true"` -} - -// FxStovepipeOrchestratorYARPCClientResult defines the output -// of NewFxStovepipeOrchestratorYARPCClient. It provides a -// StovepipeOrchestratorYARPCClient to an Fx application. -type FxStovepipeOrchestratorYARPCClientResult struct { - fx.Out - - Client StovepipeOrchestratorYARPCClient - - // We are using an fx.Out struct here instead of just returning a client - // so that we can add more values or add named versions of the client in - // the future without breaking any existing code. -} - -// NewFxStovepipeOrchestratorYARPCClient provides a StovepipeOrchestratorYARPCClient -// to an Fx application using the given name for routing. -// -// fx.Provide( -// protopb.NewFxStovepipeOrchestratorYARPCClient("service-name"), -// ... -// ) -func NewFxStovepipeOrchestratorYARPCClient(name string, options ...v2.ClientOption) interface{} { - return func(params FxStovepipeOrchestratorYARPCClientParams) FxStovepipeOrchestratorYARPCClientResult { - cc := params.Provider.ClientConfig(name) - - if params.Restriction != nil { - if namer, ok := cc.GetUnaryOutbound().(transport.Namer); ok { - if err := params.Restriction.Check(v2.Encoding, namer.TransportName()); err != nil { - panic(err.Error()) - } - } - } - - return FxStovepipeOrchestratorYARPCClientResult{ - Client: newStovepipeOrchestratorYARPCClient(cc, params.AnyResolver, options...), - } - } -} - -// FxStovepipeOrchestratorYARPCProceduresParams defines the input -// for NewFxStovepipeOrchestratorYARPCProcedures. It provides the -// paramaters to get StovepipeOrchestratorYARPCServer procedures in an -// Fx application. -type FxStovepipeOrchestratorYARPCProceduresParams struct { - fx.In - - Server StovepipeOrchestratorYARPCServer - AnyResolver v2.AnyResolver `name:"yarpcfx" optional:"true"` -} - -// FxStovepipeOrchestratorYARPCProceduresResult defines the output -// of NewFxStovepipeOrchestratorYARPCProcedures. It provides -// StovepipeOrchestratorYARPCServer procedures to an Fx application. -// -// The procedures are provided to the "yarpcfx" value group. -// Dig 1.2 or newer must be used for this feature to work. -type FxStovepipeOrchestratorYARPCProceduresResult struct { - fx.Out - - Procedures []transport.Procedure `group:"yarpcfx"` - ReflectionMeta reflection.ServerMeta `group:"yarpcfx"` -} - -// NewFxStovepipeOrchestratorYARPCProcedures provides StovepipeOrchestratorYARPCServer procedures to an Fx application. -// It expects a StovepipeOrchestratorYARPCServer to be present in the container. -// -// fx.Provide( -// protopb.NewFxStovepipeOrchestratorYARPCProcedures(), -// ... -// ) -func NewFxStovepipeOrchestratorYARPCProcedures() interface{} { - return func(params FxStovepipeOrchestratorYARPCProceduresParams) FxStovepipeOrchestratorYARPCProceduresResult { - return FxStovepipeOrchestratorYARPCProceduresResult{ - Procedures: buildStovepipeOrchestratorYARPCProcedures(buildStovepipeOrchestratorYARPCProceduresParams{ - Server: params.Server, - AnyResolver: params.AnyResolver, - }), - ReflectionMeta: reflection.ServerMeta{ - ServiceName: "uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator", - FileDescriptors: yarpcFileDescriptorClosure96b6e6782baaa298, - }, - } - } -} - -type _StovepipeOrchestratorYARPCCaller struct { - streamClient v2.StreamClient -} - -func (c *_StovepipeOrchestratorYARPCCaller) Ping(ctx context.Context, request *PingRequest, options ...yarpc.CallOption) (*PingResponse, error) { - responseMessage, err := c.streamClient.Call(ctx, "Ping", request, newStovepipeOrchestratorServicePingYARPCResponse, options...) - if responseMessage == nil { - return nil, err - } - response, ok := responseMessage.(*PingResponse) - if !ok { - return nil, v2.CastError(emptyStovepipeOrchestratorServicePingYARPCResponse, responseMessage) - } - return response, err -} - -type _StovepipeOrchestratorYARPCHandler struct { - server StovepipeOrchestratorYARPCServer -} - -func (h *_StovepipeOrchestratorYARPCHandler) Ping(ctx context.Context, requestMessage proto.Message) (proto.Message, error) { - var request *PingRequest - var ok bool - if requestMessage != nil { - request, ok = requestMessage.(*PingRequest) - if !ok { - return nil, v2.CastError(emptyStovepipeOrchestratorServicePingYARPCRequest, requestMessage) - } - } - response, err := h.server.Ping(ctx, request) - if response == nil { - return nil, err - } - return response, err -} - -func newStovepipeOrchestratorServicePingYARPCRequest() proto.Message { - return &PingRequest{} -} - -func newStovepipeOrchestratorServicePingYARPCResponse() proto.Message { - return &PingResponse{} -} - -var ( - emptyStovepipeOrchestratorServicePingYARPCRequest = &PingRequest{} - emptyStovepipeOrchestratorServicePingYARPCResponse = &PingResponse{} -) - -var yarpcFileDescriptorClosure96b6e6782baaa298 = [][]byte{ - // orchestrator.proto - []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x91, 0xcf, 0x4a, 0x03, 0x31, - 0x10, 0xc6, 0x8d, 0x2d, 0x6a, 0xd3, 0x5e, 0x0c, 0x08, 0x4b, 0xf1, 0x50, 0xf7, 0xd2, 0x82, 0x90, - 0x80, 0x7f, 0xae, 0x1e, 0xfa, 0x00, 0xba, 0xac, 0x37, 0x2f, 0x92, 0x5d, 0x86, 0xdd, 0x1c, 0xb2, - 0x49, 0x33, 0x93, 0x3e, 0x80, 0xe0, 0xd5, 0x67, 0x96, 0xdd, 0xda, 0x9a, 0x8b, 0x50, 0x6f, 0x99, - 0xc9, 0xf7, 0x7d, 0xcc, 0x6f, 0x86, 0x0b, 0x17, 0xea, 0x16, 0x90, 0x82, 0x26, 0x17, 0xa4, 0x0f, - 0x8e, 0x9c, 0x58, 0xc6, 0x0a, 0x82, 0xc4, 0x58, 0x59, 0x43, 0x9b, 0x08, 0x11, 0x24, 0x92, 0xdb, - 0x82, 0x37, 0x1e, 0x64, 0x2a, 0xcf, 0x97, 0x7c, 0x5a, 0x98, 0xae, 0x29, 0x61, 0x13, 0x01, 0x49, - 0x64, 0xfc, 0xdc, 0x02, 0xa2, 0x6e, 0x20, 0x63, 0x0b, 0xb6, 0x9a, 0x94, 0xfb, 0x32, 0xff, 0x64, - 0x7c, 0xb6, 0x53, 0xa2, 0x77, 0x1d, 0xc2, 0xdf, 0x52, 0x71, 0xc3, 0x67, 0x08, 0x61, 0x6b, 0x6a, - 0x78, 0xef, 0xb4, 0x85, 0xec, 0x74, 0xf8, 0x9e, 0xfe, 0xf4, 0x9e, 0xb5, 0x05, 0x71, 0xcd, 0x27, - 0x64, 0x2c, 0x20, 0x69, 0xeb, 0xb3, 0xd1, 0x82, 0xad, 0x46, 0xe5, 0x6f, 0x43, 0xcc, 0xf9, 0x45, - 0xeb, 0x90, 0x06, 0xf3, 0x78, 0x30, 0x1f, 0xea, 0xbb, 0x2f, 0xc6, 0xaf, 0x5e, 0xf7, 0x2c, 0x2f, - 0x09, 0x8a, 0x88, 0x7c, 0xdc, 0x0f, 0x28, 0x1e, 0xe4, 0x91, 0xf0, 0x32, 0x21, 0x9f, 0x3f, 0xfe, - 0xd3, 0xb5, 0xdb, 0x42, 0x7e, 0xb2, 0xfe, 0x60, 0xfc, 0xb6, 0x76, 0xf6, 0x58, 0xfb, 0xfa, 0x32, - 0x1d, 0xba, 0xe8, 0xaf, 0x55, 0xb0, 0xb7, 0xa7, 0xc6, 0x50, 0x1b, 0x2b, 0x59, 0x3b, 0xab, 0xfa, - 0x20, 0x95, 0x04, 0x29, 0xed, 0x8d, 0x3a, 0x84, 0xa9, 0x34, 0x4c, 0x0d, 0xd7, 0xf6, 0x55, 0x75, - 0x36, 0x3c, 0xee, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf1, 0x27, 0x20, 0xff, 0x0c, 0x02, 0x00, - 0x00, - }, -} - -func init() { - yarpc.RegisterClientBuilder( - func(clientConfig transport.ClientConfig, structField reflect.StructField) StovepipeOrchestratorYARPCClient { - return NewStovepipeOrchestratorYARPCClient(clientConfig, v2.ClientBuilderOptions(clientConfig, structField)...) - }, - ) -} diff --git a/api/stovepipe/orchestrator/proto/BUILD.bazel b/api/stovepipe/proto/BUILD.bazel similarity index 54% rename from api/stovepipe/orchestrator/proto/BUILD.bazel rename to api/stovepipe/proto/BUILD.bazel index f630a6cc..6a26d753 100644 --- a/api/stovepipe/orchestrator/proto/BUILD.bazel +++ b/api/stovepipe/proto/BUILD.bazel @@ -3,38 +3,38 @@ load("@rules_go//proto:def.bzl", "go_proto_library") load("@rules_proto//proto:defs.bzl", "proto_library") exports_files( - ["orchestrator.proto"], + ["stovepipe.proto"], visibility = ["//tool/proto:__pkg__"], ) proto_library( - name = "orchestratorpb_proto", - srcs = ["orchestrator.proto"], + name = "stovepipepb_proto", + srcs = ["stovepipe.proto"], visibility = ["//visibility:public"], ) # keep go_proto_library( - name = "orchestratorpb_go_proto", + name = "stovepipepb_go_proto", compilers = [ "@rules_go//proto:go_proto", "@rules_go//proto:go_grpc_v2", ], - importpath = "github.com/uber/submitqueue/api/stovepipe/orchestrator/proto", - proto = ":orchestratorpb_proto", + importpath = "github.com/uber/submitqueue/api/stovepipe/proto", + proto = ":stovepipepb_proto", visibility = ["//visibility:public"], ) go_library( name = "proto", - embed = [":orchestratorpb_go_proto"], - importpath = "github.com/uber/submitqueue/api/stovepipe/orchestrator/proto", + embed = [":stovepipepb_go_proto"], + importpath = "github.com/uber/submitqueue/api/stovepipe/proto", visibility = ["//visibility:public"], ) go_library( name = "protopb", - embed = [":orchestratorpb_go_proto"], - importpath = "github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb", + embed = [":stovepipepb_go_proto"], + importpath = "github.com/uber/submitqueue/api/stovepipe/protopb", visibility = ["//visibility:public"], ) diff --git a/api/stovepipe/orchestrator/proto/orchestrator.proto b/api/stovepipe/proto/stovepipe.proto similarity index 78% rename from api/stovepipe/orchestrator/proto/orchestrator.proto rename to api/stovepipe/proto/stovepipe.proto index 1cd1bcd9..a56f841a 100644 --- a/api/stovepipe/orchestrator/proto/orchestrator.proto +++ b/api/stovepipe/proto/stovepipe.proto @@ -14,12 +14,12 @@ syntax = "proto3"; -package uber.submitqueue.stovepipe.orchestrator; +package uber.submitqueue.stovepipe; -option go_package = "github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb"; +option go_package = "github.com/uber/submitqueue/api/stovepipe/protopb"; option java_multiple_files = true; -option java_outer_classname = "OrchestratorProto"; -option java_package = "com.uber.submitqueue.stovepipe.orchestrator"; +option java_outer_classname = "StovepipeProto"; +option java_package = "com.uber.submitqueue.stovepipe"; // PingRequest is the request for the Ping method message PingRequest { @@ -39,8 +39,8 @@ message PingResponse { string hostname = 4; } -// StovepipeOrchestrator provides the Stovepipe orchestrator API. -service StovepipeOrchestrator { +// Stovepipe provides the Stovepipe API. +service Stovepipe { // Ping returns a response indicating the service is alive rpc Ping(PingRequest) returns (PingResponse) {} } diff --git a/api/stovepipe/orchestrator/protopb/BUILD.bazel b/api/stovepipe/protopb/BUILD.bazel similarity index 80% rename from api/stovepipe/orchestrator/protopb/BUILD.bazel rename to api/stovepipe/protopb/BUILD.bazel index 5ece6304..4d969eed 100644 --- a/api/stovepipe/orchestrator/protopb/BUILD.bazel +++ b/api/stovepipe/protopb/BUILD.bazel @@ -3,11 +3,11 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "protopb", srcs = [ - "orchestrator.pb.go", - "orchestrator.pb.yarpc.go", - "orchestrator_grpc.pb.go", + "stovepipe.pb.go", + "stovepipe.pb.yarpc.go", + "stovepipe_grpc.pb.go", ], - importpath = "github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb", + importpath = "github.com/uber/submitqueue/api/stovepipe/protopb", visibility = ["//visibility:public"], deps = [ "@org_golang_google_grpc//:grpc", diff --git a/api/stovepipe/orchestrator/protopb/orchestrator.pb.go b/api/stovepipe/protopb/stovepipe.pb.go similarity index 66% rename from api/stovepipe/orchestrator/protopb/orchestrator.pb.go rename to api/stovepipe/protopb/stovepipe.pb.go index c78852f1..1ded621b 100644 --- a/api/stovepipe/orchestrator/protopb/orchestrator.pb.go +++ b/api/stovepipe/protopb/stovepipe.pb.go @@ -16,7 +16,7 @@ // versions: // protoc-gen-go v1.36.10 // protoc v5.29.3 -// source: orchestrator.proto +// source: stovepipe.proto package protopb @@ -47,7 +47,7 @@ type PingRequest struct { func (x *PingRequest) Reset() { *x = PingRequest{} - mi := &file_orchestrator_proto_msgTypes[0] + mi := &file_stovepipe_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -59,7 +59,7 @@ func (x *PingRequest) String() string { func (*PingRequest) ProtoMessage() {} func (x *PingRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[0] + mi := &file_stovepipe_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -72,7 +72,7 @@ func (x *PingRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. func (*PingRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{0} + return file_stovepipe_proto_rawDescGZIP(), []int{0} } func (x *PingRequest) GetMessage() string { @@ -99,7 +99,7 @@ type PingResponse struct { func (x *PingResponse) Reset() { *x = PingResponse{} - mi := &file_orchestrator_proto_msgTypes[1] + mi := &file_stovepipe_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -111,7 +111,7 @@ func (x *PingResponse) String() string { func (*PingResponse) ProtoMessage() {} func (x *PingResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[1] + mi := &file_stovepipe_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -124,7 +124,7 @@ func (x *PingResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. func (*PingResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{1} + return file_stovepipe_proto_rawDescGZIP(), []int{1} } func (x *PingResponse) GetMessage() string { @@ -155,42 +155,42 @@ func (x *PingResponse) GetHostname() string { return "" } -var File_orchestrator_proto protoreflect.FileDescriptor +var File_stovepipe_proto protoreflect.FileDescriptor -const file_orchestrator_proto_rawDesc = "" + +const file_stovepipe_proto_rawDesc = "" + "\n" + - "\x12orchestrator.proto\x12'uber.submitqueue.stovepipe.orchestrator\"'\n" + + "\x0fstovepipe.proto\x12\x1auber.submitqueue.stovepipe\"'\n" + "\vPingRequest\x12\x18\n" + "\amessage\x18\x01 \x01(\tR\amessage\"\x85\x01\n" + "\fPingResponse\x12\x18\n" + "\amessage\x18\x01 \x01(\tR\amessage\x12!\n" + "\fservice_name\x18\x02 \x01(\tR\vserviceName\x12\x1c\n" + "\ttimestamp\x18\x03 \x01(\x03R\ttimestamp\x12\x1a\n" + - "\bhostname\x18\x04 \x01(\tR\bhostname2\x8e\x01\n" + - "\x15StovepipeOrchestrator\x12u\n" + - "\x04Ping\x124.uber.submitqueue.stovepipe.orchestrator.PingRequest\x1a5.uber.submitqueue.stovepipe.orchestrator.PingResponse\"\x00B\x82\x01\n" + - "+com.uber.submitqueue.stovepipe.orchestratorB\x11OrchestratorProtoP\x01Z>github.com/uber/submitqueue/api/stovepipe/orchestrator/protopbb\x06proto3" + "\bhostname\x18\x04 \x01(\tR\bhostname2h\n" + + "\tStovepipe\x12[\n" + + "\x04Ping\x12'.uber.submitqueue.stovepipe.PingRequest\x1a(.uber.submitqueue.stovepipe.PingResponse\"\x00Be\n" + + "\x1ecom.uber.submitqueue.stovepipeB\x0eStovepipeProtoP\x01Z1github.com/uber/submitqueue/api/stovepipe/protopbb\x06proto3" var ( - file_orchestrator_proto_rawDescOnce sync.Once - file_orchestrator_proto_rawDescData []byte + file_stovepipe_proto_rawDescOnce sync.Once + file_stovepipe_proto_rawDescData []byte ) -func file_orchestrator_proto_rawDescGZIP() []byte { - file_orchestrator_proto_rawDescOnce.Do(func() { - file_orchestrator_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_orchestrator_proto_rawDesc), len(file_orchestrator_proto_rawDesc))) +func file_stovepipe_proto_rawDescGZIP() []byte { + file_stovepipe_proto_rawDescOnce.Do(func() { + file_stovepipe_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_stovepipe_proto_rawDesc), len(file_stovepipe_proto_rawDesc))) }) - return file_orchestrator_proto_rawDescData + return file_stovepipe_proto_rawDescData } -var file_orchestrator_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_orchestrator_proto_goTypes = []any{ - (*PingRequest)(nil), // 0: uber.submitqueue.stovepipe.orchestrator.PingRequest - (*PingResponse)(nil), // 1: uber.submitqueue.stovepipe.orchestrator.PingResponse +var file_stovepipe_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_stovepipe_proto_goTypes = []any{ + (*PingRequest)(nil), // 0: uber.submitqueue.stovepipe.PingRequest + (*PingResponse)(nil), // 1: uber.submitqueue.stovepipe.PingResponse } -var file_orchestrator_proto_depIdxs = []int32{ - 0, // 0: uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator.Ping:input_type -> uber.submitqueue.stovepipe.orchestrator.PingRequest - 1, // 1: uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator.Ping:output_type -> uber.submitqueue.stovepipe.orchestrator.PingResponse +var file_stovepipe_proto_depIdxs = []int32{ + 0, // 0: uber.submitqueue.stovepipe.Stovepipe.Ping:input_type -> uber.submitqueue.stovepipe.PingRequest + 1, // 1: uber.submitqueue.stovepipe.Stovepipe.Ping:output_type -> uber.submitqueue.stovepipe.PingResponse 1, // [1:2] is the sub-list for method output_type 0, // [0:1] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name @@ -198,26 +198,26 @@ var file_orchestrator_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for field type_name } -func init() { file_orchestrator_proto_init() } -func file_orchestrator_proto_init() { - if File_orchestrator_proto != nil { +func init() { file_stovepipe_proto_init() } +func file_stovepipe_proto_init() { + if File_stovepipe_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_orchestrator_proto_rawDesc), len(file_orchestrator_proto_rawDesc)), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_stovepipe_proto_rawDesc), len(file_stovepipe_proto_rawDesc)), NumEnums: 0, NumMessages: 2, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_orchestrator_proto_goTypes, - DependencyIndexes: file_orchestrator_proto_depIdxs, - MessageInfos: file_orchestrator_proto_msgTypes, + GoTypes: file_stovepipe_proto_goTypes, + DependencyIndexes: file_stovepipe_proto_depIdxs, + MessageInfos: file_stovepipe_proto_msgTypes, }.Build() - File_orchestrator_proto = out.File - file_orchestrator_proto_goTypes = nil - file_orchestrator_proto_depIdxs = nil + File_stovepipe_proto = out.File + file_stovepipe_proto_goTypes = nil + file_stovepipe_proto_depIdxs = nil } diff --git a/api/stovepipe/protopb/stovepipe.pb.yarpc.go b/api/stovepipe/protopb/stovepipe.pb.yarpc.go new file mode 100644 index 00000000..6731b089 --- /dev/null +++ b/api/stovepipe/protopb/stovepipe.pb.yarpc.go @@ -0,0 +1,254 @@ +// Code generated by protoc-gen-yarpc-go. DO NOT EDIT. +// source: stovepipe.proto + +package protopb + +import ( + "context" + "io/ioutil" + "reflect" + + "go.uber.org/fx" + "go.uber.org/yarpc" + "go.uber.org/yarpc/api/transport" + "go.uber.org/yarpc/api/x/restriction" + "go.uber.org/yarpc/encoding/protobuf/reflection" + v2 "go.uber.org/yarpc/encoding/protobuf/v2" + "google.golang.org/protobuf/proto" +) + +var _ = ioutil.NopCloser + +// StovepipeYARPCClient is the YARPC client-side interface for the Stovepipe service. +type StovepipeYARPCClient interface { + Ping(context.Context, *PingRequest, ...yarpc.CallOption) (*PingResponse, error) +} + +func newStovepipeYARPCClient(clientConfig transport.ClientConfig, anyResolver v2.AnyResolver, options ...v2.ClientOption) StovepipeYARPCClient { + return &_StovepipeYARPCCaller{v2.NewStreamClient( + v2.ClientParams{ + ServiceName: "uber.submitqueue.stovepipe.Stovepipe", + ClientConfig: clientConfig, + AnyResolver: anyResolver, + Options: options, + }, + )} +} + +// NewStovepipeYARPCClient builds a new YARPC client for the Stovepipe service. +func NewStovepipeYARPCClient(clientConfig transport.ClientConfig, options ...v2.ClientOption) StovepipeYARPCClient { + return newStovepipeYARPCClient(clientConfig, nil, options...) +} + +// StovepipeYARPCServer is the YARPC server-side interface for the Stovepipe service. +type StovepipeYARPCServer interface { + Ping(context.Context, *PingRequest) (*PingResponse, error) +} + +type buildStovepipeYARPCProceduresParams struct { + Server StovepipeYARPCServer + AnyResolver v2.AnyResolver +} + +func buildStovepipeYARPCProcedures(params buildStovepipeYARPCProceduresParams) []transport.Procedure { + handler := &_StovepipeYARPCHandler{params.Server} + return v2.BuildProcedures( + v2.BuildProceduresParams{ + ServiceName: "uber.submitqueue.stovepipe.Stovepipe", + UnaryHandlerParams: []v2.BuildProceduresUnaryHandlerParams{ + { + MethodName: "Ping", + Handler: v2.NewUnaryHandler( + v2.UnaryHandlerParams{ + Handle: handler.Ping, + NewRequest: newStovepipeServicePingYARPCRequest, + AnyResolver: params.AnyResolver, + }, + ), + }, + }, + OnewayHandlerParams: []v2.BuildProceduresOnewayHandlerParams{}, + StreamHandlerParams: []v2.BuildProceduresStreamHandlerParams{}, + }, + ) +} + +// BuildStovepipeYARPCProcedures prepares an implementation of the Stovepipe service for YARPC registration. +func BuildStovepipeYARPCProcedures(server StovepipeYARPCServer) []transport.Procedure { + return buildStovepipeYARPCProcedures(buildStovepipeYARPCProceduresParams{Server: server}) +} + +// FxStovepipeYARPCClientParams defines the input +// for NewFxStovepipeYARPCClient. It provides the +// paramaters to get a StovepipeYARPCClient in an +// Fx application. +type FxStovepipeYARPCClientParams struct { + fx.In + + Provider yarpc.ClientConfig + AnyResolver v2.AnyResolver `name:"yarpcfx" optional:"true"` + Restriction restriction.Checker `optional:"true"` +} + +// FxStovepipeYARPCClientResult defines the output +// of NewFxStovepipeYARPCClient. It provides a +// StovepipeYARPCClient to an Fx application. +type FxStovepipeYARPCClientResult struct { + fx.Out + + Client StovepipeYARPCClient + + // We are using an fx.Out struct here instead of just returning a client + // so that we can add more values or add named versions of the client in + // the future without breaking any existing code. +} + +// NewFxStovepipeYARPCClient provides a StovepipeYARPCClient +// to an Fx application using the given name for routing. +// +// fx.Provide( +// protopb.NewFxStovepipeYARPCClient("service-name"), +// ... +// ) +func NewFxStovepipeYARPCClient(name string, options ...v2.ClientOption) interface{} { + return func(params FxStovepipeYARPCClientParams) FxStovepipeYARPCClientResult { + cc := params.Provider.ClientConfig(name) + + if params.Restriction != nil { + if namer, ok := cc.GetUnaryOutbound().(transport.Namer); ok { + if err := params.Restriction.Check(v2.Encoding, namer.TransportName()); err != nil { + panic(err.Error()) + } + } + } + + return FxStovepipeYARPCClientResult{ + Client: newStovepipeYARPCClient(cc, params.AnyResolver, options...), + } + } +} + +// FxStovepipeYARPCProceduresParams defines the input +// for NewFxStovepipeYARPCProcedures. It provides the +// paramaters to get StovepipeYARPCServer procedures in an +// Fx application. +type FxStovepipeYARPCProceduresParams struct { + fx.In + + Server StovepipeYARPCServer + AnyResolver v2.AnyResolver `name:"yarpcfx" optional:"true"` +} + +// FxStovepipeYARPCProceduresResult defines the output +// of NewFxStovepipeYARPCProcedures. It provides +// StovepipeYARPCServer procedures to an Fx application. +// +// The procedures are provided to the "yarpcfx" value group. +// Dig 1.2 or newer must be used for this feature to work. +type FxStovepipeYARPCProceduresResult struct { + fx.Out + + Procedures []transport.Procedure `group:"yarpcfx"` + ReflectionMeta reflection.ServerMeta `group:"yarpcfx"` +} + +// NewFxStovepipeYARPCProcedures provides StovepipeYARPCServer procedures to an Fx application. +// It expects a StovepipeYARPCServer to be present in the container. +// +// fx.Provide( +// protopb.NewFxStovepipeYARPCProcedures(), +// ... +// ) +func NewFxStovepipeYARPCProcedures() interface{} { + return func(params FxStovepipeYARPCProceduresParams) FxStovepipeYARPCProceduresResult { + return FxStovepipeYARPCProceduresResult{ + Procedures: buildStovepipeYARPCProcedures(buildStovepipeYARPCProceduresParams{ + Server: params.Server, + AnyResolver: params.AnyResolver, + }), + ReflectionMeta: reflection.ServerMeta{ + ServiceName: "uber.submitqueue.stovepipe.Stovepipe", + FileDescriptors: yarpcFileDescriptorClosurefabdb6b3c0b09022, + }, + } + } +} + +type _StovepipeYARPCCaller struct { + streamClient v2.StreamClient +} + +func (c *_StovepipeYARPCCaller) Ping(ctx context.Context, request *PingRequest, options ...yarpc.CallOption) (*PingResponse, error) { + responseMessage, err := c.streamClient.Call(ctx, "Ping", request, newStovepipeServicePingYARPCResponse, options...) + if responseMessage == nil { + return nil, err + } + response, ok := responseMessage.(*PingResponse) + if !ok { + return nil, v2.CastError(emptyStovepipeServicePingYARPCResponse, responseMessage) + } + return response, err +} + +type _StovepipeYARPCHandler struct { + server StovepipeYARPCServer +} + +func (h *_StovepipeYARPCHandler) Ping(ctx context.Context, requestMessage proto.Message) (proto.Message, error) { + var request *PingRequest + var ok bool + if requestMessage != nil { + request, ok = requestMessage.(*PingRequest) + if !ok { + return nil, v2.CastError(emptyStovepipeServicePingYARPCRequest, requestMessage) + } + } + response, err := h.server.Ping(ctx, request) + if response == nil { + return nil, err + } + return response, err +} + +func newStovepipeServicePingYARPCRequest() proto.Message { + return &PingRequest{} +} + +func newStovepipeServicePingYARPCResponse() proto.Message { + return &PingResponse{} +} + +var ( + emptyStovepipeServicePingYARPCRequest = &PingRequest{} + emptyStovepipeServicePingYARPCResponse = &PingResponse{} +) + +var yarpcFileDescriptorClosurefabdb6b3c0b09022 = [][]byte{ + // stovepipe.proto + []byte{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x91, 0xb1, 0x4e, 0xc3, 0x30, + 0x10, 0x86, 0x31, 0xad, 0x80, 0x5e, 0x2b, 0x90, 0x3c, 0x45, 0x11, 0x42, 0x25, 0x4b, 0x33, 0xd9, + 0x02, 0xde, 0xa0, 0x0f, 0x80, 0xa2, 0xb0, 0xc1, 0x80, 0xec, 0xe8, 0x94, 0x78, 0x70, 0xec, 0xe6, + 0xec, 0xbe, 0x01, 0xef, 0x8d, 0xe2, 0xd2, 0xd0, 0xa5, 0xb0, 0xf9, 0xce, 0xdf, 0x27, 0xfd, 0xbf, + 0x0d, 0x77, 0x14, 0xdc, 0x1e, 0xbd, 0xf1, 0x28, 0xfc, 0xe0, 0x82, 0xe3, 0x79, 0xd4, 0x38, 0x08, + 0x8a, 0xda, 0x9a, 0xb0, 0x8b, 0x18, 0x51, 0x4c, 0x44, 0xb1, 0x81, 0x65, 0x65, 0xfa, 0xb6, 0xc6, + 0x5d, 0x44, 0x0a, 0x3c, 0x83, 0x6b, 0x8b, 0x44, 0xaa, 0xc5, 0x8c, 0xad, 0x59, 0xb9, 0xa8, 0x8f, + 0x63, 0xf1, 0xc5, 0x60, 0x75, 0x20, 0xc9, 0xbb, 0x9e, 0xf0, 0x3c, 0xca, 0x1f, 0x61, 0x45, 0x38, + 0xec, 0x4d, 0x83, 0x9f, 0xbd, 0xb2, 0x98, 0x5d, 0xa6, 0xeb, 0xe5, 0xcf, 0xee, 0x55, 0x59, 0xe4, + 0xf7, 0xb0, 0x08, 0xc6, 0x22, 0x05, 0x65, 0x7d, 0x36, 0x5b, 0xb3, 0x72, 0x56, 0xff, 0x2e, 0x78, + 0x0e, 0x37, 0x9d, 0xa3, 0x90, 0xe4, 0x79, 0x92, 0xa7, 0xf9, 0xb9, 0x83, 0xc5, 0xdb, 0x31, 0x3d, + 0xff, 0x80, 0xf9, 0x98, 0x89, 0x6f, 0xc4, 0xf9, 0x8a, 0xe2, 0xa4, 0x5f, 0x5e, 0xfe, 0x0f, 0x1e, + 0xea, 0x15, 0x17, 0x5b, 0x84, 0x87, 0xc6, 0xd9, 0x3f, 0x84, 0xed, 0xed, 0x94, 0xa4, 0x1a, 0x1f, + 0xba, 0x62, 0xef, 0x4f, 0xad, 0x09, 0x5d, 0xd4, 0xa2, 0x71, 0x56, 0x8e, 0xa2, 0x3c, 0x11, 0xa5, + 0xf2, 0x46, 0x4e, 0xb2, 0x4c, 0x7f, 0xe3, 0xb5, 0xbe, 0x4a, 0x87, 0x97, 0xef, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xb6, 0x5b, 0x9f, 0x24, 0xb7, 0x01, 0x00, 0x00, + }, +} + +func init() { + yarpc.RegisterClientBuilder( + func(clientConfig transport.ClientConfig, structField reflect.StructField) StovepipeYARPCClient { + return NewStovepipeYARPCClient(clientConfig, v2.ClientBuilderOptions(clientConfig, structField)...) + }, + ) +} diff --git a/api/stovepipe/orchestrator/protopb/orchestrator_grpc.pb.go b/api/stovepipe/protopb/stovepipe_grpc.pb.go similarity index 53% rename from api/stovepipe/orchestrator/protopb/orchestrator_grpc.pb.go rename to api/stovepipe/protopb/stovepipe_grpc.pb.go index af7aec99..3cf25bc3 100644 --- a/api/stovepipe/orchestrator/protopb/orchestrator_grpc.pb.go +++ b/api/stovepipe/protopb/stovepipe_grpc.pb.go @@ -16,7 +16,7 @@ // versions: // - protoc-gen-go-grpc v1.5.1 // - protoc v5.29.3 -// source: orchestrator.proto +// source: stovepipe.proto package protopb @@ -34,109 +34,109 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - StovepipeOrchestrator_Ping_FullMethodName = "/uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator/Ping" + Stovepipe_Ping_FullMethodName = "/uber.submitqueue.stovepipe.Stovepipe/Ping" ) -// StovepipeOrchestratorClient is the client API for StovepipeOrchestrator service. +// StovepipeClient is the client API for Stovepipe service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// StovepipeOrchestrator provides the Stovepipe orchestrator API. -type StovepipeOrchestratorClient interface { +// Stovepipe provides the Stovepipe API. +type StovepipeClient interface { // Ping returns a response indicating the service is alive Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) } -type stovepipeOrchestratorClient struct { +type stovepipeClient struct { cc grpc.ClientConnInterface } -func NewStovepipeOrchestratorClient(cc grpc.ClientConnInterface) StovepipeOrchestratorClient { - return &stovepipeOrchestratorClient{cc} +func NewStovepipeClient(cc grpc.ClientConnInterface) StovepipeClient { + return &stovepipeClient{cc} } -func (c *stovepipeOrchestratorClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { +func (c *stovepipeClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PingResponse) - err := c.cc.Invoke(ctx, StovepipeOrchestrator_Ping_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, Stovepipe_Ping_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } -// StovepipeOrchestratorServer is the server API for StovepipeOrchestrator service. -// All implementations must embed UnimplementedStovepipeOrchestratorServer +// StovepipeServer is the server API for Stovepipe service. +// All implementations must embed UnimplementedStovepipeServer // for forward compatibility. // -// StovepipeOrchestrator provides the Stovepipe orchestrator API. -type StovepipeOrchestratorServer interface { +// Stovepipe provides the Stovepipe API. +type StovepipeServer interface { // Ping returns a response indicating the service is alive Ping(context.Context, *PingRequest) (*PingResponse, error) - mustEmbedUnimplementedStovepipeOrchestratorServer() + mustEmbedUnimplementedStovepipeServer() } -// UnimplementedStovepipeOrchestratorServer must be embedded to have +// UnimplementedStovepipeServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. -type UnimplementedStovepipeOrchestratorServer struct{} +type UnimplementedStovepipeServer struct{} -func (UnimplementedStovepipeOrchestratorServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { +func (UnimplementedStovepipeServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") } -func (UnimplementedStovepipeOrchestratorServer) mustEmbedUnimplementedStovepipeOrchestratorServer() {} -func (UnimplementedStovepipeOrchestratorServer) testEmbeddedByValue() {} +func (UnimplementedStovepipeServer) mustEmbedUnimplementedStovepipeServer() {} +func (UnimplementedStovepipeServer) testEmbeddedByValue() {} -// UnsafeStovepipeOrchestratorServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to StovepipeOrchestratorServer will +// UnsafeStovepipeServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to StovepipeServer will // result in compilation errors. -type UnsafeStovepipeOrchestratorServer interface { - mustEmbedUnimplementedStovepipeOrchestratorServer() +type UnsafeStovepipeServer interface { + mustEmbedUnimplementedStovepipeServer() } -func RegisterStovepipeOrchestratorServer(s grpc.ServiceRegistrar, srv StovepipeOrchestratorServer) { - // If the following call pancis, it indicates UnimplementedStovepipeOrchestratorServer was +func RegisterStovepipeServer(s grpc.ServiceRegistrar, srv StovepipeServer) { + // If the following call pancis, it indicates UnimplementedStovepipeServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } - s.RegisterService(&StovepipeOrchestrator_ServiceDesc, srv) + s.RegisterService(&Stovepipe_ServiceDesc, srv) } -func _StovepipeOrchestrator_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _Stovepipe_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PingRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(StovepipeOrchestratorServer).Ping(ctx, in) + return srv.(StovepipeServer).Ping(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: StovepipeOrchestrator_Ping_FullMethodName, + FullMethod: Stovepipe_Ping_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(StovepipeOrchestratorServer).Ping(ctx, req.(*PingRequest)) + return srv.(StovepipeServer).Ping(ctx, req.(*PingRequest)) } return interceptor(ctx, in, info, handler) } -// StovepipeOrchestrator_ServiceDesc is the grpc.ServiceDesc for StovepipeOrchestrator service. +// Stovepipe_ServiceDesc is the grpc.ServiceDesc for Stovepipe service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) -var StovepipeOrchestrator_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator", - HandlerType: (*StovepipeOrchestratorServer)(nil), +var Stovepipe_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "uber.submitqueue.stovepipe.Stovepipe", + HandlerType: (*StovepipeServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Ping", - Handler: _StovepipeOrchestrator_Ping_Handler, + Handler: _Stovepipe_Ping_Handler, }, }, Streams: []grpc.StreamDesc{}, - Metadata: "orchestrator.proto", + Metadata: "stovepipe.proto", } diff --git a/doc/howto/TESTING.md b/doc/howto/TESTING.md index 79367837..0d051197 100644 --- a/doc/howto/TESTING.md +++ b/doc/howto/TESTING.md @@ -118,7 +118,7 @@ sq-test-{context}-{shortid} ### Context naming convention -The `{context}` passed to `NewComposeStack` is **domain-qualified** so that the same kind of suite in different domains (SubmitQueue and Stovepipe both have a gateway and an orchestrator) yields distinct, self-describing names: +The `{context}` passed to `NewComposeStack` is **domain-qualified** so that the same kind of suite in different domains yields distinct, self-describing names: ``` {category}-{domain}-{name} @@ -136,7 +136,7 @@ Shared (cross-domain) suites carry no domain segment — e.g. the shared queue e |-------|---------|-------------------| | SubmitQueue gateway | `svc-submitqueue-gateway` | `sq-test-svc-submitqueue-gateway-abc123-gateway-service-1` | | SubmitQueue orchestrator | `svc-submitqueue-orchestrator` | `sq-test-svc-submitqueue-orchestrator-xyz789-orchestrator-service-1` | -| Stovepipe gateway | `svc-stovepipe-gateway` | `sq-test-svc-stovepipe-gateway-abc123-gateway-service-1` | +| Stovepipe | `svc-stovepipe` | `sq-test-svc-stovepipe-abc123-stovepipe-service-1` | | SubmitQueue storage extension | `ext-submitqueue-storage-mysql` | `sq-test-ext-submitqueue-storage-mysql-2ce1d0-mysql-1` | | Counter extension (shared) | `ext-counter-mysql` | `sq-test-ext-counter-mysql-…-mysql-1` | | SubmitQueue changestore extension | `ext-submitqueue-changestore-mysql` | `sq-test-ext-submitqueue-changestore-mysql-…-mysql-1` | diff --git a/example/README.md b/example/README.md index 4b6d4879..106a6386 100644 --- a/example/README.md +++ b/example/README.md @@ -6,8 +6,7 @@ Example gRPC servers and clients for running the submitqueue services locally. - **SubmitQueue Gateway** (port 8081) — entry point for land requests. Exposes `Ping` and `Land` RPCs. - **SubmitQueue Orchestrator** (port 8082) — coordinates the pipeline. Exposes `Ping` RPC and consumes queue messages across 9 pipeline topics. -- **Stovepipe Gateway** (port 8083) - entry point for commit deployment verification requests. Exposes `Ping` RPC. -- **Stovepipe Orchestrator** (port 8084) - coordinates the commit verification pipeline. Exposes `Ping` RPC. +- **Stovepipe** (port 8083) - single Ping-only service. Exposes `Ping` RPC. Services require MySQL (app database + queue database). Docker Compose handles this automatically. @@ -30,19 +29,11 @@ example/ │ │ └── docker-compose.yml # Orchestrator-only stack │ └── client/main.go # Orchestrator ping client └── stovepipe/ - ├── docker-compose.yml # Full stack (Stovepipe Gateway + Orchestrator + 2x MySQL) - ├── gateway/ - │ ├── server/ - │ │ ├── main.go # Stovepipe gateway gRPC server (Docker: :8080; go run default :8083) - │ │ ├── Dockerfile - │ │ └── docker-compose.yml # Gateway-only stack - │ └── client/main.go # Stovepipe gateway ping client - └── orchestrator/ - ├── server/ - │ ├── main.go # Stovepipe orchestrator gRPC server (Docker: :8080; go run default :8084) - │ ├── Dockerfile - │ └── docker-compose.yml # Orchestrator-only stack - └── client/main.go # Stovepipe orchestrator ping client + ├── docker-compose.yml # Single Ping-only service stack + ├── server/ + │ ├── main.go # Stovepipe gRPC server (Docker: :8080; go run default :8083) + │ └── Dockerfile + └── client/main.go # Stovepipe ping client ``` ## Running @@ -56,10 +47,8 @@ make local-submitqueue-start make local-submitqueue-gateway-start make local-submitqueue-orchestrator-start -# Start full Stovepipe stack (Gateway + Orchestrator + MySQL) +# Start Stovepipe service (single Ping-only gRPC service) make local-stovepipe-start -make local-stovepipe-gateway-start -make local-stovepipe-orchestrator-start # View logs and status make local-submitqueue-logs @@ -69,7 +58,7 @@ make local-submitqueue-ps make local-stop ``` -For Docker, `make build-stovepipe-gateway-linux` copies a Linux binary to `.docker-bin/stovepipe-gateway` so it does not overwrite SubmitQueue’s `.docker-bin/gateway`. Stovepipe `make local-stovepipe-gateway-start` applies **only the queue schema** on `mysql-queue` (`make local-init-stovepipe-queue-schema`); SubmitQueue storage/counter schemas on `mysql-app` are skipped until Stovepipe has its own app schema. Compose service keys are **`gateway-service`** and **`orchestrator-service`** (same as `example/submitqueue`), so with default project **`stovepipe`** you should see **`stovepipe-gateway-service-1`** and **`stovepipe-orchestrator-service-1`** — not `stovepipe-stovepipe-*` (that pattern was from older service names; run **`make local-stop`** to run `docker compose down --remove-orphans` and drop orphans). `make local-stop` also stops the SubmitQueue stack. SubmitQueue examples use project **`submitqueue`** by default (`make SUBMITQUEUE_LOCAL_PROJECT=myname ...`). Stovepipe containers are named like `stovepipe-mysql-app-1`, not `submitqueue-*`. +For Docker, `make build-stovepipe-linux` copies a Linux binary to `.docker-bin/stovepipe` so it does not overwrite SubmitQueue’s `.docker-bin/gateway`. Stovepipe is currently a single Ping-only service with no storage or queue dependencies, so the compose stack has no MySQL. The compose service key is **`stovepipe-service`**, so with default project **`stovepipe`** you should see **`stovepipe-stovepipe-service-1`**. `make local-stop` stops the SubmitQueue, Stovepipe, and Runway stacks; `make local-stovepipe-stop` stops only Stovepipe. ### Bazel @@ -77,14 +66,12 @@ For Docker, `make build-stovepipe-gateway-linux` copies a Linux binary to `.dock # Build servers bazel build //example/submitqueue/gateway/server:gateway bazel build //example/submitqueue/orchestrator/server:orchestrator -bazel build //example/stovepipe/gateway/server:gateway -bazel build //example/stovepipe/orchestrator/server:orchestrator +bazel build //example/stovepipe/server:stovepipe # Build clients bazel build //example/submitqueue/gateway/client:gateway bazel build //example/submitqueue/orchestrator/client:orchestrator -bazel build //example/stovepipe/gateway/client:gateway -bazel build //example/stovepipe/orchestrator/client:orchestrator +bazel build //example/stovepipe/client:stovepipe ``` ### Go @@ -92,8 +79,7 @@ bazel build //example/stovepipe/orchestrator/client:orchestrator ```bash go run example/submitqueue/gateway/server/main.go go run example/submitqueue/orchestrator/server/main.go -go run example/stovepipe/gateway/server/main.go -go run example/stovepipe/orchestrator/server/main.go +go run example/stovepipe/server/main.go ``` ## Testing with Clients @@ -102,8 +88,7 @@ go run example/stovepipe/orchestrator/server/main.go # Go clients go run example/submitqueue/gateway/client/main.go -addr localhost:8081 -message "hello" go run example/submitqueue/orchestrator/client/main.go -addr localhost:8082 -message "hello" -go run example/stovepipe/gateway/client/main.go -addr localhost:8083 -message "hello" -go run example/stovepipe/orchestrator/client/main.go -addr localhost:8084 -message "hello" +go run example/stovepipe/client/main.go -addr localhost:8083 -message "hello" ``` Client flags: @@ -124,20 +109,17 @@ go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest # Ping grpcurl -plaintext -d '{"message": "hello"}' localhost:8081 uber.submitqueue.gateway.SubmitQueueGateway/Ping grpcurl -plaintext -d '{"message": "hello"}' localhost:8082 uber.submitqueue.orchestrator.SubmitQueueOrchestrator/Ping -grpcurl -plaintext -d '{"message": "hello"}' localhost:8083 uber.submitqueue.stovepipe.StovepipeGateway/Ping -grpcurl -plaintext -d '{"message": "hello"}' localhost:8084 uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator/Ping +grpcurl -plaintext -d '{"message": "hello"}' localhost:8083 uber.submitqueue.stovepipe.Stovepipe/Ping # List services grpcurl -plaintext localhost:8081 list grpcurl -plaintext localhost:8082 list grpcurl -plaintext localhost:8083 list -grpcurl -plaintext localhost:8084 list # Describe a service grpcurl -plaintext localhost:8081 describe uber.submitqueue.gateway.SubmitQueueGateway grpcurl -plaintext localhost:8082 describe uber.submitqueue.orchestrator.SubmitQueueOrchestrator -grpcurl -plaintext localhost:8083 describe uber.submitqueue.stovepipe.StovepipeGateway -grpcurl -plaintext localhost:8084 describe uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator +grpcurl -plaintext localhost:8083 describe uber.submitqueue.stovepipe.Stovepipe ``` ## API Reference @@ -161,19 +143,10 @@ grpcurl -plaintext localhost:8084 describe uber.submitqueue.stovepipe.orchestrat |--------|-------------| | `Ping` | Health check, returns service name and timestamp | -### Stovepipe Gateway +### Stovepipe -**Service**: `uber.submitqueue.stovepipe.StovepipeGateway` -**Proto**: `api/stovepipe/gateway/proto/gateway.proto` - -| Method | Description | -|--------|-------------| -| `Ping` | Health check | - -### Stovepipe Orchestrator - -**Service**: `uber.submitqueue.stovepipe.orchestrator.StovepipeOrchestrator` -**Proto**: `api/stovepipe/orchestrator/proto/orchestrator.proto` +**Service**: `uber.submitqueue.stovepipe.Stovepipe` +**Proto**: `api/stovepipe/proto/stovepipe.proto` | Method | Description | |--------|-------------| diff --git a/example/stovepipe/gateway/client/BUILD.bazel b/example/stovepipe/client/BUILD.bazel similarity index 73% rename from example/stovepipe/gateway/client/BUILD.bazel rename to example/stovepipe/client/BUILD.bazel index 9912cc20..2adf44c4 100644 --- a/example/stovepipe/gateway/client/BUILD.bazel +++ b/example/stovepipe/client/BUILD.bazel @@ -1,19 +1,19 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( - name = "gateway_lib", + name = "client_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/stovepipe/gateway/client", + importpath = "github.com/uber/submitqueue/example/stovepipe/client", visibility = ["//visibility:private"], deps = [ - "//api/stovepipe/gateway/protopb", + "//api/stovepipe/protopb", "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//credentials/insecure", ], ) go_binary( - name = "gateway", - embed = [":gateway_lib"], + name = "stovepipe", + embed = [":client_lib"], visibility = ["//visibility:public"], ) diff --git a/example/stovepipe/gateway/client/main.go b/example/stovepipe/client/main.go similarity index 88% rename from example/stovepipe/gateway/client/main.go rename to example/stovepipe/client/main.go index 0fdd4da7..ed79956e 100644 --- a/example/stovepipe/gateway/client/main.go +++ b/example/stovepipe/client/main.go @@ -21,13 +21,13 @@ import ( "os" "time" - pb "github.com/uber/submitqueue/api/stovepipe/gateway/protopb" + pb "github.com/uber/submitqueue/api/stovepipe/protopb" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) func main() { - addr := flag.String("addr", "localhost:8083", "stovepipe gateway server address") + addr := flag.String("addr", "localhost:8083", "stovepipe server address") message := flag.String("message", "", "message to send in ping request") timeout := flag.Duration("timeout", 5*time.Second, "request timeout") flag.Parse() @@ -50,7 +50,7 @@ func run(addr, message string, timeout time.Duration) error { defer conn.Close() // Create a client - client := pb.NewStovepipeGatewayClient(conn) + client := pb.NewStovepipeClient(conn) // Create context with timeout ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -61,7 +61,7 @@ func run(addr, message string, timeout time.Duration) error { Message: message, } - fmt.Printf("Sending ping to stovepipe gateway at %s...\n", addr) + fmt.Printf("Sending ping to stovepipe at %s...\n", addr) resp, err := client.Ping(ctx, req) if err != nil { return fmt.Errorf("ping failed: %w", err) diff --git a/example/stovepipe/docker-compose.yml b/example/stovepipe/docker-compose.yml index 4952f914..aa3866d1 100644 --- a/example/stovepipe/docker-compose.yml +++ b/example/stovepipe/docker-compose.yml @@ -1,88 +1,24 @@ -# Docker Compose for full Stovepipe stack (Gateway + Orchestrator + MySQL) +# Docker Compose for the Stovepipe service. # -# Use with `make local-stovepipe-start`; use per-service compose files -# under example/stovepipe/gateway/server/ or example/stovepipe/orchestrator/server/ -# for single-service stacks. +# Stovepipe is currently a single Ping-only gRPC service with no storage or +# queue dependencies, so this stack runs just the one service. # -# IMPORTANT: Before running compose, build the Linux binaries: -# make build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux +# IMPORTANT: Before running compose, build the Linux binary: +# make build-stovepipe-linux # OR # bazel build --platforms=@rules_go//go/toolchain:linux_amd64 \ -# //example/stovepipe/gateway/server:gateway \ -# //example/stovepipe/orchestrator/server:orchestrator +# //example/stovepipe/server:stovepipe # # Quick start: # make local-stovepipe-start # services: - # Application Database - Stores business data (requests, counters, etc.) - mysql-app: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: submitqueue - ports: - - "3306" # Random ephemeral port to avoid conflicts - healthcheck: - # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats - # "localhost" as a socket connection, which can be ready before the TCP - # listener — causing dependent services that connect over TCP to fail. - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] - interval: 5s - timeout: 5s - retries: 10 - - # Queue Database - Messaging infrastructure (messages, offsets, partition leases) - # Separate from app DB to demonstrate queue is pluggable infrastructure - mysql-queue: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: submitqueue - ports: - - "3306" # Random ephemeral port to avoid conflicts - healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] - interval: 5s - timeout: 5s - retries: 10 - - gateway-service: - build: - context: ${REPO_ROOT} - dockerfile: example/stovepipe/gateway/server/Dockerfile - ports: - - "8080" # Random ephemeral port to avoid conflicts - environment: - - PORT=:8080 - # Application database connection - - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true - # Queue infrastructure connection (separate database) - - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true - # Path to YAML queue configuration baked into the image - - QUEUE_CONFIG_PATH=/root/queues.yaml - depends_on: - mysql-app: - condition: service_healthy - mysql-queue: - condition: service_healthy - - orchestrator-service: + stovepipe-service: build: context: ${REPO_ROOT} - dockerfile: example/stovepipe/orchestrator/server/Dockerfile + dockerfile: example/stovepipe/server/Dockerfile ports: - "8080" # Random ephemeral port to avoid conflicts environment: - PORT=:8080 - # Application database connection (for request state, batches, etc.) - - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true - # Queue infrastructure connection (separate database) - - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true - - HOSTNAME=orchestrator-dev - depends_on: - mysql-app: - condition: service_healthy - mysql-queue: - condition: service_healthy diff --git a/example/stovepipe/gateway/server/Dockerfile b/example/stovepipe/gateway/server/Dockerfile deleted file mode 100644 index b00717a6..00000000 --- a/example/stovepipe/gateway/server/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM debian:bookworm-slim - -RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* -WORKDIR /root/ - -# Copy pre-built Linux binary (host path is stovepipe-* to avoid clobbering main .docker-bin/gateway). -# Built via: make build-stovepipe-gateway-linux -COPY .docker-bin/stovepipe-gateway ./gateway - -# Stovepipe's own sample queue YAML; compose sets QUEUE_CONFIG_PATH to point at it. -COPY example/stovepipe/gateway/server/queues.yaml ./queues.yaml - -EXPOSE 8080 - -CMD ["./gateway"] diff --git a/example/stovepipe/gateway/server/docker-compose.yml b/example/stovepipe/gateway/server/docker-compose.yml deleted file mode 100644 index 59660f17..00000000 --- a/example/stovepipe/gateway/server/docker-compose.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Docker Compose for Stovepipe gateway manual testing -# -# Mirrors example/submitqueue/gateway/server/docker-compose.yml: same MySQL pair, healthchecks, -# env wiring, and startup ordering. The Stovepipe gateway binary is Ping-only today -# and does not open MySQL yet; variables are set so future work matches SubmitQueue. -# -# IMPORTANT: Before running compose, build the Linux binary: -# make build-stovepipe-gateway-linux -# OR -# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/stovepipe/gateway/server:gateway -# -# Quick start: -# make local-stovepipe-gateway-start -# -# After `up`, only the queue schema is applied (`local-init-stovepipe-queue-schema`). - -services: - # Application Database - Stores business data (requests, counters, etc.) - mysql-app: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: submitqueue - ports: - - "3306" # Random ephemeral port to avoid conflicts - healthcheck: - # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats - # "localhost" as a socket connection, which can be ready before the TCP - # listener — causing dependent services that connect over TCP to fail. - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] - interval: 5s - timeout: 5s - retries: 10 - - # Queue Database - Messaging infrastructure (messages, offsets, partition leases) - # Separate from app DB to demonstrate queue is pluggable infrastructure - mysql-queue: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: submitqueue - ports: - - "3306" # Random ephemeral port to avoid conflicts - healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] - interval: 5s - timeout: 5s - retries: 10 - - gateway-service: - build: - context: ${REPO_ROOT} - dockerfile: example/stovepipe/gateway/server/Dockerfile - ports: - - "8080" # Random ephemeral port to avoid conflicts - environment: - - PORT=:8080 - # Application database connection - - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true - # Queue infrastructure connection (separate database) - - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true - # Path to YAML queue configuration baked into the image - - QUEUE_CONFIG_PATH=/root/queues.yaml - depends_on: - mysql-app: - condition: service_healthy - mysql-queue: - condition: service_healthy diff --git a/example/stovepipe/gateway/server/queues.yaml b/example/stovepipe/gateway/server/queues.yaml deleted file mode 100644 index 09925a79..00000000 --- a/example/stovepipe/gateway/server/queues.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Stovepipe queue configuration placeholder. -# -# The queue-config schema (build_runner / change_provider / merge_checker / -# land_provider) is SubmitQueue-specific — those fields select SubmitQueue -# extension implementations that Stovepipe does not yet define. The Stovepipe -# gateway image ships this file only so QUEUE_CONFIG_PATH resolves; populate it -# once Stovepipe defines its own queue-config extensions. -queues: [] diff --git a/example/stovepipe/orchestrator/client/BUILD.bazel b/example/stovepipe/orchestrator/client/BUILD.bazel deleted file mode 100644 index 3524b678..00000000 --- a/example/stovepipe/orchestrator/client/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "orchestrator_lib", - srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/stovepipe/orchestrator/client", - visibility = ["//visibility:private"], - deps = [ - "//api/stovepipe/orchestrator/protopb", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//credentials/insecure", - ], -) - -go_binary( - name = "orchestrator", - embed = [":orchestrator_lib"], - visibility = ["//visibility:public"], -) diff --git a/example/stovepipe/orchestrator/client/main.go b/example/stovepipe/orchestrator/client/main.go deleted file mode 100644 index 5513f96a..00000000 --- a/example/stovepipe/orchestrator/client/main.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "flag" - "fmt" - "os" - "time" - - pb "github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -func main() { - addr := flag.String("addr", "localhost:8084", "stovepipe orchestrator server address") - message := flag.String("message", "", "message to send in ping request") - timeout := flag.Duration("timeout", 5*time.Second, "request timeout") - flag.Parse() - - if err := run(*addr, *message, *timeout); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func run(addr, message string, timeout time.Duration) error { - conn, err := grpc.NewClient( - addr, - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - return fmt.Errorf("failed to connect: %w", err) - } - defer conn.Close() - - client := pb.NewStovepipeOrchestratorClient(conn) - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - req := &pb.PingRequest{ - Message: message, - } - - fmt.Printf("Sending ping to stovepipe orchestrator at %s...\n", addr) - resp, err := client.Ping(ctx, req) - if err != nil { - return fmt.Errorf("ping failed: %w", err) - } - - fmt.Printf("\nResponse:\n") - fmt.Printf(" Message: %s\n", resp.Message) - fmt.Printf(" Service Name: %s\n", resp.ServiceName) - fmt.Printf(" Timestamp: %d (%s)\n", resp.Timestamp, time.Unix(resp.Timestamp, 0)) - fmt.Printf(" Hostname: %s\n", resp.Hostname) - - return nil -} diff --git a/example/stovepipe/orchestrator/server/BUILD.bazel b/example/stovepipe/orchestrator/server/BUILD.bazel deleted file mode 100644 index 420e0319..00000000 --- a/example/stovepipe/orchestrator/server/BUILD.bazel +++ /dev/null @@ -1,37 +0,0 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") - -exports_files( - ["docker-compose.yml"], - visibility = ["//visibility:public"], -) - -go_library( - name = "orchestrator_server_lib", - srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/stovepipe/orchestrator/server", - visibility = ["//visibility:private"], - deps = [ - "//api/stovepipe/orchestrator/protopb", - "//platform/consumer", - "//platform/errs", - "//platform/errs/generic", - "//platform/errs/mysql", - "//platform/extension/messagequeue", - "//platform/extension/messagequeue/mysql", - "//stovepipe/core/topickey", - "//stovepipe/orchestrator/controller", - "//stovepipe/orchestrator/controller/start", - "//stovepipe/orchestrator/controller/validate", - "@com_github_go_sql_driver_mysql//:mysql", - "@com_github_uber_go_tally//:tally", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//reflection", - "@org_uber_go_zap//:zap", - ], -) - -go_binary( - name = "orchestrator", - embed = [":orchestrator_server_lib"], - visibility = ["//visibility:public"], -) diff --git a/example/stovepipe/orchestrator/server/docker-compose.yml b/example/stovepipe/orchestrator/server/docker-compose.yml deleted file mode 100644 index 28331a03..00000000 --- a/example/stovepipe/orchestrator/server/docker-compose.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Docker Compose for Stovepipe orchestrator manual testing -# -# -# IMPORTANT: Before running compose, build the Linux binary: -# make build-stovepipe-orchestrator-linux -# OR -# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/stovepipe/orchestrator/server:orchestrator -# -# Quick start: -# make local-stovepipe-orchestrator-start -# -# After `up`, only the queue schema is applied (`local-init-stovepipe-queue-schema`). - -services: - # Application Database - Stores business data (requests, batches, etc.) - mysql-app: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: submitqueue - ports: - - "3306" # Random ephemeral port to avoid conflicts - healthcheck: - # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats - # "localhost" as a socket connection, which can be ready before the TCP - # listener — causing dependent services that connect over TCP to fail. - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] - interval: 5s - timeout: 5s - retries: 10 - - # Queue Database - Messaging infrastructure (messages, offsets, partition leases) - # Separate from app DB to demonstrate queue is pluggable infrastructure - mysql-queue: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: submitqueue - ports: - - "3306" # Random ephemeral port to avoid conflicts - healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"] - interval: 5s - timeout: 5s - retries: 10 - - orchestrator-service: - build: - context: ${REPO_ROOT} - dockerfile: example/stovepipe/orchestrator/server/Dockerfile - ports: - - "8080" # Random ephemeral port to avoid conflicts - environment: - - PORT=:8080 - # Application database connection (for request state, batches, etc.) - - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true - # Queue infrastructure connection (separate database) - - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true - - HOSTNAME=orchestrator-dev - depends_on: - mysql-app: - condition: service_healthy - mysql-queue: - condition: service_healthy diff --git a/example/stovepipe/orchestrator/server/main.go b/example/stovepipe/orchestrator/server/main.go deleted file mode 100644 index b0e24976..00000000 --- a/example/stovepipe/orchestrator/server/main.go +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "database/sql" - "errors" - "fmt" - "net" - "os" - "os/signal" - "sync" - "syscall" - "time" - - _ "github.com/go-sql-driver/mysql" - "github.com/uber-go/tally" - pb "github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb" - "github.com/uber/submitqueue/platform/consumer" - "github.com/uber/submitqueue/platform/errs" - genericerrs "github.com/uber/submitqueue/platform/errs/generic" - mysqlerrs "github.com/uber/submitqueue/platform/errs/mysql" - extqueue "github.com/uber/submitqueue/platform/extension/messagequeue" - queueMySQL "github.com/uber/submitqueue/platform/extension/messagequeue/mysql" - "github.com/uber/submitqueue/stovepipe/core/topickey" - "github.com/uber/submitqueue/stovepipe/orchestrator/controller" - "github.com/uber/submitqueue/stovepipe/orchestrator/controller/start" - "github.com/uber/submitqueue/stovepipe/orchestrator/controller/validate" - "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/reflection" -) - -// OrchestratorServer wraps the controller and implements the gRPC service interface. -type OrchestratorServer struct { - pb.UnimplementedStovepipeOrchestratorServer - pingController *controller.PingController -} - -// Ping delegates to the controller. -func (s *OrchestratorServer) Ping(ctx context.Context, req *pb.PingRequest) (*pb.PingResponse, error) { - return s.pingController.Ping(ctx, req) -} - -func main() { - code := 0 - if err := run(); err != nil { - if errors.Is(err, context.Canceled) { - fmt.Println("Stovepipe orchestrator server stopped by signal") - - // Return 143 (128 + SIGTERM) as per POSIX standard if the application receives any termination signal from the OS. Ideally we should return 128+SIGINT for SIGINT and 128+SIGTERM for SIGTERM, - // but it will require a special processing not yet available in the standard library. - code = 128 + int(syscall.SIGTERM) - } else { - fmt.Fprintf(os.Stderr, "Stovepipe orchestrator server failure: %v\n", err) - // TODO: classify errors and implement a binary protocol for exit codes, so far 1 for everything - code = 1 - } - } - os.Exit(code) -} - -func run() error { - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer cancel() - - logger, err := zap.NewDevelopment() - if err != nil { - return fmt.Errorf("failed to create logger: %w", err) - } - defer logger.Sync() - - scope := tally.NewTestScope("stovepipe_orchestrator", nil) - metricsStopCh := make(chan interface{}, 1) - metricsWgDone := sync.WaitGroup{} - metricsWgDone.Add(1) - go func() { - defer metricsWgDone.Done() - - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - for { - select { - case <-metricsStopCh: - return - case <-ticker.C: - snapshot := scope.Snapshot() - logger.Info("metrics snapshot", - zap.Any("counters", snapshot.Counters()), - zap.Any("gauges", snapshot.Gauges()), - zap.Any("timers", snapshot.Timers()), - ) - } - } - }() - - defer func() { - close(metricsStopCh) - metricsWgDone.Wait() - }() - - queueDSN := os.Getenv("QUEUE_MYSQL_DSN") - if queueDSN == "" { - return fmt.Errorf("QUEUE_MYSQL_DSN environment variable is required") - } - queueDB, err := sql.Open("mysql", queueDSN) - if err != nil { - return fmt.Errorf("failed to open queue database: %w", err) - } - defer queueDB.Close() - - mysqlQueue, err := queueMySQL.NewQueue(queueMySQL.Params{ - DB: queueDB, - Logger: logger, - MetricsScope: scope.SubScope("queue"), - }) - if err != nil { - return fmt.Errorf("failed to create queue: %w", err) - } - defer mysqlQueue.Close() - - logger.Info("initialized queue", zap.String("dsn", queueDSN)) - - subscriberName := os.Getenv("HOSTNAME") - if subscriberName == "" { - subscriberName = fmt.Sprintf("stovepipe-orchestrator-%d", time.Now().Unix()) - } - - registry, err := newTopicRegistry(mysqlQueue, subscriberName) - if err != nil { - return fmt.Errorf("failed to create topic registry: %w", err) - } - - primaryConsumer := consumer.New(logger.Sugar(), scope.SubScope("consumer"), registry, - errs.NewClassifierProcessor( - genericerrs.Classifier, - mysqlerrs.Classifier, - ), - ) - - startController := start.NewController(start.Params{ - Logger: logger.Sugar(), - Scope: scope, - Registry: registry, - TopicKey: topickey.TopicKeyStart, - ConsumerGroup: "orchestrator-start", - }) - if err := primaryConsumer.Register(startController); err != nil { - return fmt.Errorf("failed to register start controller: %w", err) - } - - validateController := validate.NewController(validate.Params{ - Logger: logger.Sugar(), - Scope: scope, - Registry: registry, - TopicKey: topickey.TopicKeyValidate, - ConsumerGroup: "orchestrator-validate", - }) - if err := primaryConsumer.Register(validateController); err != nil { - return fmt.Errorf("failed to register validate controller: %w", err) - } - logger.Info("controllers registered", zap.Int("primary", 2)) - - if err := primaryConsumer.Start(ctx); err != nil { - return fmt.Errorf("failed to start primary consumer: %w", err) - } - logger.Info("consumer started") - - grpcServer := grpc.NewServer() - - pingController := controller.NewPingController(logger, scope) - srv := &OrchestratorServer{ - pingController: pingController, - } - pb.RegisterStovepipeOrchestratorServer(grpcServer, srv) - - reflection.Register(grpcServer) - - port := os.Getenv("PORT") - if port == "" { - port = ":8084" - } - listener, err := net.Listen("tcp", port) - if err != nil { - return fmt.Errorf("failed to listen on port %s: %w", port, err) - } - - fmt.Printf("Stovepipe orchestrator gRPC server is running on %s\n", port) - fmt.Println("Press Ctrl+C to stop, or send a SIGTERM.") - - serverErrCh := make(chan error, 1) - go func() { - serverErrCh <- grpcServer.Serve(listener) - }() - - var serverErr error - select { - case <-ctx.Done(): - fmt.Println("Shutting down stovepipe orchestrator server due to interruption signal...") - - err = ctx.Err() - - grpcServer.GracefulStop() - serverErr = <-serverErrCh - case serverErr = <-serverErrCh: - fmt.Println("Shutting down stovepipe orchestrator server due to critical GRPC server error...") - cancel() - } - - if serverErr != nil { - serverErr = fmt.Errorf("GRPC server exited with error: %w", serverErr) - } - - primaryStopErr := primaryConsumer.Stop(30000) - if primaryStopErr != nil { - primaryStopErr = fmt.Errorf("failed to stop consumer: %w", primaryStopErr) - } - - if primaryStopErr != nil || serverErr != nil { - err = errors.Join(primaryStopErr, serverErr) - } - - return err -} - -func newTopicRegistry(q extqueue.Queue, subscriberName string) (consumer.TopicRegistry, error) { - return consumer.NewTopicRegistry([]consumer.TopicConfig{ - { - Key: topickey.TopicKeyStart, - Name: "start", - Queue: q, - Subscription: extqueue.DefaultSubscriptionConfig( - subscriberName, "orchestrator-start", - ), - }, - { - Key: topickey.TopicKeyValidate, - Name: "validate", - Queue: q, - Subscription: extqueue.DefaultSubscriptionConfig( - subscriberName, "orchestrator-validate", - ), - }, - { - Key: topickey.TopicKeyBatch, - Name: "batch", - Queue: q, - Subscription: extqueue.DefaultSubscriptionConfig( - subscriberName, "orchestrator-batch", - ), - }, - }) -} diff --git a/example/stovepipe/gateway/server/BUILD.bazel b/example/stovepipe/server/BUILD.bazel similarity index 61% rename from example/stovepipe/gateway/server/BUILD.bazel rename to example/stovepipe/server/BUILD.bazel index ea216c0c..017d8a96 100644 --- a/example/stovepipe/gateway/server/BUILD.bazel +++ b/example/stovepipe/server/BUILD.bazel @@ -1,18 +1,13 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") -exports_files( - ["docker-compose.yml"], - visibility = ["//visibility:public"], -) - go_library( - name = "gateway_server_lib", + name = "server_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/stovepipe/gateway/server", + importpath = "github.com/uber/submitqueue/example/stovepipe/server", visibility = ["//visibility:private"], deps = [ - "//api/stovepipe/gateway/protopb", - "//stovepipe/gateway/controller", + "//api/stovepipe/protopb", + "//stovepipe/controller", "@com_github_uber_go_tally//:tally", "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//reflection", @@ -21,7 +16,7 @@ go_library( ) go_binary( - name = "gateway", - embed = [":gateway_server_lib"], + name = "stovepipe", + embed = [":server_lib"], visibility = ["//visibility:public"], ) diff --git a/example/stovepipe/orchestrator/server/Dockerfile b/example/stovepipe/server/Dockerfile similarity index 52% rename from example/stovepipe/orchestrator/server/Dockerfile rename to example/stovepipe/server/Dockerfile index 6c797c7d..adbe1aeb 100644 --- a/example/stovepipe/orchestrator/server/Dockerfile +++ b/example/stovepipe/server/Dockerfile @@ -3,9 +3,10 @@ FROM debian:bookworm-slim RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* WORKDIR /root/ -# Built via: make build-stovepipe-orchestrator-linux -COPY .docker-bin/stovepipe-orchestrator ./orchestrator +# Copy pre-built Linux binary. +# Built via: make build-stovepipe-linux +COPY .docker-bin/stovepipe ./stovepipe EXPOSE 8080 -CMD ["./orchestrator"] +CMD ["./stovepipe"] diff --git a/example/stovepipe/gateway/server/main.go b/example/stovepipe/server/main.go similarity index 81% rename from example/stovepipe/gateway/server/main.go rename to example/stovepipe/server/main.go index ce693ebd..5428c8ed 100644 --- a/example/stovepipe/gateway/server/main.go +++ b/example/stovepipe/server/main.go @@ -26,21 +26,21 @@ import ( "time" "github.com/uber-go/tally" - pb "github.com/uber/submitqueue/api/stovepipe/gateway/protopb" - "github.com/uber/submitqueue/stovepipe/gateway/controller" + pb "github.com/uber/submitqueue/api/stovepipe/protopb" + "github.com/uber/submitqueue/stovepipe/controller" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/reflection" ) -// GatewayServer wraps the controller and implements the gRPC service interface. -type GatewayServer struct { - pb.UnimplementedStovepipeGatewayServer +// StovepipeServer wraps the controller and implements the gRPC service interface. +type StovepipeServer struct { + pb.UnimplementedStovepipeServer pingController *controller.PingController } // Ping delegates to the controller. -func (s *GatewayServer) Ping(ctx context.Context, req *pb.PingRequest) (*pb.PingResponse, error) { +func (s *StovepipeServer) Ping(ctx context.Context, req *pb.PingRequest) (*pb.PingResponse, error) { return s.pingController.Ping(ctx, req) } @@ -48,13 +48,13 @@ func main() { code := 0 if err := run(); err != nil { if errors.Is(err, context.Canceled) { - fmt.Println("Stovepipe gateway server stopped by signal") + fmt.Println("Stovepipe server stopped by signal") // Return 143 (128 + SIGTERM) as per POSIX standard if the application receives any termination signal from the OS. Ideally we should return 128+SIGINT for SIGINT and 128+SIGTERM for SIGTERM, // but it will require a special processing not yet available in the standard library. code = 128 + int(syscall.SIGTERM) } else { - fmt.Fprintf(os.Stderr, "Stovepipe gateway server failure: %v\n", err) + fmt.Fprintf(os.Stderr, "Stovepipe server failure: %v\n", err) // TODO: classify errors and implement a binary protocol for exit codes, so far 1 for everything code = 1 } @@ -75,7 +75,7 @@ func run() error { defer logger.Sync() // Initialize metrics scope - scope := tally.NewTestScope("stovepipe_gateway", nil) + scope := tally.NewTestScope("stovepipe", nil) metricsStopCh := make(chan interface{}, 1) metricsWgDone := sync.WaitGroup{} metricsWgDone.Add(1) @@ -110,10 +110,10 @@ func run() error { // Create ping controller and wrap it for gRPC pingController := controller.NewPingController(logger, scope) - srv := &GatewayServer{ + srv := &StovepipeServer{ pingController: pingController, } - pb.RegisterStovepipeGatewayServer(grpcServer, srv) + pb.RegisterStovepipeServer(grpcServer, srv) // Register reflection service for debugging with grpcurl reflection.Register(grpcServer) @@ -128,7 +128,7 @@ func run() error { return fmt.Errorf("failed to listen on port %s: %w", port, err) } - fmt.Printf("Stovepipe gateway gRPC server is running on %s\n", port) + fmt.Printf("Stovepipe gRPC server is running on %s\n", port) fmt.Println("Press Ctrl+C to stop, or send a SIGTERM.") // Start server in a goroutine and wait for it to finish @@ -143,7 +143,7 @@ func run() error { var serverErr error select { case <-ctx.Done(): - fmt.Println("Shutting down stovepipe gateway server due to interruption signal...") + fmt.Println("Shutting down stovepipe server due to interruption signal...") // Set the error to the context cancellation error to be surfaced as a desired exit code by the main function // to indicate that the server was stopped as intended @@ -154,7 +154,7 @@ func run() error { grpcServer.GracefulStop() serverErr = <-serverErrCh case serverErr = <-serverErrCh: - fmt.Println("Shutting down stovepipe gateway server due to critical GRPC server error...") + fmt.Println("Shutting down stovepipe server due to critical GRPC server error...") } if serverErr != nil { diff --git a/stovepipe/README.md b/stovepipe/README.md index 59bbab64..475de884 100644 --- a/stovepipe/README.md +++ b/stovepipe/README.md @@ -1,8 +1,7 @@ # Stovepipe -Stovepipe service layout: +Stovepipe is currently a single Ping-only service. Its layout: -- `gateway/` — Gateway service to check commit validation status and handle API interactions. -- `orchestrator/` — Orchestrator service to process validation workflows -- `extension/` — Stovepipe-specific extension implementations -- `entity/` — Stovepipe-specific domain entities +- `controller/` — business logic (transport-agnostic). Currently exposes the `Ping` RPC. + +The wire contract lives under `api/stovepipe/` (`proto/` for the `.proto` source, `protopb/` for the committed generated stubs). Entities, extensions, and the orchestration pipeline will be added back as the service grows. diff --git a/stovepipe/orchestrator/controller/BUILD.bazel b/stovepipe/controller/BUILD.bazel similarity index 76% rename from stovepipe/orchestrator/controller/BUILD.bazel rename to stovepipe/controller/BUILD.bazel index 991e77a2..e9a9f4cc 100644 --- a/stovepipe/orchestrator/controller/BUILD.bazel +++ b/stovepipe/controller/BUILD.bazel @@ -3,10 +3,10 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "controller", srcs = ["ping.go"], - importpath = "github.com/uber/submitqueue/stovepipe/orchestrator/controller", + importpath = "github.com/uber/submitqueue/stovepipe/controller", visibility = ["//visibility:public"], deps = [ - "//api/stovepipe/orchestrator/protopb", + "//api/stovepipe/protopb", "//platform/metrics", "@com_github_uber_go_tally//:tally", "@org_uber_go_zap//:zap", @@ -18,7 +18,7 @@ go_test( srcs = ["ping_test.go"], embed = [":controller"], deps = [ - "//api/stovepipe/orchestrator/protopb", + "//api/stovepipe/protopb", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@com_github_uber_go_tally//:tally", diff --git a/stovepipe/gateway/controller/ping.go b/stovepipe/controller/ping.go similarity index 95% rename from stovepipe/gateway/controller/ping.go rename to stovepipe/controller/ping.go index a6dedb56..f43586d0 100644 --- a/stovepipe/gateway/controller/ping.go +++ b/stovepipe/controller/ping.go @@ -20,7 +20,7 @@ import ( "time" "github.com/uber-go/tally" - pb "github.com/uber/submitqueue/api/stovepipe/gateway/protopb" + pb "github.com/uber/submitqueue/api/stovepipe/protopb" "github.com/uber/submitqueue/platform/metrics" "go.uber.org/zap" ) @@ -64,7 +64,7 @@ func (c *PingController) Ping(ctx context.Context, req *pb.PingRequest) (resp *p return &pb.PingResponse{ Message: message, - ServiceName: "stovepipe-gateway", + ServiceName: "stovepipe", Timestamp: time.Now().Unix(), Hostname: hostname, }, nil diff --git a/stovepipe/gateway/controller/ping_test.go b/stovepipe/controller/ping_test.go similarity index 95% rename from stovepipe/gateway/controller/ping_test.go rename to stovepipe/controller/ping_test.go index f365ed04..383f1a14 100644 --- a/stovepipe/gateway/controller/ping_test.go +++ b/stovepipe/controller/ping_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber-go/tally" - pb "github.com/uber/submitqueue/api/stovepipe/gateway/protopb" + pb "github.com/uber/submitqueue/api/stovepipe/protopb" "go.uber.org/zap" ) @@ -75,7 +75,7 @@ func TestPing_ServiceName(t *testing.T) { resp, err := controller.Ping(ctx, req) require.NoError(t, err) - assert.Equal(t, "stovepipe-gateway", resp.ServiceName) + assert.Equal(t, "stovepipe", resp.ServiceName) } func TestPing_Timestamp(t *testing.T) { diff --git a/stovepipe/core/BUILD.bazel b/stovepipe/core/BUILD.bazel deleted file mode 100644 index 163292f8..00000000 --- a/stovepipe/core/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "core", - srcs = ["core.go"], - importpath = "github.com/uber/submitqueue/stovepipe/core", - visibility = ["//visibility:public"], -) diff --git a/stovepipe/core/core.go b/stovepipe/core/core.go deleted file mode 100644 index a6b25d4a..00000000 --- a/stovepipe/core/core.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package core groups infrastructure shared across Stovepipe's own services -// (gateway and orchestrator) — the Stovepipe-scoped analogue of the repo-level -// core/. Cross-domain infrastructure lives in the top-level core/; this package -// is for plumbing private to Stovepipe. Subpackages (e.g. core/consumer, -// core/request) are added here as shared needs emerge, mirroring submitqueue/core. -package core diff --git a/stovepipe/core/filter/BUILD.bazel b/stovepipe/core/filter/BUILD.bazel deleted file mode 100644 index 923cee0a..00000000 --- a/stovepipe/core/filter/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "filter", - srcs = ["filter.go"], - importpath = "github.com/uber/submitqueue/stovepipe/core/filter", - visibility = ["//visibility:public"], -) diff --git a/stovepipe/core/filter/filter.go b/stovepipe/core/filter/filter.go deleted file mode 100644 index 5dd86eeb..00000000 --- a/stovepipe/core/filter/filter.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package filter implements a filter for commit events. -package filter - -import "strings" - -// ShouldProcess reports whether a commit URI should be processed by the pipeline. -// watchedPrefixes is a list of URI prefixes to match against the commit URI. -// Example prefix: "git://github.com/uber/go-code/refs/heads/main" -func ShouldProcess(uri string, watchedPrefixes []string) bool { - for _, prefix := range watchedPrefixes { - if strings.HasPrefix(uri, prefix) { - return true - } - } - return false -} diff --git a/stovepipe/core/topickey/BUILD.bazel b/stovepipe/core/topickey/BUILD.bazel deleted file mode 100644 index 0ce34bf0..00000000 --- a/stovepipe/core/topickey/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "topickey", - srcs = ["topickey.go"], - importpath = "github.com/uber/submitqueue/stovepipe/core/topickey", - visibility = ["//visibility:public"], - deps = ["//platform/consumer"], -) diff --git a/stovepipe/core/topickey/topickey.go b/stovepipe/core/topickey/topickey.go deleted file mode 100644 index ccab7f3f..00000000 --- a/stovepipe/core/topickey/topickey.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package topickey defines Stovepipe pipeline stage identifiers. -package topickey - -import "github.com/uber/submitqueue/platform/consumer" - -// TopicKey is the shared pipeline stage identifier type. -type TopicKey = consumer.TopicKey - -const ( - // TopicKeyStart is the pipeline stage where trunk push events arrive from the gateway. - TopicKeyStart TopicKey = "start" - // TopicKeyValidate is the pipeline stage where commits are published for metadata resolution. - TopicKeyValidate TopicKey = "validate" - // TopicKeyBatch is the pipeline stage where validated commits are aggregated, since the last - // known green, into a contiguous validation batch. - TopicKeyBatch TopicKey = "batch" -) diff --git a/stovepipe/entity/BUILD.bazel b/stovepipe/entity/BUILD.bazel deleted file mode 100644 index 290244c9..00000000 --- a/stovepipe/entity/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "entity", - srcs = [ - "ingest_request.go", - "request.go", - "request_log.go", - ], - importpath = "github.com/uber/submitqueue/stovepipe/entity", - visibility = ["//visibility:public"], - deps = ["//platform/base/change"], -) - -go_test( - name = "entity_test", - srcs = [ - "ingest_request_test.go", - "request_log_test.go", - "request_test.go", - ], - embed = [":entity"], - deps = [ - "//platform/base/change", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - ], -) diff --git a/stovepipe/entity/ingest_request.go b/stovepipe/entity/ingest_request.go deleted file mode 100644 index e03da23d..00000000 --- a/stovepipe/entity/ingest_request.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "encoding/json" - "fmt" - - "github.com/uber/submitqueue/platform/base/change" -) - -// IngestRequest is the gateway-owned contract for a Stovepipe ingest request as it -// travels over the queue to the orchestrator. It carries the validated inputs and the -// generated request ID (the "spid"); downstream stages resolve any further detail. -type IngestRequest struct { - // ID is the globally unique identifier for the ingest request (the "spid"). - // Format: "/". - ID string `json:"id"` - // Queue is the name of the queue processing the ingest request. - Queue string `json:"queue"` - // Change is the set of trunk commits to verify, identified by URI. The scheme names - // the VCS; the rest is provider-specific (e.g. git://remote/repo/ref/commit_sha). - Change change.Change `json:"change"` -} - -// ToBytes serializes the IngestRequest to JSON bytes for queue message payload. -func (r IngestRequest) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// Validate checks that the request carries the identity and change URIs the pipeline -// needs to act on it. Selecting a resolver for each URI's scheme is the resolver/wiring -// layer's job, so Validate stays VCS-agnostic and does not inspect the scheme. -func (r IngestRequest) Validate() error { - if r.ID == "" { - return fmt.Errorf("ingest request requires an id") - } - if r.Queue == "" { - return fmt.Errorf("ingest request requires a queue") - } - if len(r.Change.URIs) == 0 { - return fmt.Errorf("ingest request requires at least one change URI") - } - for _, uri := range r.Change.URIs { - if uri == "" { - return fmt.Errorf("ingest request change URIs must be non-empty") - } - } - return nil -} - -// IngestRequestFromBytes deserializes an IngestRequest from JSON bytes and validates it. -func IngestRequestFromBytes(data []byte) (IngestRequest, error) { - var req IngestRequest - if err := json.Unmarshal(data, &req); err != nil { - return IngestRequest{}, err - } - if err := req.Validate(); err != nil { - return IngestRequest{}, err - } - return req, nil -} diff --git a/stovepipe/entity/ingest_request_test.go b/stovepipe/entity/ingest_request_test.go deleted file mode 100644 index 534cd606..00000000 --- a/stovepipe/entity/ingest_request_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber/submitqueue/platform/base/change" -) - -const ( - testURI = "git://git.example.com/uber/monorepo/refs%2Fheads%2Fmain/abcdef0123456789abcdef0123456789abcdef01" - testSPID = "stovepipe-monorepo/1" - testQueue = "stovepipe-monorepo" -) - -func validIngestRequest() IngestRequest { - return IngestRequest{ - ID: testSPID, - Queue: testQueue, - Change: change.Change{URIs: []string{testURI}}, - } -} - -func TestIngestRequest_Validate(t *testing.T) { - tests := []struct { - name string - req IngestRequest - wantErr bool - }{ - {name: "valid", req: validIngestRequest()}, - {name: "missing id", req: IngestRequest{Queue: testQueue, Change: change.Change{URIs: []string{testURI}}}, wantErr: true}, - {name: "missing queue", req: IngestRequest{ID: testSPID, Change: change.Change{URIs: []string{testURI}}}, wantErr: true}, - {name: "no uris", req: IngestRequest{ID: testSPID, Queue: testQueue}, wantErr: true}, - {name: "empty uri", req: IngestRequest{ID: testSPID, Queue: testQueue, Change: change.Change{URIs: []string{""}}}, wantErr: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.req.Validate() - if tt.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - }) - } -} - -func TestIngestRequestFromBytes(t *testing.T) { - t.Run("round-trips a valid request", func(t *testing.T) { - original := validIngestRequest() - data, err := original.ToBytes() - require.NoError(t, err) - - got, err := IngestRequestFromBytes(data) - require.NoError(t, err) - assert.Equal(t, original, got) - }) - - t.Run("rejects invalid json", func(t *testing.T) { - _, err := IngestRequestFromBytes([]byte(`{"invalid": json"}`)) - require.Error(t, err) - }) - - t.Run("rejects a payload that fails validation", func(t *testing.T) { - _, err := IngestRequestFromBytes([]byte(`{"id":"x","queue":"q","change":{"uris":[]}}`)) - require.Error(t, err) - }) -} diff --git a/stovepipe/entity/request.go b/stovepipe/entity/request.go deleted file mode 100644 index e4a10ccc..00000000 --- a/stovepipe/entity/request.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "encoding/json" - - "github.com/uber/submitqueue/platform/base/change" -) - -// RequestState defines the possible states of a Stovepipe trunk-validation request. They are internal -// and used to implement a state machine. A separate RequestStatus type tracks the customer-friendly -// status of a request. Stovepipe validates commits after they land, so the terminal states describe a -// commit's health (succeeded / failed) rather than a merge outcome. -type RequestState string - -const ( - // RequestStateUnknown is the unreachable sentinel state. It is set by default when the structure is - // initialized and should never be seen in the system. - RequestStateUnknown RequestState = "" - // RequestStateStarted is the initial state of a request. The commit has been recorded by the - // orchestrator and is entering the pipeline, but no validation has happened yet. - RequestStateStarted RequestState = "started" - // RequestStateValidated indicates that the commit metadata needed for ordering and batching has been - // resolved successfully. - RequestStateValidated RequestState = "validated" - // RequestStateBatched indicates that the request has been enrolled in a validation batch (a contiguous - // range of commits since the last known green). - RequestStateBatched RequestState = "batched" - // RequestStateBuilding indicates that the batch containing the request is being built and tested. - RequestStateBuilding RequestState = "building" - // RequestStateSucceeded is the terminal state of a request whose commit validated green. This is a - // final state. - RequestStateSucceeded RequestState = "succeeded" - // RequestStateFailed is the terminal state of a request whose commit was found to break a target. This - // is a final state. - RequestStateFailed RequestState = "failed" - // RequestStateError is the terminal state of a request that encountered an infrastructure error and - // could not be validated. This is a final state. - RequestStateError RequestState = "error" -) - -// IsRequestStateTerminal returns true if the state represents a final, irreversible state (succeeded, -// failed, or error). Forward-progress controllers use this to short-circuit work for requests that have -// already reached a terminal outcome. -func IsRequestStateTerminal(s RequestState) bool { - return s == RequestStateSucceeded || s == RequestStateFailed || s == RequestStateError -} - -// Request defines a Stovepipe request to validate a set of trunk commits after they have landed on the -// target branch. The immutable fields are fixed at creation; the mutable fields advance as the request -// moves through the pipeline. -type Request struct { - // **************** - // Immutable fields, fixed at request entity creation - // **************** - - // ID is the globally unique identifier for the request (the "spid"). Format: "/". - ID string `json:"id"` - // Queue is the name of the queue processing the request. Queue name is defined in the configuration and - // should be unique within the system. - Queue string `json:"queue"` - // Change is the set of trunk commits to validate, identified by URI. The scheme names the VCS; the rest - // is provider-specific (e.g. git://remote/repo/ref/commit_sha). - Change change.Change `json:"change"` - - // **************** - // Following fields could be changed throughout the lifecycle of the request - // **************** - - // State is the current state of the request. - State RequestState `json:"state"` - // Version is the version of the object. It is used for optimistic locking. - // Versioning starts at 1 and is incremented for each change to the object. - Version int32 `json:"version"` -} - -// ToBytes serializes the Request to JSON bytes for queue message payload. -func (r Request) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// RequestFromBytes deserializes a Request from JSON bytes. -func RequestFromBytes(data []byte) (Request, error) { - var req Request - err := json.Unmarshal(data, &req) - return req, err -} - -// RequestID is a lightweight entity for publishing and consuming just the request identifier via the -// queue. Internal pipeline hops carry the ID and reload the full request from storage. -type RequestID struct { - // ID is the globally unique identifier for the request. - ID string `json:"id"` -} - -// ToBytes serializes the RequestID to JSON bytes for queue message payload. -func (r RequestID) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// RequestIDFromBytes deserializes a RequestID from JSON bytes. -func RequestIDFromBytes(data []byte) (RequestID, error) { - var rid RequestID - err := json.Unmarshal(data, &rid) - return rid, err -} diff --git a/stovepipe/entity/request_log.go b/stovepipe/entity/request_log.go deleted file mode 100644 index 8ea25809..00000000 --- a/stovepipe/entity/request_log.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "encoding/json" - "time" -) - -// RequestStatus defines the possible customer-friendly status of a request. Status is display-friendly -// and can be shown to callers; it is different from the internal RequestState used to implement the state -// machine. Request statuses can generally be added freely by the system without breaking the state machine. -// Some statuses correspond to a request state, in which case they should be supplemented with the request -// state version for reconciliation. Other statuses are purely informational and can be added freely. Every -// status may be accompanied by a last error message and free-form metadata in the request log; those are -// used for display or debugging purposes only. -type RequestStatus string - -const ( - // RequestStatusUnknown is the unknown sentinel status. It is set by default when the structure is - // initialized and should never be seen in the system. - RequestStatusUnknown RequestStatus = "" - - // RequestStatusAccepted indicates that the request has been accepted by the system. The gateway sets - // this status when the ingest request is received and persisted to the logging database. - RequestStatusAccepted RequestStatus = "accepted" - - // RequestStatusStarted is the initial status of a request. It corresponds to the RequestStateStarted - // state and is typically set by the orchestrator when the commit is recorded in the operating database. - RequestStatusStarted RequestStatus = "started" - - // RequestStatusValidating indicates that the request's commit metadata is currently being resolved for - // ordering and batching. - RequestStatusValidating RequestStatus = "validating" - - // RequestStatusValidated indicates that the request has been validated successfully. It corresponds to - // the RequestStateValidated state. - RequestStatusValidated RequestStatus = "validated" - - // RequestStatusBatching indicates that the request is waiting to be included in a validation batch. - RequestStatusBatching RequestStatus = "batching" - - // RequestStatusBatched indicates that the request has been included in a validation batch. It - // corresponds to the RequestStateBatched state. - RequestStatusBatched RequestStatus = "batched" - - // RequestStatusBuilding indicates that the batch containing the request is being built and tested. It - // corresponds to the RequestStateBuilding state. - RequestStatusBuilding RequestStatus = "building" - - // RequestStatusBuilt indicates that the batch containing the request has finished building and can move - // to the next phase. - RequestStatusBuilt RequestStatus = "built" - - // RequestStatusSucceeded indicates that the request's commit validated green. It corresponds to the - // RequestStateSucceeded state. - RequestStatusSucceeded RequestStatus = "succeeded" - - // RequestStatusFailed indicates that the request's commit was found to break a target. It corresponds - // to the RequestStateFailed state. - RequestStatusFailed RequestStatus = "failed" - - // RequestStatusError indicates that the request encountered an infrastructure error and could not be - // validated. It corresponds to the RequestStateError state. - RequestStatusError RequestStatus = "error" -) - -// RequestLog is an append-only record that captures a point-in-time snapshot of a request's status for -// reconciliation purposes. It is stored in a separate database from the request store to support eventual -// consistency reconciliation. -type RequestLog struct { - // RequestID is the ID of the request this log entry belongs to. References entity.Request.ID. - RequestID string `json:"request_id"` - // TimestampMs is the time this log entry was created, in milliseconds since Unix epoch. - TimestampMs int64 `json:"timestamp_ms"` - // Status is the request status at the time this log entry was created. It may contain request states - // from the state machine and also display-friendly intermediate statuses. - Status RequestStatus `json:"status"` - // RequestVersion is the version of the request at the time this log entry was created. - // Zero if the version is not available. - RequestVersion int32 `json:"request_version"` - // LastError is the last error message associated with the status at the time of this log entry. - // Empty string if no error. - LastError string `json:"last_error"` - // Metadata is a set of key-value pairs providing additional context for this log entry. - // Empty map if no metadata. - Metadata map[string]string `json:"metadata"` -} - -// NewRequestLog creates a new RequestLog with the given fields. -// TimestampMs is set to the current time. If metadata is nil, it will be initialized as an empty map. -// requestVersion is the version of the request entity; it should only be set when reporting a request -// state as a status, otherwise it should be 0. -// lastError is the last error message associated with the status at the time of this log entry, empty -// string if no error. -// metadata is a set of key-value pairs providing additional context for this log entry. It is not -// constrained to any specific format or schema and is used for display or debugging purposes. -func NewRequestLog(requestID string, status RequestStatus, requestVersion int32, lastError string, metadata map[string]string) RequestLog { - if metadata == nil { - metadata = make(map[string]string) - } - return RequestLog{ - RequestID: requestID, - TimestampMs: time.Now().UnixMilli(), - Status: status, - RequestVersion: requestVersion, - LastError: lastError, - Metadata: metadata, - } -} - -// ToBytes serializes the RequestLog to JSON bytes for queue message payload. -func (r RequestLog) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// RequestLogFromBytes deserializes a RequestLog from JSON bytes. -// If metadata is absent from the JSON, it will be initialized as an empty map. -func RequestLogFromBytes(data []byte) (RequestLog, error) { - var log RequestLog - err := json.Unmarshal(data, &log) - if err != nil { - return log, err - } - if log.Metadata == nil { - log.Metadata = make(map[string]string) - } - return log, nil -} diff --git a/stovepipe/entity/request_log_test.go b/stovepipe/entity/request_log_test.go deleted file mode 100644 index 7cfbc67e..00000000 --- a/stovepipe/entity/request_log_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewRequestLog_NilMetadata(t *testing.T) { - log := NewRequestLog("queue1/100", RequestStatusStarted, 0, "", nil) - - assert.NotNil(t, log.Metadata) - assert.Empty(t, log.Metadata) -} - -func TestRequestLog_ToBytes(t *testing.T) { - log := RequestLog{ - RequestID: "stovepipe-monorepo/123", - TimestampMs: 1709568000000, - Status: RequestStatusStarted, - RequestVersion: 1, - LastError: "", - Metadata: map[string]string{"source": "gateway"}, - } - - data, err := log.ToBytes() - require.NoError(t, err) - assert.NotEmpty(t, data) - - jsonStr := string(data) - assert.Contains(t, jsonStr, "stovepipe-monorepo/123") - assert.Contains(t, jsonStr, "1709568000000") - assert.Contains(t, jsonStr, "gateway") -} - -func TestRequestLogFromBytes(t *testing.T) { - original := RequestLog{ - RequestID: "my-queue/999", - TimestampMs: 1709568000000, - Status: RequestStatusBuilding, - RequestVersion: 3, - LastError: "timeout", - Metadata: map[string]string{"step": "build", "attempt": "2"}, - } - - data, err := original.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestLogFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, original.RequestID, deserialized.RequestID) - assert.Equal(t, original.TimestampMs, deserialized.TimestampMs) - assert.Equal(t, original.Status, deserialized.Status) - assert.Equal(t, original.RequestVersion, deserialized.RequestVersion) - assert.Equal(t, original.LastError, deserialized.LastError) - assert.Equal(t, original.Metadata, deserialized.Metadata) -} - -func TestRequestLogFromBytes_InvalidJSON(t *testing.T) { - invalidJSON := []byte(`{"invalid": json"}`) - - _, err := RequestLogFromBytes(invalidJSON) - assert.Error(t, err) -} - -func TestRequestLogFromBytes_EmptyData(t *testing.T) { - emptyJSON := []byte(`{}`) - - log, err := RequestLogFromBytes(emptyJSON) - require.NoError(t, err) - - assert.Empty(t, log.RequestID) - assert.Equal(t, int64(0), log.TimestampMs) - assert.Empty(t, log.Status) - assert.Equal(t, int32(0), log.RequestVersion) - assert.Empty(t, log.LastError) - assert.NotNil(t, log.Metadata) - assert.Empty(t, log.Metadata) -} - -func TestRequestLog_SerializationRoundTrip(t *testing.T) { - tests := []struct { - name string - log RequestLog - }{ - { - name: "with all fields populated", - log: RequestLog{ - RequestID: "queue1/100", - TimestampMs: 1709568000000, - Status: RequestStatusSucceeded, - RequestVersion: 5, - LastError: "", - Metadata: map[string]string{"source": "orchestrator", "batch_id": "b-1"}, - }, - }, - { - name: "with error", - log: RequestLog{ - RequestID: "queue2/200", - TimestampMs: 1709568001000, - Status: RequestStatusError, - RequestVersion: 2, - LastError: "build infrastructure unavailable", - Metadata: map[string]string{}, - }, - }, - { - name: "with zero version", - log: RequestLog{ - RequestID: "queue3/300", - TimestampMs: 1709568002000, - Status: RequestStatusStarted, - RequestVersion: 0, - LastError: "", - Metadata: map[string]string{"key": "value"}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - data, err := tt.log.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestLogFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, tt.log, deserialized) - }) - } -} diff --git a/stovepipe/entity/request_test.go b/stovepipe/entity/request_test.go deleted file mode 100644 index 8b96f7f8..00000000 --- a/stovepipe/entity/request_test.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber/submitqueue/platform/base/change" -) - -func TestRequest_ToBytes(t *testing.T) { - req := Request{ - ID: "stovepipe-monorepo/123", - Queue: "stovepipe-monorepo", - Change: change.Change{URIs: []string{ - "git://git.example.com/uber/monorepo/refs%2Fheads%2Fmain/abcdef0123456789abcdef0123456789abcdef01", - }}, - State: RequestStateStarted, - Version: 1, - } - - data, err := req.ToBytes() - require.NoError(t, err) - assert.NotEmpty(t, data) - - jsonStr := string(data) - assert.Contains(t, jsonStr, "stovepipe-monorepo/123") - assert.Contains(t, jsonStr, "abcdef0123456789abcdef0123456789abcdef01") - assert.Contains(t, jsonStr, "started") -} - -func TestRequestFromBytes(t *testing.T) { - original := Request{ - ID: "my-queue/999", - Queue: "my-queue", - Change: change.Change{URIs: []string{"git://git.example.com/uber/monorepo/refs%2Fheads%2Fmain/0123456789"}}, - State: RequestStateBuilding, - Version: 3, - } - - data, err := original.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, original.ID, deserialized.ID) - assert.Equal(t, original.Queue, deserialized.Queue) - assert.Equal(t, original.Change.URIs, deserialized.Change.URIs) - assert.Equal(t, original.State, deserialized.State) - assert.Equal(t, original.Version, deserialized.Version) -} - -func TestRequestFromBytes_InvalidJSON(t *testing.T) { - invalidJSON := []byte(`{"invalid": json"}`) - - _, err := RequestFromBytes(invalidJSON) - assert.Error(t, err) -} - -func TestRequestFromBytes_EmptyData(t *testing.T) { - emptyJSON := []byte(`{}`) - - req, err := RequestFromBytes(emptyJSON) - require.NoError(t, err) - - assert.Empty(t, req.ID) - assert.Empty(t, req.Queue) - assert.Equal(t, RequestStateUnknown, req.State) - assert.Equal(t, int32(0), req.Version) -} - -func TestIsRequestStateTerminal(t *testing.T) { - tests := []struct { - state RequestState - terminal bool - }{ - {RequestStateUnknown, false}, - {RequestStateStarted, false}, - {RequestStateValidated, false}, - {RequestStateBatched, false}, - {RequestStateBuilding, false}, - {RequestStateSucceeded, true}, - {RequestStateFailed, true}, - {RequestStateError, true}, - } - for _, tt := range tests { - t.Run(string(tt.state), func(t *testing.T) { - assert.Equal(t, tt.terminal, IsRequestStateTerminal(tt.state)) - }) - } -} - -func TestRequest_SerializationRoundTrip(t *testing.T) { - tests := []struct { - name string - req Request - }{ - { - name: "single commit started", - req: Request{ - ID: "queue1/100", - Queue: "queue1", - Change: change.Change{URIs: []string{"git://git.example.com/uber/repo-a/refs%2Fheads%2Fmain/aaaaaaaaaa"}}, - State: RequestStateStarted, - Version: 1, - }, - }, - { - name: "succeeded terminal", - req: Request{ - ID: "queue2/200", - Queue: "queue2", - Change: change.Change{URIs: []string{"git://git.example.com/uber/repo-b/refs%2Fheads%2Fmain/bbbbbbbbbb"}}, - State: RequestStateSucceeded, - Version: 5, - }, - }, - { - name: "failed terminal", - req: Request{ - ID: "queue3/300", - Queue: "queue3", - Change: change.Change{URIs: []string{"git://git.example.com/uber/repo-c/refs%2Fheads%2Fmain/cccccccccc"}}, - State: RequestStateFailed, - Version: 10, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - data, err := tt.req.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, tt.req, deserialized) - }) - } -} - -func TestRequestID_SerializationRoundTrip(t *testing.T) { - original := RequestID{ID: "stovepipe-monorepo/42"} - - data, err := original.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestIDFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, original, deserialized) -} - -func TestRequestIDFromBytes_InvalidJSON(t *testing.T) { - _, err := RequestIDFromBytes([]byte(`{"invalid": json"}`)) - assert.Error(t, err) -} diff --git a/stovepipe/extension/BUILD.bazel b/stovepipe/extension/BUILD.bazel deleted file mode 100644 index bbf4f83b..00000000 --- a/stovepipe/extension/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "extension", - srcs = ["extension.go"], - importpath = "github.com/uber/submitqueue/stovepipe/extension", - visibility = ["//visibility:public"], -) diff --git a/stovepipe/extension/extension.go b/stovepipe/extension/extension.go deleted file mode 100644 index 92d4bcea..00000000 --- a/stovepipe/extension/extension.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package extension holds Stovepipe-specific extension implementations. -package extension diff --git a/stovepipe/extension/storage/BUILD.bazel b/stovepipe/extension/storage/BUILD.bazel deleted file mode 100644 index bf885167..00000000 --- a/stovepipe/extension/storage/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "storage", - srcs = [ - "request_log_store.go", - "request_store.go", - "storage.go", - ], - importpath = "github.com/uber/submitqueue/stovepipe/extension/storage", - visibility = ["//visibility:public"], - deps = ["//stovepipe/entity"], -) diff --git a/stovepipe/extension/storage/README.md b/stovepipe/extension/storage/README.md deleted file mode 100644 index e107556e..00000000 --- a/stovepipe/extension/storage/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Storage - -Pluggable persistence interfaces for Stovepipe entities. Implementations live under `extension/storage//`. - -Stovepipe mirrors SubmitQueue's two-store split by service ownership. The orchestrator owns the `RequestStore` — the working pipeline state machine for in-flight commits — and is the only service that writes it. The gateway owns the `RequestLogStore` — the append-only, customer-facing status log — and is the only service that reads or writes it. The orchestrator never touches the request log; it only publishes log events that the gateway persists. - -## Optimistic locking contract - -Entities that support concurrent mutation carry an `int32 Version` field. Updates are conditional on the version: the write only succeeds if the persisted version matches the caller's expected version. On mismatch, the implementation returns `storage.ErrVersionMismatch`. - -**Version arithmetic is owned by the controller, not the store.** Update methods take both `oldVersion` (the where-clause guard) and `newVersion` (the value to write): - -```go -UpdateState(ctx, id, oldVersion, newVersion int32, newState entity.RequestState) error -``` - -The store performs a pure conditional write — it does not compute `oldVersion + 1` internally. This keeps the in-memory entity and the persisted row in sync without the storage layer mutating values the caller didn't supply. - -### Caller pattern - -```go -newVersion := request.Version + 1 -if err := store.UpdateState(ctx, request.ID, request.Version, newVersion, newState); err != nil { - return err // request.Version unchanged on failure — safe to retry -} -request.Version = newVersion // only after the write succeeded -``` - -The post-success assignment matters whenever the entity is read again later in the same flow. Pre-incrementing in memory before the call is a bug pattern: if the call fails and the caller swallows the error, the in-memory version is now ahead of the database and subsequent updates will fail with `ErrVersionMismatch` for non-obvious reasons. diff --git a/stovepipe/extension/storage/mock/BUILD.bazel b/stovepipe/extension/storage/mock/BUILD.bazel deleted file mode 100644 index ea5ffb66..00000000 --- a/stovepipe/extension/storage/mock/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "mock", - srcs = [ - "request_log_store_mock.go", - "request_store_mock.go", - "storage_mock.go", - ], - importpath = "github.com/uber/submitqueue/stovepipe/extension/storage/mock", - visibility = ["//visibility:public"], - deps = [ - "//stovepipe/entity", - "//stovepipe/extension/storage", - "@org_uber_go_mock//gomock", - ], -) diff --git a/stovepipe/extension/storage/mock/request_log_store_mock.go b/stovepipe/extension/storage/mock/request_log_store_mock.go deleted file mode 100644 index c7238b04..00000000 --- a/stovepipe/extension/storage/mock/request_log_store_mock.go +++ /dev/null @@ -1,71 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: request_log_store.go -// -// Generated by this command: -// -// mockgen -source=request_log_store.go -destination=mock/request_log_store_mock.go -package=mock -// - -// Package mock is a generated GoMock package. -package mock - -import ( - context "context" - reflect "reflect" - - entity "github.com/uber/submitqueue/stovepipe/entity" - gomock "go.uber.org/mock/gomock" -) - -// MockRequestLogStore is a mock of RequestLogStore interface. -type MockRequestLogStore struct { - ctrl *gomock.Controller - recorder *MockRequestLogStoreMockRecorder - isgomock struct{} -} - -// MockRequestLogStoreMockRecorder is the mock recorder for MockRequestLogStore. -type MockRequestLogStoreMockRecorder struct { - mock *MockRequestLogStore -} - -// NewMockRequestLogStore creates a new mock instance. -func NewMockRequestLogStore(ctrl *gomock.Controller) *MockRequestLogStore { - mock := &MockRequestLogStore{ctrl: ctrl} - mock.recorder = &MockRequestLogStoreMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRequestLogStore) EXPECT() *MockRequestLogStoreMockRecorder { - return m.recorder -} - -// Insert mocks base method. -func (m *MockRequestLogStore) Insert(ctx context.Context, log entity.RequestLog) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Insert", ctx, log) - ret0, _ := ret[0].(error) - return ret0 -} - -// Insert indicates an expected call of Insert. -func (mr *MockRequestLogStoreMockRecorder) Insert(ctx, log any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockRequestLogStore)(nil).Insert), ctx, log) -} - -// List mocks base method. -func (m *MockRequestLogStore) List(ctx context.Context, requestID string) ([]entity.RequestLog, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", ctx, requestID) - ret0, _ := ret[0].([]entity.RequestLog) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// List indicates an expected call of List. -func (mr *MockRequestLogStoreMockRecorder) List(ctx, requestID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockRequestLogStore)(nil).List), ctx, requestID) -} diff --git a/stovepipe/extension/storage/mock/request_store_mock.go b/stovepipe/extension/storage/mock/request_store_mock.go deleted file mode 100644 index 19988d09..00000000 --- a/stovepipe/extension/storage/mock/request_store_mock.go +++ /dev/null @@ -1,85 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: request_store.go -// -// Generated by this command: -// -// mockgen -source=request_store.go -destination=mock/request_store_mock.go -package=mock -// - -// Package mock is a generated GoMock package. -package mock - -import ( - context "context" - reflect "reflect" - - entity "github.com/uber/submitqueue/stovepipe/entity" - gomock "go.uber.org/mock/gomock" -) - -// MockRequestStore is a mock of RequestStore interface. -type MockRequestStore struct { - ctrl *gomock.Controller - recorder *MockRequestStoreMockRecorder - isgomock struct{} -} - -// MockRequestStoreMockRecorder is the mock recorder for MockRequestStore. -type MockRequestStoreMockRecorder struct { - mock *MockRequestStore -} - -// NewMockRequestStore creates a new mock instance. -func NewMockRequestStore(ctrl *gomock.Controller) *MockRequestStore { - mock := &MockRequestStore{ctrl: ctrl} - mock.recorder = &MockRequestStoreMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRequestStore) EXPECT() *MockRequestStoreMockRecorder { - return m.recorder -} - -// Create mocks base method. -func (m *MockRequestStore) Create(ctx context.Context, request entity.Request) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", ctx, request) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create. -func (mr *MockRequestStoreMockRecorder) Create(ctx, request any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockRequestStore)(nil).Create), ctx, request) -} - -// Get mocks base method. -func (m *MockRequestStore) Get(ctx context.Context, id string) (entity.Request, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", ctx, id) - ret0, _ := ret[0].(entity.Request) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Get indicates an expected call of Get. -func (mr *MockRequestStoreMockRecorder) Get(ctx, id any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRequestStore)(nil).Get), ctx, id) -} - -// UpdateState mocks base method. -func (m *MockRequestStore) UpdateState(ctx context.Context, id string, oldVersion, newVersion int32, newState entity.RequestState) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateState", ctx, id, oldVersion, newVersion, newState) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateState indicates an expected call of UpdateState. -func (mr *MockRequestStoreMockRecorder) UpdateState(ctx, id, oldVersion, newVersion, newState any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateState", reflect.TypeOf((*MockRequestStore)(nil).UpdateState), ctx, id, oldVersion, newVersion, newState) -} diff --git a/stovepipe/extension/storage/mock/storage_mock.go b/stovepipe/extension/storage/mock/storage_mock.go deleted file mode 100644 index 1871aa6f..00000000 --- a/stovepipe/extension/storage/mock/storage_mock.go +++ /dev/null @@ -1,83 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: storage.go -// -// Generated by this command: -// -// mockgen -source=storage.go -destination=mock/storage_mock.go -package=mock -// - -// Package mock is a generated GoMock package. -package mock - -import ( - reflect "reflect" - - storage "github.com/uber/submitqueue/stovepipe/extension/storage" - gomock "go.uber.org/mock/gomock" -) - -// MockStorage is a mock of Storage interface. -type MockStorage struct { - ctrl *gomock.Controller - recorder *MockStorageMockRecorder - isgomock struct{} -} - -// MockStorageMockRecorder is the mock recorder for MockStorage. -type MockStorageMockRecorder struct { - mock *MockStorage -} - -// NewMockStorage creates a new mock instance. -func NewMockStorage(ctrl *gomock.Controller) *MockStorage { - mock := &MockStorage{ctrl: ctrl} - mock.recorder = &MockStorageMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockStorage) EXPECT() *MockStorageMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockStorage) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockStorageMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockStorage)(nil).Close)) -} - -// GetRequestLogStore mocks base method. -func (m *MockStorage) GetRequestLogStore() storage.RequestLogStore { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRequestLogStore") - ret0, _ := ret[0].(storage.RequestLogStore) - return ret0 -} - -// GetRequestLogStore indicates an expected call of GetRequestLogStore. -func (mr *MockStorageMockRecorder) GetRequestLogStore() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestLogStore", reflect.TypeOf((*MockStorage)(nil).GetRequestLogStore)) -} - -// GetRequestStore mocks base method. -func (m *MockStorage) GetRequestStore() storage.RequestStore { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRequestStore") - ret0, _ := ret[0].(storage.RequestStore) - return ret0 -} - -// GetRequestStore indicates an expected call of GetRequestStore. -func (mr *MockStorageMockRecorder) GetRequestStore() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRequestStore", reflect.TypeOf((*MockStorage)(nil).GetRequestStore)) -} diff --git a/stovepipe/extension/storage/mysql/BUILD.bazel b/stovepipe/extension/storage/mysql/BUILD.bazel deleted file mode 100644 index 289ab892..00000000 --- a/stovepipe/extension/storage/mysql/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library") - -go_library( - name = "mysql", - srcs = [ - "request_log_store.go", - "request_store.go", - "storage.go", - ], - importpath = "github.com/uber/submitqueue/stovepipe/extension/storage/mysql", - visibility = ["//visibility:public"], - deps = [ - "//platform/metrics", - "//stovepipe/entity", - "//stovepipe/extension/storage", - "@com_github_go_sql_driver_mysql//:mysql", - "@com_github_uber_go_tally//:tally", - ], -) diff --git a/stovepipe/extension/storage/mysql/request_log_store.go b/stovepipe/extension/storage/mysql/request_log_store.go deleted file mode 100644 index e8aadc3f..00000000 --- a/stovepipe/extension/storage/mysql/request_log_store.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mysql - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - "math/rand/v2" - - "github.com/uber-go/tally" - - "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/stovepipe/entity" - "github.com/uber/submitqueue/stovepipe/extension/storage" -) - -type requestLogStore struct { - db *sql.DB - scope tally.Scope -} - -// NewRequestLogStore creates a new MySQL-backed RequestLogStore. -func NewRequestLogStore(db *sql.DB, scope tally.Scope) storage.RequestLogStore { - return &requestLogStore{db: db, scope: scope} -} - -// Insert appends a new request log record. The primary key is (request_id, timestamp_ms, salt). -// Multiple log entries for the same request can share a timestamp (e.g. concurrent writers or -// millisecond-precision collisions), so a random salt is generated to guarantee uniqueness -// without requiring the caller to manage deduplication. -func (r *requestLogStore) Insert(ctx context.Context, log entity.RequestLog) (retErr error) { - op := metrics.Begin(r.scope, "insert") - defer func() { op.Complete(retErr) }() - - metadataJSON, err := json.Marshal(log.Metadata) - if err != nil { - return fmt.Errorf("failed to marshal metadata for request log request_id=%s: %w", log.RequestID, err) - } - - // Generate a random salt to break primary key ties when two inserts share the same - // (request_id, timestamp_ms). The salt is part of the composite primary key in MySQL - // but is never exposed through the storage interface or returned to callers. - salt := rand.Int64() - - _, err = r.db.ExecContext(ctx, - "INSERT INTO request_log (request_id, timestamp_ms, salt, status, request_version, last_error, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)", - log.RequestID, log.TimestampMs, salt, log.Status, log.RequestVersion, log.LastError, metadataJSON, - ) - if err != nil { - return fmt.Errorf("failed to insert request log for request_id=%s timestamp_ms=%d: %w", log.RequestID, log.TimestampMs, err) - } - - return nil -} - -// List retrieves all request log records for a given request ID, ordered by timestamp ascending. -// Salt is used as a secondary sort key to provide stable ordering for entries that share a -// timestamp, but it is not included in the SELECT columns and never returned to callers. -func (r *requestLogStore) List(ctx context.Context, requestID string) (ret []entity.RequestLog, retErr error) { - op := metrics.Begin(r.scope, "list") - defer func() { op.Complete(retErr) }() - - rows, err := r.db.QueryContext(ctx, - "SELECT request_id, timestamp_ms, status, request_version, last_error, metadata FROM request_log WHERE request_id = ? ORDER BY timestamp_ms ASC, salt ASC", - requestID, - ) - if err != nil { - return nil, fmt.Errorf("failed to list request logs for request_id=%s: %w", requestID, err) - } - defer rows.Close() - - var logs []entity.RequestLog - for rows.Next() { - var log entity.RequestLog - var metadataJSON []byte - - err := rows.Scan(&log.RequestID, &log.TimestampMs, &log.Status, &log.RequestVersion, &log.LastError, &metadataJSON) - if err != nil { - return nil, fmt.Errorf("failed to scan request log row for request_id=%s: %w", requestID, err) - } - - if err := json.Unmarshal(metadataJSON, &log.Metadata); err != nil { - return nil, fmt.Errorf("failed to unmarshal metadata for request log request_id=%s: %w", requestID, err) - } - - logs = append(logs, log) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("failed to iterate request log rows for request_id=%s: %w", requestID, err) - } - - if len(logs) == 0 { - return nil, fmt.Errorf("no request log records for request_id=%s: %w", requestID, storage.ErrNotFound) - } - - return logs, nil -} diff --git a/stovepipe/extension/storage/mysql/request_store.go b/stovepipe/extension/storage/mysql/request_store.go deleted file mode 100644 index 25b803c7..00000000 --- a/stovepipe/extension/storage/mysql/request_store.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mysql - -import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - - "github.com/go-sql-driver/mysql" - "github.com/uber-go/tally" - - "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/stovepipe/entity" - "github.com/uber/submitqueue/stovepipe/extension/storage" -) - -type requestStore struct { - db *sql.DB - scope tally.Scope -} - -// NewRequestStore creates a new MySQL-backed RequestStore. -func NewRequestStore(db *sql.DB, scope tally.Scope) storage.RequestStore { - return &requestStore{db: db, scope: scope} -} - -// Get retrieves a request by ID. Returns ErrNotFound if the request is not found. -func (r *requestStore) Get(ctx context.Context, id string) (ret entity.Request, retErr error) { - op := metrics.Begin(r.scope, "get") - defer func() { op.Complete(retErr) }() - - var req entity.Request - var changeURIsJSON []byte - - err := r.db.QueryRowContext(ctx, - "SELECT id, queue, change_uri, state, version FROM request WHERE id = ?", - id, - ).Scan(&req.ID, &req.Queue, &changeURIsJSON, &req.State, &req.Version) - - if errors.Is(err, sql.ErrNoRows) { - return entity.Request{}, storage.WrapNotFound(err) - } - if err != nil { - return entity.Request{}, fmt.Errorf("failed to get request entity id=%s from the database: %w", id, err) - } - - // Unmarshal the change URIs from JSON - if err := json.Unmarshal(changeURIsJSON, &req.Change.URIs); err != nil { - return entity.Request{}, fmt.Errorf("failed to unmarshal change URIs for request id=%s: %w", id, err) - } - - return req, nil -} - -// Create creates a new request. The request must have a unique ID already assigned. Returns ErrAlreadyExists if the request ID already exists. -func (r *requestStore) Create(ctx context.Context, request entity.Request) (retErr error) { - op := metrics.Begin(r.scope, "create") - defer func() { op.Complete(retErr) }() - - // Marshal the change URIs to JSON - changeURIsJSON, err := json.Marshal(request.Change.URIs) - if err != nil { - return fmt.Errorf("failed to marshal change URIs for request id=%s: %w", request.ID, err) - } - - _, err = r.db.ExecContext(ctx, - "INSERT INTO request (id, queue, change_uri, state, version) VALUES (?, ?, ?, ?, ?)", - request.ID, request.Queue, changeURIsJSON, request.State, request.Version, - ) - if err != nil { - var mysqlErr *mysql.MySQLError - if errors.As(err, &mysqlErr) && mysqlErr.Number == 1062 { - // MySQL error code 1062 is "Duplicate entry". Hopefully it will never change with new versions of MySQL. - // Also it requires to have a single unique index on the table. - return fmt.Errorf("request entity id=%s: %w", request.ID, storage.ErrAlreadyExists) - } - return fmt.Errorf("failed to insert request entity id=%s: %w", request.ID, err) - } - - return nil -} - -// UpdateState updates the state of a request to newState and the version to newVersion -// if the current persisted version matches oldVersion. If versions do not match, returns ErrVersionMismatch. -// Version arithmetic is owned by the caller; this is a pure conditional write. -func (r *requestStore) UpdateState(ctx context.Context, id string, oldVersion, newVersion int32, newState entity.RequestState) (retErr error) { - op := metrics.Begin(r.scope, "update_state") - defer func() { op.Complete(retErr) }() - - result, err := r.db.ExecContext(ctx, - "UPDATE request SET state = ?, version = ? WHERE id = ? AND version = ?", - newState, newVersion, id, oldVersion, - ) - if err != nil { - return fmt.Errorf( - "failed to update request state for id=%q oldVersion=%d newVersion=%d newState=%v: %w", - id, oldVersion, newVersion, newState, err, - ) - } - - rowsAffected, err := result.RowsAffected() - if err != nil { - return fmt.Errorf( - "failed to get rows affected from update for id=%q oldVersion=%d newVersion=%d newState=%v: %w", - id, oldVersion, newVersion, newState, err, - ) - } - - if rowsAffected != 1 { - return fmt.Errorf( - "version mismatch for request update: id=%q expected_version=%d newState=%v: %w", - id, oldVersion, newState, storage.ErrVersionMismatch, - ) - } - - return nil -} diff --git a/stovepipe/extension/storage/mysql/schema/BUILD.bazel b/stovepipe/extension/storage/mysql/schema/BUILD.bazel deleted file mode 100644 index 3412d773..00000000 --- a/stovepipe/extension/storage/mysql/schema/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -filegroup( - name = "schema", - srcs = glob(["*.sql"]), - visibility = ["//visibility:public"], -) diff --git a/stovepipe/extension/storage/mysql/schema/request.sql b/stovepipe/extension/storage/mysql/schema/request.sql deleted file mode 100644 index d187c04e..00000000 --- a/stovepipe/extension/storage/mysql/schema/request.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE IF NOT EXISTS request ( - id VARCHAR(255) NOT NULL, - queue VARCHAR(255) NOT NULL, - change_uri JSON NOT NULL, - state VARCHAR(64) NOT NULL, - version INT NOT NULL, - PRIMARY KEY (id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/stovepipe/extension/storage/mysql/schema/request_log.sql b/stovepipe/extension/storage/mysql/schema/request_log.sql deleted file mode 100644 index 8866eeac..00000000 --- a/stovepipe/extension/storage/mysql/schema/request_log.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TABLE IF NOT EXISTS request_log ( - request_id VARCHAR(255) NOT NULL, - timestamp_ms BIGINT NOT NULL, - salt BIGINT NOT NULL, - status VARCHAR(64) NOT NULL, - request_version INT NOT NULL, - last_error TEXT NOT NULL, - metadata JSON NOT NULL, - PRIMARY KEY (request_id, timestamp_ms, salt) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/stovepipe/extension/storage/mysql/storage.go b/stovepipe/extension/storage/mysql/storage.go deleted file mode 100644 index aadeb24c..00000000 --- a/stovepipe/extension/storage/mysql/storage.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package mysql provides a MySQL-backed implementation of the Stovepipe storage interfaces. -package mysql - -import ( - "database/sql" - - _ "github.com/go-sql-driver/mysql" - "github.com/uber-go/tally" - - "github.com/uber/submitqueue/stovepipe/extension/storage" -) - -type mysqlStorage struct { - db *sql.DB - requestStore storage.RequestStore - requestLogStore storage.RequestLogStore -} - -// NewStorage creates a new MySQL storage. -func NewStorage(db *sql.DB, scope tally.Scope) (storage.Storage, error) { - return &mysqlStorage{ - db: db, - requestStore: NewRequestStore(db, scope.SubScope("request_store")), - requestLogStore: NewRequestLogStore(db, scope.SubScope("request_log_store")), - }, nil -} - -// GetRequestStore returns the MySQL-backed RequestStore. -func (f *mysqlStorage) GetRequestStore() storage.RequestStore { - return f.requestStore -} - -// GetRequestLogStore returns the MySQL-backed RequestLogStore. -func (f *mysqlStorage) GetRequestLogStore() storage.RequestLogStore { - return f.requestLogStore -} - -// Close closes the underlying database connection. -func (f *mysqlStorage) Close() error { - return f.db.Close() -} diff --git a/stovepipe/extension/storage/request_log_store.go b/stovepipe/extension/storage/request_log_store.go deleted file mode 100644 index 85945e2f..00000000 --- a/stovepipe/extension/storage/request_log_store.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -//go:generate mockgen -source=request_log_store.go -destination=mock/request_log_store_mock.go -package=mock - -import ( - "context" - - "github.com/uber/submitqueue/stovepipe/entity" -) - -// RequestLogStore is an interface that defines methods for managing request log records in an append-only database. -// Request logs are used to reconcile request statuses with eventual consistency into a separate database from RequestStore. -// The request log is owned by the gateway. -type RequestLogStore interface { - // Insert appends a new request log record. Timestamps should be generated by the caller and not modified by the implementation. - Insert(ctx context.Context, log entity.RequestLog) error - - // List retrieves all request log records for a given request ID, ordered by timestamp ascending. - // Returns ErrNotFound if no records exist for the given request ID. - List(ctx context.Context, requestID string) ([]entity.RequestLog, error) -} diff --git a/stovepipe/extension/storage/request_store.go b/stovepipe/extension/storage/request_store.go deleted file mode 100644 index 3b9a1636..00000000 --- a/stovepipe/extension/storage/request_store.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -//go:generate mockgen -source=request_store.go -destination=mock/request_store_mock.go -package=mock - -import ( - "context" - - "github.com/uber/submitqueue/stovepipe/entity" -) - -// RequestStore is an interface that defines methods for managing Stovepipe trunk-validation requests in the database. -// It holds the orchestrator's working pipeline state and is owned by the orchestrator. -type RequestStore interface { - // Get retrieves a request by ID. Returns ErrNotFound if the request is not found. - Get(ctx context.Context, id string) (entity.Request, error) - - // Create creates a new request. The request must have a unique ID already assigned. - // Returns ErrAlreadyExists if a request with the same ID already exists. - Create(ctx context.Context, request entity.Request) error - - // UpdateState updates the state of a request to newState and the version to newVersion - // if the current persisted version matches oldVersion. If versions do not match, returns ErrVersionMismatch. - // Version arithmetic is owned by the caller; the store performs a pure conditional write. - UpdateState(ctx context.Context, id string, oldVersion, newVersion int32, newState entity.RequestState) error -} diff --git a/stovepipe/extension/storage/storage.go b/stovepipe/extension/storage/storage.go deleted file mode 100644 index dac5b39e..00000000 --- a/stovepipe/extension/storage/storage.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package storage defines pluggable persistence interfaces for Stovepipe entities. -// Implementations live under extension/storage//. -package storage - -//go:generate mockgen -source=storage.go -destination=mock/storage_mock.go -package=mock - -import ( - "errors" - "fmt" -) - -// ErrNotFound is returned by storage implementations when the requested record is not found in the database. -var ErrNotFound = errors.New("record not found") - -// IsNotFound returns true if any error in the error chain is a ErrNotFound. -func IsNotFound(err error) bool { - return errors.Is(err, ErrNotFound) -} - -// WrapNotFound wraps ErrNotFound with the original error from the storage implementation. -func WrapNotFound(err error) error { - return fmt.Errorf("%w: %w", ErrNotFound, err) -} - -// ErrAlreadyExists is returned by storage implementations when attempting to create a record with an ID that already exists. -var ErrAlreadyExists = errors.New("record already exists") - -// ErrVersionMismatch is returned by storage implementations when the expected entity version does not match the current version of the object. -// This is used to implement an optimistic locking mechanism, allowing multiple clients to update the same entity concurrently -// and either retry or implement idempotent operations. -var ErrVersionMismatch = errors.New("version mismatch") - -// Storage is a factory interface that aggregates all entity stores into a single injectable dependency. -type Storage interface { - // GetRequestStore returns the RequestStore instance. - GetRequestStore() RequestStore - - // GetRequestLogStore returns the RequestLogStore instance. - GetRequestLogStore() RequestLogStore - - // Close closes the storage and all underlying connections. Should only be called once at the end of the program. - Close() error -} diff --git a/stovepipe/gateway/controller/BUILD.bazel b/stovepipe/gateway/controller/BUILD.bazel deleted file mode 100644 index 56a6a494..00000000 --- a/stovepipe/gateway/controller/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "controller", - srcs = [ - "ingest.go", - "ping.go", - ], - importpath = "github.com/uber/submitqueue/stovepipe/gateway/controller", - visibility = ["//visibility:public"], - deps = [ - "//api/stovepipe/gateway/protopb", - "//platform/base/change", - "//platform/base/messagequeue", - "//platform/consumer", - "//platform/errs", - "//platform/extension/counter", - "//platform/metrics", - "//stovepipe/core/topickey", - "//stovepipe/entity", - "@com_github_uber_go_tally//:tally", - "@org_uber_go_zap//:zap", - ], -) - -go_test( - name = "controller_test", - srcs = [ - "ingest_test.go", - "ping_test.go", - ], - embed = [":controller"], - deps = [ - "//api/base/change/protopb", - "//api/stovepipe/gateway/protopb", - "//platform/base/messagequeue", - "//platform/consumer", - "//platform/extension/counter/mock", - "//platform/extension/messagequeue/mock", - "//stovepipe/core/topickey", - "//stovepipe/entity", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@com_github_uber_go_tally//:tally", - "@org_uber_go_mock//gomock", - "@org_uber_go_zap//:zap", - ], -) diff --git a/stovepipe/gateway/controller/ingest.go b/stovepipe/gateway/controller/ingest.go deleted file mode 100644 index 5154d5ae..00000000 --- a/stovepipe/gateway/controller/ingest.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "context" - "errors" - "fmt" - - "github.com/uber-go/tally" - pb "github.com/uber/submitqueue/api/stovepipe/gateway/protopb" - "github.com/uber/submitqueue/platform/base/change" - entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" - "github.com/uber/submitqueue/platform/consumer" - "github.com/uber/submitqueue/platform/errs" - "github.com/uber/submitqueue/platform/extension/counter" - "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/stovepipe/core/topickey" - "github.com/uber/submitqueue/stovepipe/entity" - "go.uber.org/zap" -) - -// ErrInvalidRequest is returned when the request fails validation. -// This error should be mapped to codes.InvalidArgument at the gRPC layer. -var ErrInvalidRequest = errs.NewUserError(errors.New("invalid request")) - -// IsInvalidRequest returns true if any error in the error chain is ErrInvalidRequest. -func IsInvalidRequest(err error) bool { - return errors.Is(err, ErrInvalidRequest) -} - -// IngestController handles ingest business logic for the stovepipe gateway. -type IngestController struct { - logger *zap.SugaredLogger - metricsScope tally.Scope - counter counter.Counter - registry consumer.TopicRegistry -} - -// NewIngestController creates a new instance of the stovepipe ingest controller. -// The controller publishes ingest requests to the topic registered under -// topickey.TopicKeyStart in the registry. -func NewIngestController(logger *zap.SugaredLogger, scope tally.Scope, counter counter.Counter, registry consumer.TopicRegistry) *IngestController { - return &IngestController{ - logger: logger, - metricsScope: scope.SubScope("ingest_controller"), - counter: counter, - registry: registry, - } -} - -// Ingest validates the request, generates a SPID, publishes the ingest request -// to the pipeline queue, and returns the SPID for tracking. -func (c *IngestController) Ingest(ctx context.Context, req *pb.IngestRequest) (resp *pb.IngestResponse, retErr error) { - const opName = "ingest" - - op := metrics.Begin(c.metricsScope, opName) - defer func() { op.Complete(retErr) }() - - if req.Queue == "" { - return nil, fmt.Errorf("IngestController requires the request to have a queue name specified: %w", ErrInvalidRequest) - } - if req.Change == nil || len(req.Change.Uris) == 0 { - return nil, fmt.Errorf("IngestController requires the request to have at least one change URI specified: %w", ErrInvalidRequest) - } - - queue := req.Queue - - seq, err := c.counter.Next(ctx, "ingest/"+queue) - if err != nil { - return nil, fmt.Errorf("IngestController failed to generate SPID for queue=%s: %w", queue, err) - } - - ingestRequest := entity.IngestRequest{ - ID: fmt.Sprintf("%s/%d", queue, seq), - Queue: queue, - Change: change.Change{URIs: req.Change.GetUris()}, - } - - c.logger.Debugw("ingest request created", - "queue", queue, - "spid", ingestRequest.ID, - "change_uris", ingestRequest.Change.URIs, - ) - - if err := c.publishToQueue(ctx, ingestRequest); err != nil { - return nil, fmt.Errorf("IngestController failed to publish request to queue: %w", err) - } - - c.logger.Infow("ingest request published to queue", - "queue", queue, - "spid", ingestRequest.ID, - "topic_key", topickey.TopicKeyStart, - ) - metrics.NamedCounter(c.metricsScope, opName, "publish_success", 1) - - return &pb.IngestResponse{ - Spid: ingestRequest.ID, - }, nil -} - -// publishToQueue serializes the ingest request and publishes it to the start topic. -func (c *IngestController) publishToQueue(ctx context.Context, ingestRequest entity.IngestRequest) error { - payload, err := ingestRequest.ToBytes() - if err != nil { - return fmt.Errorf("failed to serialize ingest request: %w", err) - } - - msg := entityqueue.NewMessage(ingestRequest.ID, payload, ingestRequest.Queue, nil) - - q, ok := c.registry.Queue(topickey.TopicKeyStart) - if !ok { - return fmt.Errorf("no queue registered for topic key %s", topickey.TopicKeyStart) - } - - topicName, ok := c.registry.TopicName(topickey.TopicKeyStart) - if !ok { - return fmt.Errorf("no topic name registered for topic key %s", topickey.TopicKeyStart) - } - - if err := q.Publisher().Publish(ctx, topicName, msg); err != nil { - return fmt.Errorf("failed to publish ingest request message: %w", err) - } - - return nil -} diff --git a/stovepipe/gateway/controller/ingest_test.go b/stovepipe/gateway/controller/ingest_test.go deleted file mode 100644 index 19d2c644..00000000 --- a/stovepipe/gateway/controller/ingest_test.go +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally" - changepb "github.com/uber/submitqueue/api/base/change/protopb" - pb "github.com/uber/submitqueue/api/stovepipe/gateway/protopb" - entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" - "github.com/uber/submitqueue/platform/consumer" - countermock "github.com/uber/submitqueue/platform/extension/counter/mock" - queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" - "github.com/uber/submitqueue/stovepipe/core/topickey" - "github.com/uber/submitqueue/stovepipe/entity" - "go.uber.org/mock/gomock" - "go.uber.org/zap" -) - -const ( - testGitURI = "git://git.example.com/uber/monorepo/refs%2Fheads%2Fmain/abcdef0123456789abcdef0123456789abcdef01" - testQueueName = "stovepipe-monorepo" -) - -// newIngestTestRegistry builds a TopicRegistry for TopicKeyStart wired to a mock -// Queue/Publisher and returns both so callers can set EXPECT() on the publisher. -func newIngestTestRegistry(t *testing.T, ctrl *gomock.Controller) (consumer.TopicRegistry, *queuemock.MockPublisher) { - t.Helper() - pub := queuemock.NewMockPublisher(ctrl) - q := queuemock.NewMockQueue(ctrl) - q.EXPECT().Publisher().Return(pub).AnyTimes() - - registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ - {Key: topickey.TopicKeyStart, Name: "start", Queue: q}, - }) - require.NoError(t, err) - return registry, pub -} - -// newIngestTestRegistryWithNoopPublisher returns a registry whose publisher -// silently accepts any Publish call. -func newIngestTestRegistryWithNoopPublisher(t *testing.T, ctrl *gomock.Controller) consumer.TopicRegistry { - t.Helper() - registry, pub := newIngestTestRegistry(t, ctrl) - pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - return registry -} - -func TestNewIngestController(t *testing.T) { - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, newIngestTestRegistryWithNoopPublisher(t, ctrl)) - require.NotNil(t, c) -} - -func TestIngest_ReturnsSPID(t *testing.T) { - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(1), nil) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, newIngestTestRegistryWithNoopPublisher(t, ctrl)) - resp, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: testQueueName, - Change: &changepb.Change{Uris: []string{testGitURI}}, - }) - - require.NoError(t, err) - assert.Equal(t, "stovepipe-monorepo/1", resp.Spid) -} - -func TestIngest_CounterDomainIncludesQueue(t *testing.T) { - var capturedDomain string - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - cnt.EXPECT().Next(gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, domain string) (int64, error) { - capturedDomain = domain - return 1, nil - }, - ) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, newIngestTestRegistryWithNoopPublisher(t, ctrl)) - _, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: testQueueName, - Change: &changepb.Change{Uris: []string{testGitURI}}, - }) - - require.NoError(t, err) - assert.Equal(t, "ingest/"+testQueueName, capturedDomain) -} - -func TestIngest_ReturnsErrorOnCounterFailure(t *testing.T) { - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(0), fmt.Errorf("counter unavailable")) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, newIngestTestRegistryWithNoopPublisher(t, ctrl)) - _, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: testQueueName, - Change: &changepb.Change{Uris: []string{testGitURI}}, - }) - - require.Error(t, err) -} - -func TestIngest_ReturnsErrorOnEmptyQueue(t *testing.T) { - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, newIngestTestRegistryWithNoopPublisher(t, ctrl)) - _, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: "", - Change: &changepb.Change{Uris: []string{testGitURI}}, - }) - - require.Error(t, err) - assert.True(t, IsInvalidRequest(err)) -} - -func TestIngest_ReturnsErrorOnNilChange(t *testing.T) { - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, newIngestTestRegistryWithNoopPublisher(t, ctrl)) - _, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: testQueueName, - Change: nil, - }) - - require.Error(t, err) - assert.True(t, IsInvalidRequest(err)) -} - -func TestIngest_ReturnsErrorOnEmptyChangeURIs(t *testing.T) { - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, newIngestTestRegistryWithNoopPublisher(t, ctrl)) - _, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: testQueueName, - Change: &changepb.Change{Uris: []string{}}, - }) - - require.Error(t, err) - assert.True(t, IsInvalidRequest(err)) -} - -func TestIngest_PublishesToQueue(t *testing.T) { - var publishedTopic string - var publishedMessage entityqueue.Message - - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(42), nil) - - registry, publisher := newIngestTestRegistry(t, ctrl) - publisher.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, topic string, msg entityqueue.Message) error { - publishedTopic = topic - publishedMessage = msg - return nil - }, - ) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, registry) - resp, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: testQueueName, - Change: &changepb.Change{Uris: []string{testGitURI}}, - }) - - require.NoError(t, err) - assert.Equal(t, "stovepipe-monorepo/42", resp.Spid) - assert.Equal(t, "start", publishedTopic) - assert.Equal(t, "stovepipe-monorepo/42", publishedMessage.ID) - assert.Equal(t, testQueueName, publishedMessage.PartitionKey) - - deserialized, err := entity.IngestRequestFromBytes(publishedMessage.Payload) - require.NoError(t, err) - assert.Equal(t, "stovepipe-monorepo/42", deserialized.ID) - assert.Equal(t, testQueueName, deserialized.Queue) - assert.Equal(t, []string{testGitURI}, deserialized.Change.URIs) -} - -func TestIngest_ReturnsErrorOnPublishFailure(t *testing.T) { - ctrl := gomock.NewController(t) - cnt := countermock.NewMockCounter(ctrl) - cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(1), nil) - - registry, publisher := newIngestTestRegistry(t, ctrl) - publisher.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("queue unavailable")) - - c := NewIngestController(zap.NewNop().Sugar(), tally.NoopScope, cnt, registry) - _, err := c.Ingest(context.Background(), &pb.IngestRequest{ - Queue: testQueueName, - Change: &changepb.Change{Uris: []string{testGitURI}}, - }) - - require.Error(t, err) -} diff --git a/stovepipe/orchestrator/controller/ping.go b/stovepipe/orchestrator/controller/ping.go deleted file mode 100644 index bd254cdb..00000000 --- a/stovepipe/orchestrator/controller/ping.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "context" - "os" - "time" - - "github.com/uber-go/tally" - pb "github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb" - "github.com/uber/submitqueue/platform/metrics" - "go.uber.org/zap" -) - -// PingController handles ping business logic for the Stovepipe orchestrator. -type PingController struct { - logger *zap.Logger - metricsScope tally.Scope -} - -// NewPingController creates a new instance of the Stovepipe orchestrator ping controller. -func NewPingController(logger *zap.Logger, scope tally.Scope) *PingController { - return &PingController{ - logger: logger, - metricsScope: scope, - } -} - -// Ping handles the ping request and returns a response. -func (c *PingController) Ping(ctx context.Context, req *pb.PingRequest) (resp *pb.PingResponse, retErr error) { - const opName = "ping" - - op := metrics.Begin(c.metricsScope, opName) - defer func() { op.Complete(retErr) }() - - message := "pong!" - isEcho := false - if req.Message != "" { - message = "echo: " + req.Message - isEcho = true - metrics.NamedCounter(c.metricsScope, opName, "echo_requests", 1) - } - - hostname, _ := os.Hostname() - - c.logger.Info("ping request received", - zap.String("message", req.Message), - zap.Bool("is_echo", isEcho), - zap.String("hostname", hostname), - ) - - return &pb.PingResponse{ - Message: message, - ServiceName: "stovepipe-orchestrator", - Timestamp: time.Now().Unix(), - Hostname: hostname, - }, nil -} diff --git a/stovepipe/orchestrator/controller/ping_test.go b/stovepipe/orchestrator/controller/ping_test.go deleted file mode 100644 index b09d6d99..00000000 --- a/stovepipe/orchestrator/controller/ping_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally" - pb "github.com/uber/submitqueue/api/stovepipe/orchestrator/protopb" - "go.uber.org/zap" -) - -func TestNewPingController(t *testing.T) { - ctrl := NewPingController(zap.NewNop(), tally.NoopScope) - require.NotNil(t, ctrl) -} - -func TestPing_DefaultMessage(t *testing.T) { - ctrl := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - req := &pb.PingRequest{} - resp, err := ctrl.Ping(ctx, req) - - require.NoError(t, err) - assert.Equal(t, "pong!", resp.Message) -} - -func TestPing_ServiceName(t *testing.T) { - ctrl := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - req := &pb.PingRequest{} - resp, err := ctrl.Ping(ctx, req) - - require.NoError(t, err) - assert.Equal(t, "stovepipe-orchestrator", resp.ServiceName) -} - -func TestPing_Timestamp(t *testing.T) { - ctrl := NewPingController(zap.NewNop(), tally.NoopScope) - ctx := context.Background() - - before := time.Now().Unix() - req := &pb.PingRequest{} - resp, err := ctrl.Ping(ctx, req) - after := time.Now().Unix() - - require.NoError(t, err) - assert.GreaterOrEqual(t, resp.Timestamp, before) - assert.LessOrEqual(t, resp.Timestamp, after) -} diff --git a/stovepipe/orchestrator/controller/start/BUILD.bazel b/stovepipe/orchestrator/controller/start/BUILD.bazel deleted file mode 100644 index 1ea8a696..00000000 --- a/stovepipe/orchestrator/controller/start/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "start", - srcs = ["start.go"], - importpath = "github.com/uber/submitqueue/stovepipe/orchestrator/controller/start", - visibility = ["//visibility:public"], - deps = [ - "//platform/base/messagequeue", - "//platform/consumer", - "//platform/metrics", - "//stovepipe/core/topickey", - "//stovepipe/entity", - "@com_github_uber_go_tally//:tally", - "@org_uber_go_zap//:zap", - ], -) - -go_test( - name = "start_test", - srcs = ["start_test.go"], - embed = [":start"], - deps = [ - "//platform/base/change", - "//platform/base/messagequeue", - "//platform/consumer", - "//platform/extension/messagequeue/mock", - "//stovepipe/core/topickey", - "//stovepipe/entity", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@com_github_uber_go_tally//:tally", - "@org_uber_go_mock//gomock", - "@org_uber_go_zap//zaptest", - ], -) diff --git a/stovepipe/orchestrator/controller/start/start.go b/stovepipe/orchestrator/controller/start/start.go deleted file mode 100644 index 955eb5ee..00000000 --- a/stovepipe/orchestrator/controller/start/start.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package start - -import ( - "context" - "fmt" - - "github.com/uber-go/tally" - entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" - "github.com/uber/submitqueue/platform/consumer" - "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/stovepipe/core/topickey" - entity "github.com/uber/submitqueue/stovepipe/entity" - "go.uber.org/zap" -) - -// Controller handles start queue messages. It is the pipeline entry point: it -// deserializes the gateway-produced ingest request and forwards it to the validate -// stage, propagating the envelope partition key for ordering. -// -// The ordering key is decided once at ingestion and carried through the pipeline. -// -// Currently a forwarding stub. Per the Stovepipe workflow RFC, start will also record -// each commit as `unknown` (keyed by SHA, making ingest idempotent across the webhook -// and poll producers) and emit status + log events. Until a commit store exists it -// forwards the full request downstream rather than a thin ID reference — submitqueue -// persists the request here and forwards only the ID, but Stovepipe has no store yet. - -var _ consumer.Controller = (*Controller)(nil) - -type Controller struct { - logger *zap.SugaredLogger - metricsScope tally.Scope - registry consumer.TopicRegistry - topicKey consumer.TopicKey - consumerGroup string -} - -// Params are the parameters for creating a new start controller. -type Params struct { - Registry consumer.TopicRegistry - TopicKey consumer.TopicKey - ConsumerGroup string - - Scope tally.Scope - Logger *zap.SugaredLogger -} - -// NewController creates a new start controller for the orchestrator. -func NewController(p Params) *Controller { - return &Controller{ - logger: p.Logger.Named("start_controller"), - metricsScope: p.Scope.SubScope("start_controller"), - registry: p.Registry, - topicKey: p.TopicKey, - consumerGroup: p.ConsumerGroup, - } -} - -// Process deserializes the ingest request and forwards it to the validate stage. -func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) (retErr error) { - const opName = "process" - - op := metrics.Begin(c.metricsScope, opName) - defer func() { op.Complete(retErr) }() - - msg := delivery.Message() - - request, err := entity.IngestRequestFromBytes(msg.Payload) - if err != nil { - metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) - // Non-retryable: malformed messages will never succeed regardless of retry count. - return fmt.Errorf("failed to deserialize ingest request: %w", err) - } - - // The ordering key lives on the message envelope, stamped by the gateway at - // ingestion; the controller propagates it verbatim to the next stage. - partitionKey := msg.PartitionKey - if partitionKey == "" { - metrics.NamedCounter(c.metricsScope, opName, "missing_partition_key", 1) - return fmt.Errorf("ingest request %s is missing a partition key (must be stamped by the producer)", request.ID) - } - - c.logger.Infow("received ingest request", - "spid", request.ID, - "queue", request.Queue, - "change_uris", request.Change.URIs, - "change_count", len(request.Change.URIs), - "attempt", delivery.Attempt(), - "partition_key", partitionKey, - ) - - // Core logic to be added here: - // - Record each commit as `unknown` (keyed by SHA) - // - Emit status + log events - - if err := c.publish(ctx, topickey.TopicKeyValidate, request, partitionKey); err != nil { - metrics.NamedCounter(c.metricsScope, opName, "publish_errors", 1) - return fmt.Errorf("failed to publish to validate: %w", err) - } - - c.logger.Infow("published ingest request to validate", - "spid", request.ID, - "topic_key", topickey.TopicKeyValidate, - ) - - return nil -} - -func (c *Controller) publish(ctx context.Context, key consumer.TopicKey, request entity.IngestRequest, partitionKey string) error { - payload, err := request.ToBytes() - if err != nil { - return fmt.Errorf("failed to serialize ingest request: %w", err) - } - - msg := entityqueue.NewMessage(request.ID, payload, partitionKey, nil) - - q, ok := c.registry.Queue(key) - if !ok { - return fmt.Errorf("no queue registered for topic key %s", key) - } - - topicName, ok := c.registry.TopicName(key) - if !ok { - return fmt.Errorf("no topic name registered for topic key %s", key) - } - - if err := q.Publisher().Publish(ctx, topicName, msg); err != nil { - return fmt.Errorf("failed to publish message: %w", err) - } - - return nil -} - -// Name returns the controller name for logging and metrics. -func (c *Controller) Name() string { - return "start" -} - -// TopicKey returns the topic key this controller subscribes to. -func (c *Controller) TopicKey() consumer.TopicKey { - return c.topicKey -} - -// ConsumerGroup returns the consumer group for offset tracking. -func (c *Controller) ConsumerGroup() string { - return c.consumerGroup -} diff --git a/stovepipe/orchestrator/controller/start/start_test.go b/stovepipe/orchestrator/controller/start/start_test.go deleted file mode 100644 index 6496ca9f..00000000 --- a/stovepipe/orchestrator/controller/start/start_test.go +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package start - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally" - "github.com/uber/submitqueue/platform/base/change" - entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" - "github.com/uber/submitqueue/platform/consumer" - queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" - "github.com/uber/submitqueue/stovepipe/core/topickey" - entity "github.com/uber/submitqueue/stovepipe/entity" - "go.uber.org/mock/gomock" - "go.uber.org/zap/zaptest" -) - -const ( - testURI = "git://git.example.com/uber/monorepo/refs%2Fheads%2Fmain/abcdef0123456789abcdef0123456789abcdef01" - testSPID = "stovepipe-monorepo/1" - testQueue = "stovepipe-monorepo" - testPartitionKey = "stovepipe-monorepo" -) - -// captureRegistry builds a topic registry whose validate publisher records the -// last message it received into captured (when non-nil) and returns publishErr. -func captureRegistry(t *testing.T, ctrl *gomock.Controller, publishErr error, captured *entityqueue.Message) consumer.TopicRegistry { - t.Helper() - - mockPub := queuemock.NewMockPublisher(ctrl) - mockPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ string, msg entityqueue.Message) error { - if captured != nil { - *captured = msg - } - return publishErr - }, - ).AnyTimes() - - mockQ := queuemock.NewMockQueue(ctrl) - mockQ.EXPECT().Publisher().Return(mockPub).AnyTimes() - - registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ - {Key: topickey.TopicKeyValidate, Name: "validate", Queue: mockQ}, - }) - require.NoError(t, err) - return registry -} - -func newTestController(t *testing.T, ctrl *gomock.Controller, publishErr error, captured *entityqueue.Message) *Controller { - t.Helper() - - return NewController(Params{ - Logger: zaptest.NewLogger(t).Sugar(), - Scope: tally.NoopScope, - Registry: captureRegistry(t, ctrl, publishErr, captured), - TopicKey: topickey.TopicKeyStart, - ConsumerGroup: "orchestrator-start", - }) -} - -// makeDelivery builds a delivery whose envelope carries partitionKey, the -// ordering key the producer stamps at ingestion. -func makeDelivery(t *testing.T, ctrl *gomock.Controller, payload []byte, partitionKey string) *queuemock.MockDelivery { - t.Helper() - - msg := entityqueue.NewMessage(testSPID, payload, partitionKey, nil) - delivery := queuemock.NewMockDelivery(ctrl) - delivery.EXPECT().Message().Return(msg).AnyTimes() - delivery.EXPECT().Attempt().Return(1).AnyTimes() - return delivery -} - -// validPayload is an ingest request as the gateway produces it: identity, queue, and -// the change URIs. The ordering key rides on the message envelope, not the payload. -func validPayload(t *testing.T) []byte { - t.Helper() - payload, err := entity.IngestRequest{ - ID: testSPID, - Queue: testQueue, - Change: change.Change{URIs: []string{testURI}}, - }.ToBytes() - require.NoError(t, err) - return payload -} - -func TestNewController(t *testing.T) { - ctrl := gomock.NewController(t) - - controller := newTestController(t, ctrl, nil, nil) - - require.NotNil(t, controller) - assert.Equal(t, topickey.TopicKeyStart, controller.TopicKey()) - assert.Equal(t, "orchestrator-start", controller.ConsumerGroup()) - assert.Equal(t, "start", controller.Name()) -} - -func TestController_Process_PublishesToValidate(t *testing.T) { - ctrl := gomock.NewController(t) - - var captured entityqueue.Message - controller := newTestController(t, ctrl, nil, &captured) - delivery := makeDelivery(t, ctrl, validPayload(t), testPartitionKey) - - require.NoError(t, controller.Process(context.Background(), delivery)) - - // start forwards the request to validate, keyed by spid for idempotency and - // propagating the envelope partition key verbatim to the next hop. - assert.Equal(t, testSPID, captured.ID) - assert.Equal(t, testPartitionKey, captured.PartitionKey) - - forwarded, err := entity.IngestRequestFromBytes(captured.Payload) - require.NoError(t, err) - assert.Equal(t, testSPID, forwarded.ID) - assert.Equal(t, []string{testURI}, forwarded.Change.URIs) -} - -func TestController_Process_Errors(t *testing.T) { - tests := []struct { - name string - payload []byte - partitionKey string - }{ - {name: "invalid json", payload: []byte(`{"invalid": json"}`), partitionKey: testPartitionKey}, - {name: "missing id", payload: []byte(`{"queue":"q","change":{"uris":["git://x/y/z/sha"]}}`), partitionKey: testPartitionKey}, - {name: "no uris", payload: []byte(`{"id":"q/1","queue":"q","change":{"uris":[]}}`), partitionKey: testPartitionKey}, - // Valid request, but the producer failed to stamp an envelope partition key. - {name: "missing partition key", payload: validPayload(t), partitionKey: ""}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - controller := newTestController(t, ctrl, nil, nil) - delivery := makeDelivery(t, ctrl, tt.payload, tt.partitionKey) - - require.Error(t, controller.Process(context.Background(), delivery)) - }) - } -} - -func TestController_Process_PublishError(t *testing.T) { - ctrl := gomock.NewController(t) - - controller := newTestController(t, ctrl, assert.AnError, nil) - delivery := makeDelivery(t, ctrl, validPayload(t), testPartitionKey) - - require.Error(t, controller.Process(context.Background(), delivery)) -} diff --git a/stovepipe/orchestrator/controller/validate/BUILD.bazel b/stovepipe/orchestrator/controller/validate/BUILD.bazel deleted file mode 100644 index 40dc4141..00000000 --- a/stovepipe/orchestrator/controller/validate/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "validate", - srcs = ["validate.go"], - importpath = "github.com/uber/submitqueue/stovepipe/orchestrator/controller/validate", - visibility = ["//visibility:public"], - deps = [ - "//platform/base/messagequeue", - "//platform/consumer", - "//platform/metrics", - "//stovepipe/core/topickey", - "//stovepipe/entity", - "@com_github_uber_go_tally//:tally", - "@org_uber_go_zap//:zap", - ], -) - -go_test( - name = "validate_test", - srcs = ["validate_test.go"], - embed = [":validate"], - deps = [ - "//platform/base/change", - "//platform/base/messagequeue", - "//platform/consumer", - "//platform/extension/messagequeue/mock", - "//stovepipe/core/topickey", - "//stovepipe/entity", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@com_github_uber_go_tally//:tally", - "@org_uber_go_mock//gomock", - "@org_uber_go_zap//zaptest", - ], -) diff --git a/stovepipe/orchestrator/controller/validate/validate.go b/stovepipe/orchestrator/controller/validate/validate.go deleted file mode 100644 index cb5e8f46..00000000 --- a/stovepipe/orchestrator/controller/validate/validate.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package validate - -import ( - "context" - "fmt" - - "github.com/uber-go/tally" - entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" - "github.com/uber/submitqueue/platform/consumer" - "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/stovepipe/core/topickey" - entity "github.com/uber/submitqueue/stovepipe/entity" - "go.uber.org/zap" -) - -// Controller handles validate queue messages. It consumes the validate topic and -// forwards the ingest request to the batch stage, propagating the envelope partition -// key for ordering. -// -// This step will include any validation activities prior to adding the commit to a batch. -// -// The ordering key is decided once at ingestion and carried through the pipeline. -// -// Currently a forwarding stub. - -var _ consumer.Controller = (*Controller)(nil) - -type Controller struct { - logger *zap.SugaredLogger - metricsScope tally.Scope - registry consumer.TopicRegistry - topicKey consumer.TopicKey - consumerGroup string -} - -// Params are the parameters for creating a new validate controller. -type Params struct { - Registry consumer.TopicRegistry - TopicKey consumer.TopicKey - ConsumerGroup string - - Scope tally.Scope - Logger *zap.SugaredLogger -} - -// NewController creates a new validate controller for the orchestrator. -func NewController(p Params) *Controller { - return &Controller{ - logger: p.Logger.Named("validate_controller"), - metricsScope: p.Scope.SubScope("validate_controller"), - registry: p.Registry, - topicKey: p.TopicKey, - consumerGroup: p.ConsumerGroup, - } -} - -// Process validates the ingest request and forwards it to the batch stage. -func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) (retErr error) { - const opName = "process" - - op := metrics.Begin(c.metricsScope, opName) - defer func() { op.Complete(retErr) }() - - msg := delivery.Message() - - request, err := entity.IngestRequestFromBytes(msg.Payload) - if err != nil { - metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) - // Non-retryable: malformed messages will never succeed regardless of retry count. - return fmt.Errorf("failed to deserialize ingest request: %w", err) - } - - // The ordering key lives on the message envelope, stamped by the gateway at - // ingestion; the controller propagates it verbatim to the next stage. - partitionKey := msg.PartitionKey - if partitionKey == "" { - metrics.NamedCounter(c.metricsScope, opName, "missing_partition_key", 1) - return fmt.Errorf("ingest request %s is missing a partition key (must be stamped by the producer)", request.ID) - } - - c.logger.Infow("received ingest request", - "spid", request.ID, - "queue", request.Queue, - "change_uris", request.Change.URIs, - "change_count", len(request.Change.URIs), - "attempt", delivery.Attempt(), - "partition_key", partitionKey, - ) - - // Core logic to be added here: - // - Validation before publishing to batch - // - Emit status + log events - - if err := c.publish(ctx, topickey.TopicKeyBatch, request, partitionKey); err != nil { - metrics.NamedCounter(c.metricsScope, opName, "publish_errors", 1) - return fmt.Errorf("failed to publish to batch: %w", err) - } - - c.logger.Infow("published ingest request to batch", - "spid", request.ID, - "topic_key", topickey.TopicKeyBatch, - ) - - return nil -} - -func (c *Controller) publish(ctx context.Context, key consumer.TopicKey, request entity.IngestRequest, partitionKey string) error { - payload, err := request.ToBytes() - if err != nil { - return fmt.Errorf("failed to serialize ingest request: %w", err) - } - - msg := entityqueue.NewMessage(request.ID, payload, partitionKey, nil) - - q, ok := c.registry.Queue(key) - if !ok { - return fmt.Errorf("no queue registered for topic key %s", key) - } - - topicName, ok := c.registry.TopicName(key) - if !ok { - return fmt.Errorf("no topic name registered for topic key %s", key) - } - - if err := q.Publisher().Publish(ctx, topicName, msg); err != nil { - return fmt.Errorf("failed to publish message: %w", err) - } - - return nil -} - -// Name returns the controller name for logging and metrics. -func (c *Controller) Name() string { - return "validate" -} - -// TopicKey returns the topic key this controller subscribes to. -func (c *Controller) TopicKey() consumer.TopicKey { - return c.topicKey -} - -// ConsumerGroup returns the consumer group for offset tracking. -func (c *Controller) ConsumerGroup() string { - return c.consumerGroup -} diff --git a/stovepipe/orchestrator/controller/validate/validate_test.go b/stovepipe/orchestrator/controller/validate/validate_test.go deleted file mode 100644 index 7b342b1a..00000000 --- a/stovepipe/orchestrator/controller/validate/validate_test.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package validate - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber-go/tally" - "github.com/uber/submitqueue/platform/base/change" - entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" - "github.com/uber/submitqueue/platform/consumer" - queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" - "github.com/uber/submitqueue/stovepipe/core/topickey" - entity "github.com/uber/submitqueue/stovepipe/entity" - "go.uber.org/mock/gomock" - "go.uber.org/zap/zaptest" -) - -const ( - testURI = "git://git.example.com/uber/monorepo/refs%2Fheads%2Fmain/abcdef0123456789abcdef0123456789abcdef01" - testSPID = "stovepipe-monorepo/1" - testQueue = "stovepipe-monorepo" - testPartitionKey = "stovepipe-monorepo" -) - -// captureRegistry builds a topic registry whose batch publisher records the -// last message it received into captured (when non-nil) and returns publishErr. -func captureRegistry(t *testing.T, ctrl *gomock.Controller, publishErr error, captured *entityqueue.Message) consumer.TopicRegistry { - t.Helper() - - mockPub := queuemock.NewMockPublisher(ctrl) - mockPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, _ string, msg entityqueue.Message) error { - if captured != nil { - *captured = msg - } - return publishErr - }, - ).AnyTimes() - - mockQ := queuemock.NewMockQueue(ctrl) - mockQ.EXPECT().Publisher().Return(mockPub).AnyTimes() - - registry, err := consumer.NewTopicRegistry([]consumer.TopicConfig{ - {Key: topickey.TopicKeyBatch, Name: "batch", Queue: mockQ}, - }) - require.NoError(t, err) - return registry -} - -func newTestController(t *testing.T, ctrl *gomock.Controller, publishErr error, captured *entityqueue.Message) *Controller { - t.Helper() - - return NewController(Params{ - Logger: zaptest.NewLogger(t).Sugar(), - Scope: tally.NoopScope, - Registry: captureRegistry(t, ctrl, publishErr, captured), - TopicKey: topickey.TopicKeyValidate, - ConsumerGroup: "orchestrator-validate", - }) -} - -// makeDelivery builds a delivery whose envelope carries partitionKey, the -// ordering key the producer stamps at ingestion. -func makeDelivery(t *testing.T, ctrl *gomock.Controller, payload []byte, partitionKey string) *queuemock.MockDelivery { - t.Helper() - - msg := entityqueue.NewMessage(testSPID, payload, partitionKey, nil) - delivery := queuemock.NewMockDelivery(ctrl) - delivery.EXPECT().Message().Return(msg).AnyTimes() - delivery.EXPECT().Attempt().Return(1).AnyTimes() - return delivery -} - -// validPayload is an ingest request as the upstream stage forwards it: identity, -// queue, and the change URIs. The ordering key rides on the message envelope, not -// the payload. -func validPayload(t *testing.T) []byte { - t.Helper() - payload, err := entity.IngestRequest{ - ID: testSPID, - Queue: testQueue, - Change: change.Change{URIs: []string{testURI}}, - }.ToBytes() - require.NoError(t, err) - return payload -} - -func TestNewController(t *testing.T) { - ctrl := gomock.NewController(t) - - controller := newTestController(t, ctrl, nil, nil) - - require.NotNil(t, controller) - assert.Equal(t, topickey.TopicKeyValidate, controller.TopicKey()) - assert.Equal(t, "orchestrator-validate", controller.ConsumerGroup()) - assert.Equal(t, "validate", controller.Name()) -} - -func TestController_Process_PublishesToBatch(t *testing.T) { - ctrl := gomock.NewController(t) - - var captured entityqueue.Message - controller := newTestController(t, ctrl, nil, &captured) - delivery := makeDelivery(t, ctrl, validPayload(t), testPartitionKey) - - require.NoError(t, controller.Process(context.Background(), delivery)) - - // validate forwards the request to batch, keyed by spid for idempotency and - // propagating the envelope partition key verbatim to the next hop. - assert.Equal(t, testSPID, captured.ID) - assert.Equal(t, testPartitionKey, captured.PartitionKey) - - forwarded, err := entity.IngestRequestFromBytes(captured.Payload) - require.NoError(t, err) - assert.Equal(t, testSPID, forwarded.ID) - assert.Equal(t, []string{testURI}, forwarded.Change.URIs) -} - -func TestController_Process_Errors(t *testing.T) { - tests := []struct { - name string - payload []byte - partitionKey string - }{ - {name: "invalid json", payload: []byte(`{"invalid": json"}`), partitionKey: testPartitionKey}, - {name: "missing id", payload: []byte(`{"queue":"q","change":{"uris":["git://x/y/z/sha"]}}`), partitionKey: testPartitionKey}, - {name: "no uris", payload: []byte(`{"id":"q/1","queue":"q","change":{"uris":[]}}`), partitionKey: testPartitionKey}, - // Valid request, but the producer failed to stamp an envelope partition key. - {name: "missing partition key", payload: validPayload(t), partitionKey: ""}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - controller := newTestController(t, ctrl, nil, nil) - delivery := makeDelivery(t, ctrl, tt.payload, tt.partitionKey) - - require.Error(t, controller.Process(context.Background(), delivery)) - }) - } -} - -func TestController_Process_PublishError(t *testing.T) { - ctrl := gomock.NewController(t) - - controller := newTestController(t, ctrl, assert.AnError, nil) - delivery := makeDelivery(t, ctrl, validPayload(t), testPartitionKey) - - require.Error(t, controller.Process(context.Background(), delivery)) -} diff --git a/test/integration/stovepipe/gateway/BUILD.bazel b/test/integration/stovepipe/BUILD.bazel similarity index 69% rename from test/integration/stovepipe/gateway/BUILD.bazel rename to test/integration/stovepipe/BUILD.bazel index a984fb29..f91f8042 100644 --- a/test/integration/stovepipe/gateway/BUILD.bazel +++ b/test/integration/stovepipe/BUILD.bazel @@ -1,20 +1,19 @@ load("@rules_go//go:def.bzl", "go_test") go_test( - name = "gateway_test", + name = "stovepipe_test", srcs = ["suite_test.go"], data = [ "//:MODULE.bazel", "//:go.mod", - "//example/stovepipe/gateway/server:docker-compose.yml", - "//platform/extension/messagequeue/mysql/schema", + "//example/stovepipe:docker-compose.yml", ], tags = [ "external", "integration", ], deps = [ - "//api/stovepipe/gateway/protopb", + "//api/stovepipe/protopb", "//test/testutil", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", diff --git a/test/integration/stovepipe/gateway/suite_test.go b/test/integration/stovepipe/gateway/suite_test.go deleted file mode 100644 index 9d63ecd3..00000000 --- a/test/integration/stovepipe/gateway/suite_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gateway - -// Stovepipe gateway integration tests -// -// These tests use compose from example/stovepipe/gateway/server/docker-compose.yml -// and require a pre-built Linux gateway binary (make integration-test runs -// //test/integration/... and builds all Linux binaries via build-all-linux). -// Only the queue database schema is applied (no SubmitQueue app schema until -// Stovepipe has its own storage schema). -// -// Run with: -// make integration-test -// or only this package: -// bazel test //test/integration/stovepipe/gateway:gateway_test - -import ( - "context" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - pb "github.com/uber/submitqueue/api/stovepipe/gateway/protopb" - "github.com/uber/submitqueue/test/testutil" - "google.golang.org/grpc" -) - -type StovepipeGatewayIntegrationSuite struct { - suite.Suite - ctx context.Context - log *testutil.TestLogger - stack *testutil.ComposeStack - client pb.StovepipeGatewayClient -} - -func TestStovepipeGatewayIntegration(t *testing.T) { - suite.Run(t, new(StovepipeGatewayIntegrationSuite)) -} - -func (s *StovepipeGatewayIntegrationSuite) SetupSuite() { - t := s.T() - s.ctx = context.Background() - s.log = testutil.NewTestLogger(t) - - s.log.Logf("Starting Stovepipe gateway integration test suite using compose") - - repoRoot := testutil.FindRepoRoot(t) - t.Setenv("REPO_ROOT", repoRoot) - - composeFile := filepath.Join(repoRoot, "example/stovepipe/gateway/server/docker-compose.yml") - s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-stovepipe-gateway") - - err := s.stack.Up() - require.NoError(t, err, "failed to start compose stack") - - queueDB, err := s.stack.ConnectMySQLService("mysql-queue") - require.NoError(t, err, "failed to connect to queue MySQL") - - testutil.ApplySchema(t, s.log, queueDB, testutil.SchemaDir("platform/extension/messagequeue/mysql/schema")) - - var conn *grpc.ClientConn - conn, err = s.stack.ConnectGRPC("gateway-service", 8080) - require.NoError(t, err, "failed to connect to stovepipe gateway") - s.client = pb.NewStovepipeGatewayClient(conn) -} - -func (s *StovepipeGatewayIntegrationSuite) TearDownSuite() { - s.log.Logf("Tearing down Stovepipe gateway integration test suite") -} - -func (s *StovepipeGatewayIntegrationSuite) TestPingAPI() { - t := s.T() - - resp, err := s.client.Ping(s.ctx, &pb.PingRequest{Message: "integration test"}) - require.NoError(t, err, "Stovepipe Ping failed") - assert.Equal(t, "stovepipe-gateway", resp.ServiceName) - assert.NotEmpty(t, resp.Message) - assert.NotZero(t, resp.Timestamp) -} diff --git a/test/integration/stovepipe/suite_test.go b/test/integration/stovepipe/suite_test.go new file mode 100644 index 00000000..871fd78a --- /dev/null +++ b/test/integration/stovepipe/suite_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stovepipe + +// Stovepipe integration tests +// +// These tests use compose from example/stovepipe/docker-compose.yml and require +// a pre-built Linux binary (make integration-test runs //test/integration/... +// and builds all Linux binaries via build-all-linux). Stovepipe is currently a +// Ping-only service with no storage or queue dependencies. +// +// Run with: +// make integration-test +// or only this package: +// bazel test //test/integration/stovepipe:stovepipe_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + pb "github.com/uber/submitqueue/api/stovepipe/protopb" + "github.com/uber/submitqueue/test/testutil" + "google.golang.org/grpc" +) + +type StovepipeIntegrationSuite struct { + suite.Suite + ctx context.Context + log *testutil.TestLogger + stack *testutil.ComposeStack + client pb.StovepipeClient +} + +func TestStovepipeIntegration(t *testing.T) { + suite.Run(t, new(StovepipeIntegrationSuite)) +} + +func (s *StovepipeIntegrationSuite) SetupSuite() { + t := s.T() + s.ctx = context.Background() + s.log = testutil.NewTestLogger(t) + + s.log.Logf("Starting Stovepipe integration test suite using compose") + + repoRoot := testutil.FindRepoRoot(t) + t.Setenv("REPO_ROOT", repoRoot) + + composeFile := filepath.Join(repoRoot, "example/stovepipe/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-stovepipe") + + err := s.stack.Up() + require.NoError(t, err, "failed to start compose stack") + + var conn *grpc.ClientConn + conn, err = s.stack.ConnectGRPC("stovepipe-service", 8080) + require.NoError(t, err, "failed to connect to stovepipe service") + s.client = pb.NewStovepipeClient(conn) +} + +func (s *StovepipeIntegrationSuite) TearDownSuite() { + s.log.Logf("Tearing down Stovepipe integration test suite") +} + +func (s *StovepipeIntegrationSuite) TestPingAPI() { + t := s.T() + + resp, err := s.client.Ping(s.ctx, &pb.PingRequest{Message: "integration test"}) + require.NoError(t, err, "Stovepipe Ping failed") + assert.Equal(t, "stovepipe", resp.ServiceName) + assert.NotEmpty(t, resp.Message) + assert.NotZero(t, resp.Timestamp) +} diff --git a/tool/proto/BUILD.bazel b/tool/proto/BUILD.bazel index 8ef56571..a434c3bc 100644 --- a/tool/proto/BUILD.bazel +++ b/tool/proto/BUILD.bazel @@ -58,16 +58,9 @@ go_proto_generated_files( ) go_proto_generated_files( - name = "api_stovepipe_gateway", - srcs = ["//api/stovepipe/gateway/proto:gateway.proto"], - imports = ["//api/base/change/proto:change.proto"], - out_dir = "api_stovepipe_gateway", -) - -go_proto_generated_files( - name = "api_stovepipe_orchestrator", - srcs = ["//api/stovepipe/orchestrator/proto:orchestrator.proto"], - out_dir = "api_stovepipe_orchestrator", + name = "api_stovepipe", + srcs = ["//api/stovepipe/proto:stovepipe.proto"], + out_dir = "api_stovepipe", ) filegroup( @@ -78,8 +71,7 @@ filegroup( ":api_base_messagequeue", ":api_runway_messagequeue", ":api_runway_orchestrator", - ":api_stovepipe_gateway", - ":api_stovepipe_orchestrator", + ":api_stovepipe", ":api_submitqueue_gateway", ":api_submitqueue_orchestrator", ],