From 0d12d443c368eeee0ffe3518f5bffe839f2acd33 Mon Sep 17 00:00:00 2001 From: Alex Vulaj Date: Fri, 17 Apr 2026 13:35:20 -0400 Subject: [PATCH 1/3] Support multi-cluster deployments (Central + SecuredCluster on separate clusters) --- README.md | 17 ++++ cmd/deploy.go | 31 ++++++ cmd/main.go | 5 + internal/deployer/central_endpoint_test.go | 107 +++++++++++++++++++++ internal/deployer/deployer.go | 33 ++++++- 5 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 internal/deployer/central_endpoint_test.go diff --git a/README.md b/README.md index fd2b6a3..024bc6d 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,23 @@ Similarly, the deployment(s) can be torn down using: ./bin/roxie teardown [ ] ``` +### Multi-cluster deployments + +roxie supports hub + spoke architectures where Central and SecuredCluster run on separate clusters. + +1. Deploy Central on the hub cluster: +```bash +MAIN_IMAGE_TAG=4.9.2 ./roxie deploy central +``` + +2. Switch kubectl context to the spoke cluster and deploy SecuredCluster: +```bash +./roxie deploy secured-cluster \ + --central-endpoint=:443 \ + --central-password= \ + --ca-cert-file=/tmp/roxie-ca-cert.pem +``` + ## Development Enter the dev shell: diff --git a/cmd/deploy.go b/cmd/deploy.go index 87d7caa..51f0ef8 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -240,6 +240,11 @@ Examples: return pflag.NormalizedName(name) }) + cmd.Flags().StringVar(¢ralEndpointFlag, "central-endpoint", "", "Central endpoint for multi-cluster SecuredCluster deployments (e.g., central.example.com:443)") + cmd.Flags().StringVar(¢ralPasswordFlag, "central-password", "", "Central admin password (takes precedence over ROX_ADMIN_PASSWORD)") + cmd.Flags().StringVar(&caCertFileFlag, "ca-cert-file", "", "Path to Central CA certificate file (takes precedence over ROX_CA_CERT_FILE)") + + return cmd } @@ -299,6 +304,20 @@ func runDeploy(cmd *cobra.Command, args []string) error { } } + hasMultiClusterFlags := centralEndpointFlag != "" || centralPasswordFlag != "" || caCertFileFlag != "" + if hasMultiClusterFlags && components != component.SecuredCluster { + return errors.New("--central-endpoint, --central-password, and --ca-cert-file flags can only be used with 'secured-cluster' component") + } + + if centralEndpointFlag != "" { + if centralPasswordFlag == "" && os.Getenv("ROX_ADMIN_PASSWORD") == "" { + return errors.New("--central-endpoint requires a Central admin password (set --central-password or ROX_ADMIN_PASSWORD)") + } + if caCertFileFlag == "" && os.Getenv("ROX_CA_CERT_FILE") == "" { + return errors.New("--central-endpoint requires a Central CA certificate (set --ca-cert-file or ROX_CA_CERT_FILE)") + } + } + d, err := deployer.New(log) if err != nil { return fmt.Errorf("failed to create deployer: %w", err) @@ -319,6 +338,18 @@ func runDeploy(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() + if centralEndpointFlag != "" { + d.SetCentralEndpoint(centralEndpointFlag) + } + if centralPasswordFlag != "" { + d.SetCentralPassword(centralPasswordFlag) + } + if caCertFileFlag != "" { + if err := d.SetCACertFile(caCertFileFlag); err != nil { + return err + } + } + if components.IncludesCentral() { d.PrintCentralDeploymentSummary() } diff --git a/cmd/main.go b/cmd/main.go index ac1fa51..590c46b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,11 @@ var ( envrc string dryRun bool + // Multi-cluster flags + centralEndpointFlag string + centralPasswordFlag string + caCertFileFlag string + // We need this set up before command line flags are parsed. deploySettings = deployer.NewConfig() ) diff --git a/internal/deployer/central_endpoint_test.go b/internal/deployer/central_endpoint_test.go new file mode 100644 index 0000000..fbc4dc7 --- /dev/null +++ b/internal/deployer/central_endpoint_test.go @@ -0,0 +1,107 @@ +package deployer + +import ( + "testing" +) + +func TestSetCentralEndpoint(t *testing.T) { + tests := []struct { + name string + input string + expectedCentralEndpoint string + expectedUserProvidedEndpoint string + }{ + { + name: "plain host:port", + input: "10.0.0.1:443", + expectedCentralEndpoint: "10.0.0.1:443", + expectedUserProvidedEndpoint: "10.0.0.1:443", + }, + { + name: "strips https prefix", + input: "https://10.0.0.1:443", + expectedCentralEndpoint: "10.0.0.1:443", + expectedUserProvidedEndpoint: "10.0.0.1:443", + }, + { + name: "hostname with port", + input: "central.example.com:443", + expectedCentralEndpoint: "central.example.com:443", + expectedUserProvidedEndpoint: "central.example.com:443", + }, + { + name: "strips https from hostname", + input: "https://central.example.com:443", + expectedCentralEndpoint: "central.example.com:443", + expectedUserProvidedEndpoint: "central.example.com:443", + }, + { + name: "strips http prefix", + input: "http://10.0.0.1:443", + expectedCentralEndpoint: "10.0.0.1:443", + expectedUserProvidedEndpoint: "10.0.0.1:443", + }, + { + name: "strips http from hostname", + input: "http://central.example.com:443", + expectedCentralEndpoint: "central.example.com:443", + expectedUserProvidedEndpoint: "central.example.com:443", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &Deployer{} + d.SetCentralEndpoint(tt.input) + + if d.centralEndpoint != tt.expectedCentralEndpoint { + t.Errorf("centralEndpoint: got %q, want %q", d.centralEndpoint, tt.expectedCentralEndpoint) + } + if d.userProvidedCentralEndpoint != tt.expectedUserProvidedEndpoint { + t.Errorf("userProvidedCentralEndpoint: got %q, want %q", d.userProvidedCentralEndpoint, tt.expectedUserProvidedEndpoint) + } + }) + } +} + +func TestGetCentralEndpointForSensor(t *testing.T) { + tests := []struct { + name string + userProvided string + centralNamespace string + expected string + }{ + { + name: "falls back to internal endpoint", + userProvided: "", + centralNamespace: "acs-central", + expected: "central.acs-central.svc:443", + }, + { + name: "falls back to internal endpoint with custom namespace", + userProvided: "", + centralNamespace: "stackrox", + expected: "central.stackrox.svc:443", + }, + { + name: "uses user-provided endpoint", + userProvided: "10.0.0.1:443", + centralNamespace: "acs-central", + expected: "10.0.0.1:443", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &Deployer{ + centralNamespace: tt.centralNamespace, + userProvidedCentralEndpoint: tt.userProvided, + } + + result := d.getCentralEndpointForSensor() + if result != tt.expected { + t.Errorf("got %q, want %q", result, tt.expected) + } + }) + } +} diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 2c602e2..3f26362 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -51,7 +51,8 @@ type Deployer struct { config Config // State - centralEndpoint string + centralEndpoint string + userProvidedCentralEndpoint string centralPassword string roxCACertFile string tempDir string @@ -653,6 +654,32 @@ func (d *Deployer) removePauseReconcileAnnotation(ctx context.Context, resourceT return nil } +func (d *Deployer) SetCentralEndpoint(endpoint string) { + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + d.centralEndpoint = endpoint + d.userProvidedCentralEndpoint = endpoint +} + +func (d *Deployer) SetCentralPassword(password string) { + d.centralPassword = password +} + +func (d *Deployer) SetCACertFile(path string) error { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("CA cert file not found: %w", err) + } + d.roxCACertFile = path + return nil +} + +func (d *Deployer) getCentralEndpointForSensor() string { + if d.userProvidedCentralEndpoint != "" { + return d.userProvidedCentralEndpoint + } + return internalCentralEndpoint(d.config.Central.Namespace) +} + // WaitForCentral waits for Central to be ready and responding on its endpoint // Returns true if Central is ready, false if timeout occurs func (d *Deployer) WaitForCentral(timeout time.Duration) bool { @@ -995,6 +1022,10 @@ func (d *Deployer) PrintSecuredClusterDeploymentSummary() { log.Info(cyan.Sprint("│") + createRow("OLM", "Yes")) } + if d.userProvidedCentralEndpoint != "" { + log.Info(cyan.Sprint("│") + createRow("Central Endpoint", d.userProvidedCentralEndpoint)) + } + log.Info(cyan.Sprint("└" + strings.Repeat("─", boxWidth) + "┘")) log.Info("") } From 0fdf7634d3d15d16d1da377d7748bc4811c67072 Mon Sep 17 00:00:00 2001 From: Alex Vulaj Date: Mon, 20 Apr 2026 09:51:47 -0400 Subject: [PATCH 2/3] Add subshell note to multi-cluster README section --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 024bc6d..efb8cbb 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,12 @@ MAIN_IMAGE_TAG=4.9.2 ./roxie deploy central --ca-cert-file=/tmp/roxie-ca-cert.pem ``` +> **Note:** The Central endpoint is printed during deployment. If you are in the roxie subshell, +> `API_ENDPOINT`, `ROX_ADMIN_PASSWORD`, and `ROX_CA_CERT_FILE` are already set, so you can run: +> ```bash +> ./roxie deploy secured-cluster --central-endpoint=$API_ENDPOINT +> ``` + ## Development Enter the dev shell: From 954384afd63cc970133cb58a1428fe80f4bf4bbe Mon Sep 17 00:00:00 2001 From: Alex Vulaj Date: Wed, 13 May 2026 13:29:14 -0400 Subject: [PATCH 3/3] Migrate multi-cluster from CLI flags to config property --- README.md | 10 +- cmd/deploy.go | 35 ++----- cmd/main.go | 5 - internal/deployer/central_endpoint_test.go | 104 +++++++-------------- internal/deployer/config.go | 8 +- internal/deployer/deployer.go | 33 +------ 6 files changed, 55 insertions(+), 140 deletions(-) diff --git a/README.md b/README.md index efb8cbb..51cf457 100644 --- a/README.md +++ b/README.md @@ -95,16 +95,16 @@ MAIN_IMAGE_TAG=4.9.2 ./roxie deploy central 2. Switch kubectl context to the spoke cluster and deploy SecuredCluster: ```bash +ROX_ADMIN_PASSWORD= \ +ROX_CA_CERT_FILE= \ ./roxie deploy secured-cluster \ - --central-endpoint=:443 \ - --central-password= \ - --ca-cert-file=/tmp/roxie-ca-cert.pem + --set securedCluster.centralEndpoint=:443 ``` > **Note:** The Central endpoint is printed during deployment. If you are in the roxie subshell, -> `API_ENDPOINT`, `ROX_ADMIN_PASSWORD`, and `ROX_CA_CERT_FILE` are already set, so you can run: +> `ROX_ADMIN_PASSWORD` and `ROX_CA_CERT_FILE` are already set, so you can run: > ```bash -> ./roxie deploy secured-cluster --central-endpoint=$API_ENDPOINT +> ./roxie deploy secured-cluster --set securedCluster.centralEndpoint=$API_ENDPOINT > ``` ## Development diff --git a/cmd/deploy.go b/cmd/deploy.go index 51f0ef8..b03bdb4 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -240,11 +240,6 @@ Examples: return pflag.NormalizedName(name) }) - cmd.Flags().StringVar(¢ralEndpointFlag, "central-endpoint", "", "Central endpoint for multi-cluster SecuredCluster deployments (e.g., central.example.com:443)") - cmd.Flags().StringVar(¢ralPasswordFlag, "central-password", "", "Central admin password (takes precedence over ROX_ADMIN_PASSWORD)") - cmd.Flags().StringVar(&caCertFileFlag, "ca-cert-file", "", "Path to Central CA certificate file (takes precedence over ROX_CA_CERT_FILE)") - - return cmd } @@ -304,17 +299,15 @@ func runDeploy(cmd *cobra.Command, args []string) error { } } - hasMultiClusterFlags := centralEndpointFlag != "" || centralPasswordFlag != "" || caCertFileFlag != "" - if hasMultiClusterFlags && components != component.SecuredCluster { - return errors.New("--central-endpoint, --central-password, and --ca-cert-file flags can only be used with 'secured-cluster' component") - } - - if centralEndpointFlag != "" { - if centralPasswordFlag == "" && os.Getenv("ROX_ADMIN_PASSWORD") == "" { - return errors.New("--central-endpoint requires a Central admin password (set --central-password or ROX_ADMIN_PASSWORD)") + if deploySettings.SecuredCluster.CentralEndpoint != "" { + if !components.IncludesSensor() { + return errors.New("securedCluster.centralEndpoint can only be used when deploying secured-cluster") + } + if os.Getenv("ROX_ADMIN_PASSWORD") == "" { + return errors.New("securedCluster.centralEndpoint requires ROX_ADMIN_PASSWORD to be set") } - if caCertFileFlag == "" && os.Getenv("ROX_CA_CERT_FILE") == "" { - return errors.New("--central-endpoint requires a Central CA certificate (set --ca-cert-file or ROX_CA_CERT_FILE)") + if os.Getenv("ROX_CA_CERT_FILE") == "" { + return errors.New("securedCluster.centralEndpoint requires ROX_CA_CERT_FILE to be set") } } @@ -338,18 +331,6 @@ func runDeploy(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - if centralEndpointFlag != "" { - d.SetCentralEndpoint(centralEndpointFlag) - } - if centralPasswordFlag != "" { - d.SetCentralPassword(centralPasswordFlag) - } - if caCertFileFlag != "" { - if err := d.SetCACertFile(caCertFileFlag); err != nil { - return err - } - } - if components.IncludesCentral() { d.PrintCentralDeploymentSummary() } diff --git a/cmd/main.go b/cmd/main.go index 590c46b..ac1fa51 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,11 +15,6 @@ var ( envrc string dryRun bool - // Multi-cluster flags - centralEndpointFlag string - centralPasswordFlag string - caCertFileFlag string - // We need this set up before command line flags are parsed. deploySettings = deployer.NewConfig() ) diff --git a/internal/deployer/central_endpoint_test.go b/internal/deployer/central_endpoint_test.go index fbc4dc7..5ea93cb 100644 --- a/internal/deployer/central_endpoint_test.go +++ b/internal/deployer/central_endpoint_test.go @@ -2,105 +2,65 @@ package deployer import ( "testing" -) - -func TestSetCentralEndpoint(t *testing.T) { - tests := []struct { - name string - input string - expectedCentralEndpoint string - expectedUserProvidedEndpoint string - }{ - { - name: "plain host:port", - input: "10.0.0.1:443", - expectedCentralEndpoint: "10.0.0.1:443", - expectedUserProvidedEndpoint: "10.0.0.1:443", - }, - { - name: "strips https prefix", - input: "https://10.0.0.1:443", - expectedCentralEndpoint: "10.0.0.1:443", - expectedUserProvidedEndpoint: "10.0.0.1:443", - }, - { - name: "hostname with port", - input: "central.example.com:443", - expectedCentralEndpoint: "central.example.com:443", - expectedUserProvidedEndpoint: "central.example.com:443", - }, - { - name: "strips https from hostname", - input: "https://central.example.com:443", - expectedCentralEndpoint: "central.example.com:443", - expectedUserProvidedEndpoint: "central.example.com:443", - }, - { - name: "strips http prefix", - input: "http://10.0.0.1:443", - expectedCentralEndpoint: "10.0.0.1:443", - expectedUserProvidedEndpoint: "10.0.0.1:443", - }, - { - name: "strips http from hostname", - input: "http://central.example.com:443", - expectedCentralEndpoint: "central.example.com:443", - expectedUserProvidedEndpoint: "central.example.com:443", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - d := &Deployer{} - d.SetCentralEndpoint(tt.input) - - if d.centralEndpoint != tt.expectedCentralEndpoint { - t.Errorf("centralEndpoint: got %q, want %q", d.centralEndpoint, tt.expectedCentralEndpoint) - } - if d.userProvidedCentralEndpoint != tt.expectedUserProvidedEndpoint { - t.Errorf("userProvidedCentralEndpoint: got %q, want %q", d.userProvidedCentralEndpoint, tt.expectedUserProvidedEndpoint) - } - }) - } -} + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) -func TestGetCentralEndpointForSensor(t *testing.T) { +func TestConfigureSpec_CentralEndpoint(t *testing.T) { tests := []struct { name string - userProvided string + centralEndpoint string centralNamespace string expected string }{ { name: "falls back to internal endpoint", - userProvided: "", + centralEndpoint: "", centralNamespace: "acs-central", expected: "central.acs-central.svc:443", }, { name: "falls back to internal endpoint with custom namespace", - userProvided: "", + centralEndpoint: "", centralNamespace: "stackrox", expected: "central.stackrox.svc:443", }, { - name: "uses user-provided endpoint", - userProvided: "10.0.0.1:443", + name: "uses provided central endpoint", + centralEndpoint: "central.example.com:443", centralNamespace: "acs-central", + expected: "central.example.com:443", + }, + { + name: "provided endpoint takes precedence over namespace", + centralEndpoint: "10.0.0.1:443", + centralNamespace: "stackrox", expected: "10.0.0.1:443", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &Deployer{ - centralNamespace: tt.centralNamespace, - userProvidedCentralEndpoint: tt.userProvided, + sc := &SecuredClusterConfig{ + CentralEndpoint: tt.centralEndpoint, + Spec: make(map[string]interface{}), } + roxie := &RoxieConfig{FeatureFlags: make(map[string]bool)} + central := &CentralConfig{Namespace: tt.centralNamespace} - result := d.getCentralEndpointForSensor() - if result != tt.expected { - t.Errorf("got %q, want %q", result, tt.expected) + if err := sc.ConfigureSpec(roxie, central); err != nil { + t.Fatalf("ConfigureSpec failed: %v", err) + } + + got, found, err := unstructured.NestedString(sc.Spec, "centralEndpoint") + if err != nil { + t.Fatalf("failed to get centralEndpoint from spec: %v", err) + } + if !found { + t.Fatal("centralEndpoint not found in spec") + } + if got != tt.expected { + t.Errorf("got %q, want %q", got, tt.expected) } }) } diff --git a/internal/deployer/config.go b/internal/deployer/config.go index 1a6b2ca..56bc5f6 100644 --- a/internal/deployer/config.go +++ b/internal/deployer/config.go @@ -199,6 +199,7 @@ func (c *CentralConfig) CustomResource() (map[string]interface{}, error) { // SecuredClusterConfig holds deployment settings for the SecuredCluster component. type SecuredClusterConfig struct { Namespace string `yaml:"namespace,omitempty"` + CentralEndpoint string `yaml:"centralEndpoint,omitempty"` ResourceProfile types.ResourceProfile `yaml:"resourceProfile,omitempty"` PauseReconciliation bool `yaml:"pauseReconciliation,omitempty"` DeployTimeout time.Duration `yaml:"deployTimeout,omitempty"` @@ -234,8 +235,13 @@ func (s *SecuredClusterConfig) ConfigureSpec(roxieConfig *RoxieConfig, centralCo return err } + centralEndpoint := internalCentralEndpoint(centralConfig.Namespace) + if s.CentralEndpoint != "" { + centralEndpoint = s.CentralEndpoint + } + if err := helpers.DeepMerge(s.Spec, map[string]interface{}{ - "centralEndpoint": internalCentralEndpoint(centralConfig.Namespace), + "centralEndpoint": centralEndpoint, }); err != nil { return err } diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 3f26362..36b9832 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -51,8 +51,7 @@ type Deployer struct { config Config // State - centralEndpoint string - userProvidedCentralEndpoint string + centralEndpoint string centralPassword string roxCACertFile string tempDir string @@ -654,32 +653,6 @@ func (d *Deployer) removePauseReconcileAnnotation(ctx context.Context, resourceT return nil } -func (d *Deployer) SetCentralEndpoint(endpoint string) { - endpoint = strings.TrimPrefix(endpoint, "https://") - endpoint = strings.TrimPrefix(endpoint, "http://") - d.centralEndpoint = endpoint - d.userProvidedCentralEndpoint = endpoint -} - -func (d *Deployer) SetCentralPassword(password string) { - d.centralPassword = password -} - -func (d *Deployer) SetCACertFile(path string) error { - if _, err := os.Stat(path); err != nil { - return fmt.Errorf("CA cert file not found: %w", err) - } - d.roxCACertFile = path - return nil -} - -func (d *Deployer) getCentralEndpointForSensor() string { - if d.userProvidedCentralEndpoint != "" { - return d.userProvidedCentralEndpoint - } - return internalCentralEndpoint(d.config.Central.Namespace) -} - // WaitForCentral waits for Central to be ready and responding on its endpoint // Returns true if Central is ready, false if timeout occurs func (d *Deployer) WaitForCentral(timeout time.Duration) bool { @@ -1022,8 +995,8 @@ func (d *Deployer) PrintSecuredClusterDeploymentSummary() { log.Info(cyan.Sprint("│") + createRow("OLM", "Yes")) } - if d.userProvidedCentralEndpoint != "" { - log.Info(cyan.Sprint("│") + createRow("Central Endpoint", d.userProvidedCentralEndpoint)) + if d.config.SecuredCluster.CentralEndpoint != "" { + log.Info(cyan.Sprint("│") + createRow("Central Endpoint", d.config.SecuredCluster.CentralEndpoint)) } log.Info(cyan.Sprint("└" + strings.Repeat("─", boxWidth) + "┘"))