diff --git a/.github/actions/go-cache/action.sh b/.github/actions/go-cache/action.sh new file mode 100755 index 0000000000000..5cfafe4767fb2 --- /dev/null +++ b/.github/actions/go-cache/action.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# This script sets up cigocacher, but should never fail the build if unsuccessful. +# It expects to run on a GitHub-hosted runner, and connects to cigocached over a +# private Azure network that is configured at the runner group level in GitHub. +# +# Usage: ./action.sh +# Inputs: +# URL: The cigocached server URL. +# HOST: The cigocached server host to dial. +# Outputs: +# success: Whether cigocacher was set up successfully. + +set -euo pipefail + +if [ -z "${GITHUB_ACTIONS:-}" ]; then + echo "This script is intended to run within GitHub Actions" + exit 1 +fi + +if [ -z "${URL:-}" ]; then + echo "No cigocached URL is set, skipping cigocacher setup" + exit 0 +fi + +BIN_PATH="$(PATH="$PATH:$HOME/bin" command -v cigocacher || true)" +if [ -z "${BIN_PATH}" ]; then + echo "cigocacher not found in PATH, attempting to build or fetch it" + + GOPATH=$(command -v go || true) + if [ -z "${GOPATH}" ]; then + if [ ! -f "tool/go" ]; then + echo "Go not available, unable to proceed" + exit 1 + fi + GOPATH="./tool/go" + fi + + BIN_PATH="${RUNNER_TEMP:-/tmp}/cigocacher$(${GOPATH} env GOEXE)" + if [ -d "cmd/cigocacher" ]; then + echo "cmd/cigocacher found locally, building from local source" + "${GOPATH}" build -o "${BIN_PATH}" ./cmd/cigocacher + else + echo "cmd/cigocacher not found locally, fetching from tailscale.com/cmd/cigocacher" + "${GOPATH}" build -o "${BIN_PATH}" tailscale.com/cmd/cigocacher + fi +fi + +CIGOCACHER_TOKEN="$("${BIN_PATH}" --auth --cigocached-url "${URL}" --cigocached-host "${HOST}" )" +if [ -z "${CIGOCACHER_TOKEN:-}" ]; then + echo "Failed to fetch cigocacher token, skipping cigocacher setup" + exit 0 +fi + +echo "Fetched cigocacher token successfully" +echo "::add-mask::${CIGOCACHER_TOKEN}" + +echo "GOCACHEPROG=${BIN_PATH} --cache-dir ${CACHE_DIR} --cigocached-url ${URL} --cigocached-host ${HOST} --token ${CIGOCACHER_TOKEN}" >> "${GITHUB_ENV}" +echo "success=true" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/go-cache/action.yml b/.github/actions/go-cache/action.yml new file mode 100644 index 0000000000000..7f5a66de17d0f --- /dev/null +++ b/.github/actions/go-cache/action.yml @@ -0,0 +1,35 @@ +name: go-cache +description: Set up build to use cigocacher + +inputs: + cigocached-url: + description: URL of the cigocached server + required: true + cigocached-host: + description: Host to dial for the cigocached server + required: true + checkout-path: + description: Path to cloned repository + required: true + cache-dir: + description: Directory to use for caching + required: true + +outputs: + success: + description: Whether cigocacher was set up successfully + value: ${{ steps.setup.outputs.success }} + +runs: + using: composite + steps: + - name: Setup cigocacher + id: setup + shell: bash + env: + URL: ${{ inputs.cigocached-url }} + HOST: ${{ inputs.cigocached-host }} + CACHE_DIR: ${{ inputs.cache-dir }} + working-directory: ${{ inputs.checkout-path }} + # https://github.com/orgs/community/discussions/25910 + run: $GITHUB_ACTION_PATH/action.sh diff --git a/.github/workflows/checklocks.yml b/.github/workflows/checklocks.yml index 5957e69258db5..5768cf05af634 100644 --- a/.github/workflows/checklocks.yml +++ b/.github/workflows/checklocks.yml @@ -18,7 +18,7 @@ jobs: runs-on: [ ubuntu-latest ] steps: - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Build checklocks run: ./tool/go build -o /tmp/checklocks gvisor.dev/gvisor/tools/checklocks/cmd/checklocks diff --git a/.github/workflows/cigocacher.yml b/.github/workflows/cigocacher.yml new file mode 100644 index 0000000000000..9e7f01725958e --- /dev/null +++ b/.github/workflows/cigocacher.yml @@ -0,0 +1,73 @@ +name: Build cigocacher + +on: + # Released on-demand. The commit will be used as part of the tag, so generally + # prefer to release from main where the commit is stable in linear history. + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + GOOS: ["linux", "darwin", "windows"] + GOARCH: ["amd64", "arm64"] + runs-on: ubuntu-24.04 + env: + GOOS: "${{ matrix.GOOS }}" + GOARCH: "${{ matrix.GOARCH }}" + CGO_ENABLED: "0" + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Build + run: | + OUT="cigocacher$(./tool/go env GOEXE)" + ./tool/go build -o "${OUT}" ./cmd/cigocacher/ + tar -zcf cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }}.tar.gz "${OUT}" + + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }} + path: cigocacher-${{ matrix.GOOS }}-${{ matrix.GOARCH }}.tar.gz + + release: + runs-on: ubuntu-24.04 + needs: build + permissions: + contents: write + steps: + - name: Download all artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: 'cigocacher-*' + merge-multiple: true + # This step is a simplified version of actions/create-release and + # actions/upload-release-asset, which are archived and unmaintained. + - name: Create release + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const { data: release } = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: `cmd/cigocacher/${{ github.sha }}`, + name: `cigocacher-${{ github.sha }}`, + draft: false, + prerelease: true, + target_commitish: `${{ github.sha }}` + }); + + const files = fs.readdirSync('.').filter(f => f.endsWith('.tar.gz')); + + for (const file of files) { + await github.rest.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release.id, + name: file, + data: fs.readFileSync(file) + }); + console.log(`Uploaded ${file}`); + } diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2b471e943318f..abe6a2c3ae684 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -45,17 +45,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Install a more recent Go that understands modern go.mod content. - name: Install Go - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: go.mod # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 + uses: github/codeql-action/init@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 + uses: github/codeql-action/autobuild@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 + uses: github/codeql-action/analyze@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1 diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml new file mode 100644 index 0000000000000..a3eac2c24e691 --- /dev/null +++ b/.github/workflows/docker-base.yml @@ -0,0 +1,29 @@ +name: "Validate Docker base image" +on: + workflow_dispatch: + pull_request: + paths: + - "Dockerfile.base" + - ".github/workflows/docker-base.yml" +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: "build and test" + run: | + set -e + IMG="test-base:$(head -c 8 /dev/urandom | xxd -p)" + docker build -t "$IMG" -f Dockerfile.base . + + iptables_version=$(docker run --rm "$IMG" iptables --version) + if [[ "$iptables_version" != *"(legacy)"* ]]; then + echo "ERROR: Docker base image should contain legacy iptables; found ${iptables_version}" + exit 1 + fi + + ip6tables_version=$(docker run --rm "$IMG" ip6tables --version) + if [[ "$ip6tables_version" != *"(legacy)"* ]]; then + echo "ERROR: Docker base image should contain legacy ip6tables; found ${ip6tables_version}" + exit 1 + fi diff --git a/.github/workflows/docker-file-build.yml b/.github/workflows/docker-file-build.yml index 04611e172bbea..7ee2468682695 100644 --- a/.github/workflows/docker-file-build.yml +++ b/.github/workflows/docker-file-build.yml @@ -4,12 +4,10 @@ on: branches: - main pull_request: - branches: - - "*" jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: "Build Docker image" run: docker build . diff --git a/.github/workflows/flakehub-publish-tagged.yml b/.github/workflows/flakehub-publish-tagged.yml index 9ff12c6a3fd14..c781e30e5154f 100644 --- a/.github/workflows/flakehub-publish-tagged.yml +++ b/.github/workflows/flakehub-publish-tagged.yml @@ -17,11 +17,11 @@ jobs: id-token: "write" contents: "read" steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}" - - uses: "DeterminateSystems/nix-installer-action@main" - - uses: "DeterminateSystems/flakehub-push@main" + - uses: DeterminateSystems/nix-installer-action@c5a866b6ab867e88becbed4467b93592bce69f8a # v21 + - uses: DeterminateSystems/flakehub-push@71f57208810a5d299fc6545350981de98fdbc860 # v6 with: visibility: "public" tag: "${{ inputs.tag }}" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index ee62f04bed91c..66b8497e65441 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -2,7 +2,11 @@ name: golangci-lint on: # For now, only lint pull requests, not the main branches. pull_request: - + paths: + - ".github/workflows/golangci-lint.yml" + - "**.go" + - "go.mod" + - "go.sum" # TODO(andrew): enable for main branch after an initial waiting period. #push: # branches: @@ -23,17 +27,21 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: go.mod - cache: false + cache: true - name: golangci-lint - uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 + uses: golangci/golangci-lint-action@b7bcab6379029e905e3f389a6bf301f1bc220662 # head as of 2026-03-04 with: - version: v2.0.2 + version: v2.10.1 # Show only new issues if it's a pull request. only-new-issues: true + + # Loading packages with a cold cache takes a while: + args: --timeout=10m + diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 36ed1fe9bf603..2b46aa9b06e57 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install govulncheck run: ./tool/go install golang.org/x/vuln/cmd/govulncheck@latest @@ -24,7 +24,7 @@ jobs: - name: Post to slack if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 with: method: chat.postMessage token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }} diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 0ca16ae9fa6c1..9b49c4c07a092 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -10,8 +10,6 @@ on: - scripts/installer.sh - .github/workflows/installer.yml pull_request: - branches: - - "*" paths: - scripts/installer.sh - .github/workflows/installer.yml @@ -39,8 +37,6 @@ jobs: - "elementary/docker:stable" - "elementary/docker:unstable" - "parrotsec/core:latest" - - "kalilinux/kali-rolling" - - "kalilinux/kali-dev" - "oraclelinux:9" - "oraclelinux:8" - "fedora:latest" @@ -60,6 +56,17 @@ jobs: # Check a few images with wget rather than curl. - { image: "debian:oldstable-slim", deps: "wget" } - { image: "debian:sid-slim", deps: "wget" } + - { image: "debian:stable-slim", deps: "curl" } + - { image: "ubuntu:24.04", deps: "curl" } + - { image: "fedora:latest", deps: "curl" } + # Kali doesn't have ca-certificates installed by default anymore + - { image: "kalilinux/kali-dev", "deps": "curl ca-certificates"} + - { image: "kalilinux/kali-rolling", "deps": "curl ca-certificates"} + # Test TAILSCALE_VERSION pinning on a subset of distros. + # Skip Alpine as community repos don't reliably keep old versions. + - { image: "debian:stable-slim", deps: "curl", version: "1.80.0" } + - { image: "ubuntu:24.04", deps: "curl", version: "1.80.0" } + - { image: "fedora:latest", deps: "curl", version: "1.80.0" } runs-on: ubuntu-latest container: image: ${{ matrix.image }} @@ -93,22 +100,28 @@ jobs: contains(matrix.image, 'parrotsec') || contains(matrix.image, 'kalilinux') - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: run installer run: scripts/installer.sh + env: + TAILSCALE_VERSION: ${{ matrix.version }} # Package installation can fail in docker because systemd is not running # as PID 1, so ignore errors at this step. The real check is the # `tailscale --version` command below. continue-on-error: true - name: check tailscale version - run: tailscale --version + run: | + tailscale --version + if [ -n "${{ matrix.version }}" ]; then + tailscale --version | grep -q "^${{ matrix.version }}" || { echo "Version mismatch!"; exit 1; } + fi notify-slack: needs: test runs-on: ubuntu-latest steps: - name: Notify Slack of failure on scheduled runs if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: incoming-webhook diff --git a/.github/workflows/kubemanifests.yaml b/.github/workflows/kubemanifests.yaml index 4cffea02fce6b..40734a015dad3 100644 --- a/.github/workflows/kubemanifests.yaml +++ b/.github/workflows/kubemanifests.yaml @@ -17,7 +17,7 @@ jobs: runs-on: [ ubuntu-latest ] steps: - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Build and lint Helm chart run: | eval `./tool/go run ./cmd/mkversion` diff --git a/.github/workflows/natlab-basic.yml b/.github/workflows/natlab-basic.yml new file mode 100644 index 0000000000000..1a19acfb8956f --- /dev/null +++ b/.github/workflows/natlab-basic.yml @@ -0,0 +1,45 @@ +# Run a single natlab smoke test on every PR. The full natlab suite +# is opt-in and lives in .github/workflows/natlab-test.yml. +# See https://github.com/tailscale/tailscale/issues/13038 +name: "natlab-basic" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + push: + branches: + - "main" + - "release-branch/*" + pull_request: + # all PRs on all branches + merge_group: + branches: + - "main" +jobs: + EasyEasy: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Install qemu + run: | + sudo rm -f /var/lib/man-db/auto-update + sudo apt-get -y update + sudo apt-get -y remove man-db + sudo apt-get install -y qemu-system-x86 qemu-utils + - name: Build VM image + # The test will build this if missing, but we do it explicitly + # to avoid cutting into the go test -timeout budget, and to + # fail earlier with a clearer error if the image build breaks. + run: | + make -C gokrazy natlab + - name: Run natlab integration tests + run: | + ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/natlab/vmtest --run-vm-tests diff --git a/.github/workflows/natlab-integrationtest.yml b/.github/workflows/natlab-integrationtest.yml deleted file mode 100644 index 99d58717b7beb..0000000000000 --- a/.github/workflows/natlab-integrationtest.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Run some natlab integration tests. -# See https://github.com/tailscale/tailscale/issues/13038 -name: "natlab-integrationtest" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - pull_request: - paths: - - "tstest/integration/nat/nat_test.go" -jobs: - natlab-integrationtest: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install qemu - run: | - sudo rm /var/lib/man-db/auto-update - sudo apt-get -y update - sudo apt-get -y remove man-db - sudo apt-get install -y qemu-system-x86 qemu-utils - - name: Run natlab integration tests - run: | - ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/integration/nat --run-vm-tests diff --git a/.github/workflows/natlab-test.yml b/.github/workflows/natlab-test.yml new file mode 100644 index 0000000000000..4f53c4ce44abf --- /dev/null +++ b/.github/workflows/natlab-test.yml @@ -0,0 +1,182 @@ +# Run the full natlab/vmtest opt-in test suite. These tests boot QEMU VMs +# (gokrazy, Ubuntu, FreeBSD) and exercise vnet-driven networking scenarios. +# They are gated behind --run-vm-tests because they need KVM and are slow. +# +# This workflow runs: +# - on demand (workflow_dispatch) +# - on PRs that carry the "run-natlab-tests" label +# - on main, every 12 hours, via cron +# +# Layout: +# - "prepare" builds the gokrazy VM image, downloads the cloud images +# (Ubuntu, FreeBSD), and discovers every Test* function in the two +# opt-in packages. +# - "test" is a per-TestFoo matrix that depends on prepare. Each matrix +# job restores the shared caches and runs a single test. Adding a new +# TestFoo automatically gets its own job — no workflow edits needed. +# +# A separate workflow (.github/workflows/natlab-basic.yml) runs a single +# canary natlab test on every PR; this one runs the full suite. +name: "natlab-test" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + types: [labeled, synchronize, reopened] + schedule: + # Every 12 hours, off-the-hour to avoid GitHub's :00 cron-stampede window. + - cron: "23 3,15 * * *" + +jobs: + # prepare warms the per-workflow-run caches (gokrazy image, cloud VM + # images) and emits the dynamic matrix of test names. By doing the work + # once here, the matrix test jobs never race to rebuild or re-download + # the same artifacts on a cold cache. + prepare: + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-natlab-tests')) + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + matrix: ${{ steps.list.outputs.matrix }} + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # The cloud VM image cache is keyed only on images.go (image URLs and + # SHAs), so it survives across workflow runs and is invalidated only + # when a new image source is added. + - name: Cache cloud VM images + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ~/.cache/tailscale/vmtest/images + key: natlab-vmimages-${{ hashFiles('tstest/natlab/vmtest/images.go') }} + + # The gokrazy VM image is keyed by github.sha. That means we rebuild + # it once per commit but matrix test jobs in the same run all share + # the result. Per-PR re-runs of the same sha (e.g. a rerun-failed) + # also get the cache. + - name: Cache gokrazy VM image + id: gokrazy-cache + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gokrazy/natlabapp.qcow2 + key: natlab-gokrazy-${{ github.sha }} + + # qemu-utils provides qemu-img, which the gokrazy Makefile uses to + # convert natlabapp.img to qcow2. Only install if we need it (cache + # miss); the test matrix jobs install qemu separately for the runtime. + - name: Install qemu-utils + if: steps.gokrazy-cache.outputs.cache-hit != 'true' + run: | + sudo rm -f /var/lib/man-db/auto-update + sudo apt-get -y update + sudo apt-get -y remove man-db + sudo apt-get install -y qemu-utils + + - name: Download cloud VM images + # natlabprep is idempotent: it checks the cache before downloading. + run: | + ./tool/go run ./tstest/natlab/vmtest/cmd/natlabprep + + - name: Build gokrazy VM image + if: steps.gokrazy-cache.outputs.cache-hit != 'true' + run: | + make -C gokrazy natlab + + - name: Discover tests + id: list + # Grep the test files directly rather than invoking `go test -list` + # so we don't pay the cost of compiling the test binaries here. The + # only test functions in these packages use the canonical + # `func TestFoo(t *testing.T)` signature. + # + # exclude is the set of tests that need special invocation + # (extra flags, a specific environment) and don't fit the + # single-test-per-matrix-job model. They stay runnable locally. + run: | + set -euo pipefail + exclude='^(TestGrid)$' + tmp=$(mktemp) + for pkg_dir in tstest/natlab/vmtest tstest/integration/nat; do + pkg="./${pkg_dir}/" + for f in "${pkg_dir}"/*_test.go; do + [ -e "$f" ] || continue + grep -hE '^func Test[A-Z][A-Za-z0-9_]*\(t \*testing\.T\)' "$f" \ + | sed -E 's/^func (Test[A-Za-z0-9_]+).*/\1/' \ + | grep -vE "$exclude" \ + | while read -r t; do + jq -nc --arg pkg "$pkg" --arg test "$t" \ + '{pkg: $pkg, test: $test}' >> "$tmp" + done + done + done + matrix=$(jq -s -c . "$tmp") + echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" + echo "Discovered tests:" + jq . "$tmp" + + test: + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 20 + name: "${{ matrix.test }}" + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix) }} + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Install qemu + run: | + sudo rm -f /var/lib/man-db/auto-update + sudo apt-get -y update + sudo apt-get -y remove man-db + sudo apt-get install -y qemu-system-x86 qemu-utils + + # restore-only: prepare is the single writer of these caches, so + # matrix jobs don't write back. fail-on-cache-miss would be too + # strict for the gokrazy cache (e.g. a non-fatal cache eviction + # between prepare and us); we just rebuild on miss instead. + - name: Restore cloud VM images + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ~/.cache/tailscale/vmtest/images + key: natlab-vmimages-${{ hashFiles('tstest/natlab/vmtest/images.go') }} + + - name: Restore gokrazy VM image + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gokrazy/natlabapp.qcow2 + key: natlab-gokrazy-${{ github.sha }} + + # The gokrazy-based tests boot the kernel directly from + # vmlinuz that ships in the tailscale/gokrazy-kernel module. + # Tests look it up under GOMODCACHE via findKernelPath, so the + # module has to be present even though no Go source imports it + # in the test package itself. + - name: Download gokrazy-kernel module + run: | + ./tool/go mod download github.com/tailscale/gokrazy-kernel + + - name: Run ${{ matrix.test }} + # Per-test timeout is well above the few-minute typical runtime + # but small enough that a stuck test fails fast instead of holding + # the runner for the job's 20-minute budget. + run: | + ./tool/go test -v -timeout=15m -count=1 ${{ matrix.pkg }} \ + -run='^${{ matrix.test }}$' --run-vm-tests diff --git a/.github/workflows/pin-github-actions.yml b/.github/workflows/pin-github-actions.yml new file mode 100644 index 0000000000000..836ae46dbfa89 --- /dev/null +++ b/.github/workflows/pin-github-actions.yml @@ -0,0 +1,29 @@ +# Pin images used in github actions to a hash instead of a version tag. +name: pin-github-actions +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/**" + + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + run: + name: pin-github-actions + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: pin + run: make pin-github-actions + - name: check for changed workflow files + run: git diff --no-ext-diff --exit-code .github/workflows || (echo "Some github actions versions need pinning, run make pin-github-actions."; exit 1) diff --git a/.github/workflows/request-dataplane-review.yml b/.github/workflows/request-dataplane-review.yml new file mode 100644 index 0000000000000..78bd8ff585bff --- /dev/null +++ b/.github/workflows/request-dataplane-review.yml @@ -0,0 +1,32 @@ +name: request-dataplane-review + +on: + pull_request: + types: [ opened, synchronize, reopened, ready_for_review ] + paths: + - ".github/workflows/request-dataplane-review.yml" + - "**/*derp*" + - "**/derp*/**" + - "!**/depaware.txt" + +jobs: + request-dataplane-review: + if: github.event.pull_request.draft == false + name: Request Dataplane Review + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Get access token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + id: generate-token + with: + # Get token for app: https://github.com/apps/change-visibility-bot + app-id: ${{ secrets.VISIBILITY_BOT_APP_ID }} + private-key: ${{ secrets.VISIBILITY_BOT_APP_PRIVATE_KEY }} + - name: Add reviewers + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + url: ${{ github.event.pull_request.html_url }} + run: | + gh pr edit "$url" --add-reviewer tailscale/dataplane diff --git a/.github/workflows/ssh-integrationtest.yml b/.github/workflows/ssh-integrationtest.yml index 463f4bdd4b24f..84432cd729418 100644 --- a/.github/workflows/ssh-integrationtest.yml +++ b/.github/workflows/ssh-integrationtest.yml @@ -1,5 +1,5 @@ -# Run the ssh integration tests with `make sshintegrationtest`. -# These tests can also be running locally. +# Run the ssh integration tests in various Docker containers. +# These tests can also be run locally via `make sshintegrationtest`. name: "ssh-integrationtest" concurrency: @@ -15,9 +15,25 @@ on: jobs: ssh-integrationtest: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - base: "ubuntu:focal" + tag: "ssh-ubuntu-focal" + - base: "ubuntu:jammy" + tag: "ssh-ubuntu-jammy" + - base: "ubuntu:noble" + tag: "ssh-ubuntu-noble" + - base: "alpine:latest" + tag: "ssh-alpine-latest" steps: - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Run SSH integration tests + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Build test binaries run: | - make sshintegrationtest \ No newline at end of file + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled + - name: Run SSH integration tests (${{ matrix.base }}) + run: | + docker build --build-arg="BASE=${{ matrix.base }}" -t "${{ matrix.tag }}" ssh/tailssh/testcontainers diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d8ab863ce422..32546106042d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,7 @@ env: # toplevel directories "src" (for the checked out source code), and "gomodcache" # and other caches as siblings to follow. GOMODCACHE: ${{ github.workspace }}/gomodcache + CMD_GO_USE_GIT_HASH: "true" on: push: @@ -48,7 +49,7 @@ jobs: cache-key: ${{ steps.hash.outputs.key }} steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Compute cache key from go.{mod,sum} @@ -57,7 +58,7 @@ jobs: # See if the cache entry already exists to avoid downloading it # and doing the cache write again. - id: check-cache - uses: actions/cache/restore@v4 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache # relative to workspace; see env note at top of file key: ${{ steps.hash.outputs.key }} @@ -69,7 +70,7 @@ jobs: run: go mod download - name: Cache Go modules if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache # relative to workspace; see env note at top of file key: ${{ steps.hash.outputs.key }} @@ -88,11 +89,11 @@ jobs: - shard: '4/4' steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -126,31 +127,30 @@ jobs: needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} enableCrossOsArchive: true - name: Restore Cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: restore-cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - # Note: unlike the other setups, this is only grabbing the mod download - # cache, rather than the whole mod directory, as the download cache - # contains zips that can be unpacked in parallel faster than they can be - # fetched and extracted by tar + # Note: this is only restoring the build cache. Mod cache is shared amongst + # all jobs in the workflow. path: | ~/.cache/go-build ~\AppData\Local\go-build - # The -2- here should be incremented when the scheme of data to be - # cached changes (e.g. path above changes). - key: ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + key: ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} restore-keys: | - ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-2-${{ hashFiles('**/go.sum') }} - ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-2- + ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}- + ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}- + ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go- - name: build all if: matrix.buildflags == '' # skip on race builder working-directory: src @@ -177,13 +177,13 @@ jobs: run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper - name: test all working-directory: src - run: NOBASHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} + run: NOBASHDEBUG=true NOPWSHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} env: GOARCH: ${{ matrix.goarch }} TS_TEST_SHARD: ${{ matrix.shard }} - name: bench all working-directory: src - run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done) + run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run='^$' $(for x in $(git grep -l '^func Benchmark' | xargs dirname | sort | uniq); do echo "./$x"; done) env: GOARCH: ${{ matrix.goarch }} - name: check that no tracked files changed @@ -206,55 +206,140 @@ jobs: shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete + - name: Save Cache + # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. + if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + # Note: this is only saving the build cache. Mod cache is shared amongst + # all jobs in the workflow. + path: | + ~/.cache/go-build + ~\AppData\Local\go-build + key: ${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-${{ matrix.shard }}-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} windows: - runs-on: windows-2022 + permissions: + id-token: write # This is required for requesting the GitHub action identity JWT that can auth to cigocached + contents: read # This is required for actions/checkout + # ci-windows-github-1 is a 2022 GitHub-managed runner in our org with 8 cores + # and 32 GB of RAM. It is connected to a private Azure VNet that hosts cigocached. + # https://github.com/organizations/tailscale/settings/actions/github-hosted-runners/5 + runs-on: ci-windows-github-1 needs: gomod-cache + name: Windows (${{ matrix.name || matrix.shard}}) + strategy: + fail-fast: false # don't abort the entire matrix if one element fails + matrix: + include: + - key: "win-bench" + name: "benchmarks" + - key: "win-shard-1-2" + shard: "1/2" + - key: "win-shard-2-2" + shard: "2/2" steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - path: src + path: ${{ github.workspace }}/src + + - name: Restore Go module cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - - name: Install Go - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + - name: Set up cigocacher + id: cigocacher-setup + uses: ./src/.github/actions/go-cache with: - go-version-file: src/go.mod - cache: false + checkout-path: ${{ github.workspace }}/src + cache-dir: ${{ github.workspace }}/cigocacher + cigocached-url: ${{ vars.CIGOCACHED_AZURE_URL }} + cigocached-host: ${{ vars.CIGOCACHED_AZURE_HOST }} + + - name: test + shell: bash + if: matrix.key != 'win-bench' # skip on bench builder + working-directory: src + run: ./tool/go run ./cmd/testwrapper sharded:${{ matrix.shard }} + env: + NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI + + - name: bench all + shell: bash + if: matrix.key == 'win-bench' + working-directory: src + run: ./tool/go test -bench=. -benchtime=1x -run='^$' $(for x in $(git grep -l '^func Benchmark' | xargs dirname | sort | uniq); do echo "./$x"; done) + env: + NOPWSHDEBUG: "true" # to quiet tool/gocross/gocross-wrapper.ps1 in CI + + - name: Print stats + shell: pwsh + if: steps.cigocacher-setup.outputs.success == 'true' + env: + GOCACHEPROG: ${{ env.GOCACHEPROG }} + run: | + Invoke-Expression "$env:GOCACHEPROG --stats" | jq . + macos: + runs-on: macos-latest + needs: gomod-cache + steps: + - name: checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} enableCrossOsArchive: true - - name: Restore Cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: restore-cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - path: | - ~/.cache/go-build - ~\AppData\Local\go-build - # The -2- here should be incremented when the scheme of data to be - # cached changes (e.g. path above changes). - key: ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + path: ~/Library/Caches/go-build + key: ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} restore-keys: | - ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }} - ${{ github.job }}-${{ runner.os }}-go-2- - - name: test + ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}-${{ github.job }}- + ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-go-test- + - name: build test wrapper working-directory: src - run: go run ./cmd/testwrapper ./... - - name: bench all + run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper + - name: test all working-directory: src - # Don't use -bench=. -benchtime=1x. - # Somewhere in the layers (powershell?) - # the equals signs cause great confusion. - run: go test ./... -bench . -benchtime 1x -run "^$" + run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... + - name: check that no tracked files changed + working-directory: src + run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1) + - name: check that no new files were added + working-directory: src + run: | + # Note: The "error: pathspec..." you see below is normal! + # In the success case in which there are no new untracked files, + # git ls-files complains about the pathspec not matching anything. + # That's OK. It's not worth the effort to suppress. Please ignore it. + if git ls-files --others --exclude-standard --directory --no-empty-directory --error-unmatch -- ':/*' + then + echo "Build/test created untracked files in the repo (file names above)." + exit 1 + fi - name: Tidy cache working-directory: src - shell: bash run: | - find $(go env GOCACHE) -type f -mmin +90 -delete + find $(./tool/go env GOCACHE) -type f -mmin +90 -delete + - name: Save Cache + # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. + if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ~/Library/Caches/go-build + key: ${{ runner.os }}-go-test-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} privileged: needs: gomod-cache @@ -264,11 +349,11 @@ jobs: options: --privileged steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -278,7 +363,7 @@ jobs: run: chown -R $(id -u):$(id -g) $PWD - name: privileged tests working-directory: src - run: ./tool/go test ./util/linuxfw ./derp/xdp + run: ./tool/go test $(./tool/go run ./tool/listpkgs --has-root-tests) vm: needs: gomod-cache @@ -287,18 +372,18 @@ jobs: if: github.repository == 'tailscale/tailscale' steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} enableCrossOsArchive: true - name: Run VM tests working-directory: src - run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004 + run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2404 env: HOME: "/var/lib/ghrunner/home" TMPDIR: "/tmp" @@ -343,31 +428,29 @@ jobs: runs-on: ubuntu-24.04 steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src + - name: Restore Go module cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: Restore Cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: restore-cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - # Note: unlike the other setups, this is only grabbing the mod download - # cache, rather than the whole mod directory, as the download cache - # contains zips that can be unpacked in parallel faster than they can be - # fetched and extracted by tar + # Note: this is only restoring the build cache. Mod cache is shared amongst + # all jobs in the workflow. path: | ~/.cache/go-build ~\AppData\Local\go-build - # The -2- here should be incremented when the scheme of data to be - # cached changes (e.g. path above changes). - key: ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} restore-keys: | - ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }} - ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2- - - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true + ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}- + ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go- - name: build all working-directory: src run: ./tool/go build ./cmd/... @@ -388,6 +471,17 @@ jobs: shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete + - name: Save Cache + # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. + if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + # Note: this is only saving the build cache. Mod cache is shared amongst + # all jobs in the workflow. + path: | + ~/.cache/go-build + ~\AppData\Local\go-build + key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-${{ matrix.goarm }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} ios: # similar to cross above, but iOS can't build most of the repo. So, just # make it build a few smoke packages. @@ -395,11 +489,11 @@ jobs: needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -433,31 +527,29 @@ jobs: runs-on: ubuntu-24.04 steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src + - name: Restore Go module cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: Restore Cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: restore-cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - # Note: unlike the other setups, this is only grabbing the mod download - # cache, rather than the whole mod directory, as the download cache - # contains zips that can be unpacked in parallel faster than they can be - # fetched and extracted by tar + # Note: this is only restoring the build cache. Mod cache is shared amongst + # all jobs in the workflow. path: | ~/.cache/go-build ~\AppData\Local\go-build - # The -2- here should be incremented when the scheme of data to be - # cached changes (e.g. path above changes). - key: ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} restore-keys: | - ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }} - ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2- - - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true + ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}- + ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go- - name: build core working-directory: src run: ./tool/go build ./cmd/tailscale ./cmd/tailscaled @@ -471,6 +563,17 @@ jobs: shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete + - name: Save Cache + # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. + if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + # Note: this is only saving the build cache. Mod cache is shared amongst + # all jobs in the workflow. + path: | + ~/.cache/go-build + ~\AppData\Local\go-build + key: ${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} android: # similar to cross above, but android fails to build a few pieces of the @@ -480,7 +583,7 @@ jobs: needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed @@ -488,7 +591,7 @@ jobs: # some Android breakages early. # TODO(bradfitz): better; see https://github.com/tailscale/tailscale/issues/4482 - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -505,31 +608,29 @@ jobs: needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src + - name: Restore Go module cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: Restore Cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + id: restore-cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - # Note: unlike the other setups, this is only grabbing the mod download - # cache, rather than the whole mod directory, as the download cache - # contains zips that can be unpacked in parallel faster than they can be - # fetched and extracted by tar + # Note: this is only restoring the build cache. Mod cache is shared amongst + # all jobs in the workflow. path: | ~/.cache/go-build ~\AppData\Local\go-build - # The -2- here should be incremented when the scheme of data to be - # cached changes (e.g. path above changes). - key: ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + key: ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} restore-keys: | - ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }} - ${{ github.job }}-${{ runner.os }}-go-2- - - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: gomodcache - key: ${{ needs.gomod-cache.outputs.cache-key }} - enableCrossOsArchive: true + ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}- + ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-js-wasm-go- - name: build tsconnect client working-directory: src run: ./tool/go build ./cmd/tsconnect/wasm ./cmd/tailscale/cli @@ -543,22 +644,40 @@ jobs: run: | ./tool/go run ./cmd/tsconnect --fast-compression build ./tool/go run ./cmd/tsconnect --fast-compression build-pkg + - name: verify Google Chrome is available + run: | + which google-chrome + google-chrome --version + - name: tsconnect js/wasm headless-browser tests + working-directory: src + run: ./tool/go test ./tstest/integration/jswasmtest/ -v -timeout 180s --run-headless-browser-tests - name: Tidy cache working-directory: src shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete + - name: Save Cache + # Save cache even on failure, but only on cache miss and main branch to avoid thrashing. + if: always() && steps.restore-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + # Note: this is only saving the build cache. Mod cache is shared amongst + # all jobs in the workflow. + path: | + ~/.cache/go-build + ~\AppData\Local\go-build + key: ${{ runner.os }}-js-wasm-go-${{ hashFiles('**/go.sum') }}-${{ github.job }}-${{ github.run_id }} tailscale_go: # Subset of tests that depend on our custom Go toolchain. runs-on: ubuntu-24.04 needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set GOMODCACHE env run: echo "GOMODCACHE=$HOME/.cache/go-mod" >> $GITHUB_ENV - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -583,7 +702,9 @@ jobs: steps: - name: build fuzzers id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + # As of 12 February 2026, this repo doesn't tag releases, so this commit + # hash is just the tip of master. + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@f277aafb36f358582fdb24a41a9a52f2e097a2fd # continue-on-error makes steps.build.conclusion be 'success' even if # steps.build.outcome is 'failure'. This means this step does not # contribute to the job's overall pass/fail evaluation. @@ -613,7 +734,9 @@ jobs: # report a failure because TS_FUZZ_CURRENTLY_BROKEN is set to the wrong # value. if: steps.build.outcome == 'success' - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + # As of 12 February 2026, this repo doesn't tag releases, so this commit + # hash is just the tip of master. + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@f277aafb36f358582fdb24a41a9a52f2e097a2fd with: oss-fuzz-project-name: 'tailscale' fuzz-seconds: 150 @@ -624,7 +747,7 @@ jobs: run: | echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV - name: upload crash - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: steps.run.outcome != 'success' && steps.build.outcome == 'success' with: name: artifacts @@ -635,13 +758,13 @@ jobs: needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Set GOMODCACHE env run: echo "GOMODCACHE=$HOME/.cache/go-mod" >> $GITHUB_ENV - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -655,11 +778,11 @@ jobs: needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -669,42 +792,51 @@ jobs: run: | pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp') ./tool/go generate $pkgs + git add -N . # ensure untracked files are noticed echo echo git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1) + - name: check that 'genreadme' is clean + working-directory: src + run: | + ./tool/go run ./misc/genreadme + git add -N . # ensure untracked files are noticed + echo + echo + git diff --name-only --exit-code || (echo "The files above need updating. Please run './tool/go run ./misc/genreadme'."; exit 1) - go_mod_tidy: + make_tidy: runs-on: ubuntu-24.04 needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} enableCrossOsArchive: true - - name: check that 'go mod tidy' is clean + - name: check that 'make tidy' is clean working-directory: src run: | - ./tool/go mod tidy + make tidy echo echo - git diff --name-only --exit-code || (echo "Please run 'go mod tidy'."; exit 1) + git diff --name-only --exit-code || (echo "Please run 'make tidy'"; exit 1) licenses: runs-on: ubuntu-24.04 needs: gomod-cache steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -754,11 +886,11 @@ jobs: steps: - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: src - name: Restore Go module cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} @@ -775,10 +907,11 @@ jobs: notify_slack: if: always() # Any of these jobs failing causes a slack notification. - needs: + needs: - android - test - windows + - macos - vm - cross - ios @@ -787,7 +920,7 @@ jobs: - fuzz - depaware - go_generate - - go_mod_tidy + - make_tidy - licenses - staticcheck runs-on: ubuntu-24.04 @@ -801,7 +934,7 @@ jobs: # By having the job always run, but skipping its only step as needed, we # let the CI output collapse nicely in PRs. if: failure() && github.event_name == 'push' - uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: incoming-webhook @@ -824,6 +957,7 @@ jobs: - android - test - windows + - macos - vm - cross - ios @@ -832,7 +966,7 @@ jobs: - fuzz - depaware - go_generate - - go_mod_tidy + - make_tidy - licenses - staticcheck steps: @@ -856,7 +990,7 @@ jobs: - tailscale_go - depaware - go_generate - - go_mod_tidy + - make_tidy - licenses - staticcheck steps: @@ -873,6 +1007,7 @@ jobs: - check_mergeability_strict - test - windows + - macos - vm - wasm - fuzz diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index af7bdff1ee66d..ce77cf651ad42 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -8,7 +8,7 @@ on: - main paths: - go.mod - - .github/workflows/update-flakes.yml + - .github/workflows/update-flake.yml workflow_dispatch: concurrency: @@ -21,29 +21,28 @@ jobs: steps: - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Run update-flakes - run: ./update-flake.sh + - name: Run updateflakes + run: ./tool/go run ./tool/updateflakes - name: Get access token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 id: generate-token with: - app_id: ${{ secrets.LICENSING_APP_ID }} - installation_retrieval_mode: "id" - installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }} - private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }} + # Get token for app: https://github.com/apps/tailscale-code-updater + app-id: ${{ secrets.CODE_UPDATER_APP_ID }} + private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} - name: Send pull request - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8 + uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 #v8.1.0 with: token: ${{ steps.generate-token.outputs.token }} author: Flakes Updater committer: Flakes Updater branch: flakes - commit-message: "go.mod.sri: update SRI hash for go.mod changes" - title: "go.mod.sri: update SRI hash for go.mod changes" + commit-message: "flakehashes.json: update SRI hash for go.mod changes" + title: "flakehashes.json: update SRI hash for go.mod changes" body: Triggered by ${{ github.repository }}@${{ github.sha }} signoff: true delete-branch: true diff --git a/.github/workflows/update-webclient-prebuilt.yml b/.github/workflows/update-webclient-prebuilt.yml index f1c2b0c3b9368..5bb0573a1f18c 100644 --- a/.github/workflows/update-webclient-prebuilt.yml +++ b/.github/workflows/update-webclient-prebuilt.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run go get run: | @@ -23,19 +23,16 @@ jobs: ./tool/go mod tidy - name: Get access token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 id: generate-token with: - # TODO(will): this should use the code updater app rather than licensing. - # It has the same permissions, so not a big deal, but still. - app_id: ${{ secrets.LICENSING_APP_ID }} - installation_retrieval_mode: "id" - installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }} - private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }} + # Get token for app: https://github.com/apps/tailscale-code-updater + app-id: ${{ secrets.CODE_UPDATER_APP_ID }} + private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} - name: Send pull request id: pull-request - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8 + uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 #v8.1.0 with: token: ${{ steps.generate-token.outputs.token }} author: OSS Updater diff --git a/.github/workflows/vet.yml b/.github/workflows/vet.yml new file mode 100644 index 0000000000000..2af8f24b5c8e4 --- /dev/null +++ b/.github/workflows/vet.yml @@ -0,0 +1,47 @@ +name: tailscale.com/cmd/vet + +env: + HOME: ${{ github.workspace }} + # GOMODCACHE is the same definition on all OSes. Within the workspace, we use + # toplevel directories "src" (for the checked out source code), and "gomodcache" + # and other caches as siblings to follow. + GOMODCACHE: ${{ github.workspace }}/gomodcache + CMD_GO_USE_GIT_HASH: "true" + +on: + push: + branches: + - main + - "release-branch/*" + paths: + - .github/workflows/vet.yml + - "**.go" + pull_request: + paths: + - .github/workflows/vet.yml + - "**.go" + +jobs: + vet: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src + + - name: Build 'go vet' tool + working-directory: src + run: ./tool/go build -o /tmp/vettool tailscale.com/cmd/vet + + - name: Run 'go vet' + working-directory: src + # Use listpkgs --ignore-3p to skip tempfork/ packages, which + # intentionally match upstream and may not follow our style rules. + # Must use ./... instead of tailscale.com/... because the latter will + # include the v2 go client (tailscale.com/client/tailscale/v2) if it's + # a dependency in our go.mod file. Possibly a go vet bug, but avoid + # cross-repo vetting for now so we can safely add the dependency. + run: ./tool/go vet -vettool=/tmp/vettool $(./tool/go run ./tool/listpkgs --ignore-3p ./...) diff --git a/.github/workflows/webclient.yml b/.github/workflows/webclient.yml index e64137f2b160d..1a65eacf56414 100644 --- a/.github/workflows/webclient.yml +++ b/.github/workflows/webclient.yml @@ -3,8 +3,6 @@ on: workflow_dispatch: # For now, only run on requests, not the main branches. pull_request: - branches: - - "*" paths: - "client/web/**" - ".github/workflows/webclient.yml" @@ -24,7 +22,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install deps run: ./tool/yarn --cwd client/web - name: Run lint diff --git a/.gitignore b/.gitignore index 47d2bbe959ae1..e1f6be02e002f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,15 @@ # Binaries for programs and plugins *~ *.tmp -*.exe *.dll *.so *.dylib *.spk +*.exe +# tool/go.exe is built specially and committed. +!/tool/go.exe + cmd/tailscale/tailscale cmd/tailscaled/tailscaled ssh/tailssh/testcontainers/tailscaled @@ -49,3 +52,9 @@ client/web/build/assets *.xcworkspacedata /tstest/tailmac/bin /tstest/tailmac/build + +# Ignore personal IntelliJ settings +.idea/ + +# Ignore syncthing state directory. +/.stfolder diff --git a/.golangci.yml b/.golangci.yml index eb34f9d9efc76..ff8bd07228677 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -10,9 +10,15 @@ linters: enable: - bidichk - govet + - importas - misspell - revive settings: + importas: + no-unaliased: true + alias: + - pkg: github.com/tailscale/gliderssh + alias: gliderssh # Matches what we use in corp as of 2023-12-07 govet: enable: diff --git a/.stignore b/.stignore new file mode 120000 index 0000000000000..3e4e48b0b5fe6 --- /dev/null +++ b/.stignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/ALPINE.txt b/ALPINE.txt index 318956c3d51e2..93a84c380075c 100644 --- a/ALPINE.txt +++ b/ALPINE.txt @@ -1 +1 @@ -3.19 \ No newline at end of file +3.22 \ No newline at end of file diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 03d5932c04746..0000000000000 --- a/AUTHORS +++ /dev/null @@ -1,17 +0,0 @@ -# This is the official list of Tailscale -# authors for copyright purposes. -# -# Names should be added to this file as one of -# Organization's name -# Individual's name -# Individual's name -# -# Please keep the list sorted. -# -# You do not need to add entries to this list, and we don't actively -# populate this list. If you do want to be acknowledged explicitly as -# a copyright holder, though, then please send a PR referencing your -# earlier contributions and clarifying whether it's you or your -# company that owns the rights to your contribution. - -Tailscale Inc. diff --git a/CODEOWNERS b/CODEOWNERS index af9b0d9f95928..9a43041052ffe 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,4 @@ /tailcfg/ @tailscale/control-protocol-owners + +# Do not add to this list without wide discussion. +# See https://github.com/tailscale/corp/issues/13972 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index be5564ef4a3de..348483df57558 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,135 +1,103 @@ -# Contributor Covenant Code of Conduct +# Tailscale Community Code of Conduct ## Our Pledge -We as members, contributors, and leaders pledge to make participation -in our community a harassment-free experience for everyone, regardless -of age, body size, visible or invisible disability, ethnicity, sex -characteristics, gender identity and expression, level of experience, -education, socio-economic status, nationality, personal appearance, -race, religion, or sexual identity and orientation. - -We pledge to act and interact in ways that contribute to an open, -welcoming, diverse, inclusive, and healthy community. +We are committed to creating an open, welcoming, diverse, inclusive, healthy and respectful community. +Unacceptable, harmful and inappropriate behavior will not be tolerated. ## Our Standards -Examples of behavior that contributes to a positive environment for -our community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our - mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community - -Examples of unacceptable behavior include: +Examples of behavior that contributes to a positive environment for our community include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind -* Trolling, insulting or derogatory comments, and personal or - political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in - a professional setting +- Demonstrating empathy and kindness toward other people. +- Being respectful of differing opinions, viewpoints, and experiences. +- Giving and gracefully accepting constructive feedback. +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience. +- Focusing on what is best not just for us as individuals, but for the overall community. -## Enforcement Responsibilities +Examples of unacceptable behavior include without limitation: -Community leaders are responsible for clarifying and enforcing our -standards of acceptable behavior and will take appropriate and fair -corrective action in response to any behavior that they deem -inappropriate, threatening, offensive, or harmful. +- The use of language, imagery or emojis (collectively "content") that is racist, sexist, homophobic, transphobic, or otherwise harassing or discriminatory based on any protected characteristic. +- The use of sexualized content and sexual attention or advances of any kind. +- The use of violent, intimidating or bullying content. +- Trolling, concern trolling, insulting or derogatory comments, and personal or political attacks. +- Public or private harassment. +- Publishing others' personal information, such as a photo, physical address, email address, online profile information, or other personal information, without their explicit permission or with the intent to bully or harass the other person. +- Posting deep fake or other AI generated content about or involving another person without the explicit permission. +- Spamming community channels and members, such as sending repeat messages, low-effort content, or automated messages. +- Phishing or any similar activity. +- Distributing or promoting malware. +- The use of any coded or suggestive content to hide or provoke otherwise unacceptable behavior. +- Other conduct which could reasonably be considered harmful, illegal, or inappropriate in a professional setting. -Community leaders have the right and responsibility to remove, edit, -or reject comments, commits, code, wiki edits, issues, and other -contributions that are not aligned to this Code of Conduct, and will -communicate reasons for moderation decisions when appropriate. +Please also see the Tailscale Acceptable Use Policy, available at [tailscale.com/tailscale-aup](https://tailscale.com/tailscale-aup). -## Scope +## Reporting Incidents -This Code of Conduct applies within all community spaces, and also -applies when an individual is officially representing the community in -public spaces. Examples of representing our community include using an -official e-mail address, posting via an official social media account, -or acting as an appointed representative at an online or offline -event. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to Tailscale directly via , or to the community leaders or moderators via DM or similar. +All complaints will be reviewed and investigated promptly and fairly. +We will respect the privacy and safety of the reporter of any issues. -## Enforcement +Please note that this community is not moderated by staff 24/7, and we do not have, and do not undertake, any obligation to prescreen, monitor, edit, or remove any content or data, or to actively seek facts or circumstances indicating illegal activity. +While we strive to keep the community safe and welcoming, moderation may not be immediate at all hours. +If you encounter any issues, report them using the appropriate channels. -Instances of abusive, harassing, or otherwise unacceptable behavior -may be reported to the community leaders responsible for enforcement -at [info@tailscale.com](mailto:info@tailscale.com). All complaints -will be reviewed and investigated promptly and fairly. +## Enforcement Guidelines -All community leaders are obligated to respect the privacy and -security of the reporter of any incident. +Community leaders and moderators are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. -## Enforcement Guidelines +Community leaders and moderators have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Community Code of Conduct. +Tailscale retains full discretion to take action (or not) in response to a violation of these guidelines with or without notice or liability to you. +We will interpret our policies and resolve disputes in favor of protecting users, customers, the public, our community and our company, as a whole. -Community leaders will follow these Community Impact Guidelines in -determining the consequences for any action they deem in violation of -this Code of Conduct: +Community leaders will follow these community enforcement guidelines in determining the consequences for any action they deem in violation of this Code of Conduct, +and retain full discretion to apply the enforcement guidelines as necessary depending on the circumstances: ### 1. Correction -**Community Impact**: Use of inappropriate language or other behavior -deemed unprofessional or unwelcome in the community. +Community Impact: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. -**Consequence**: A private, written warning from community leaders, -providing clarity around the nature of the violation and an -explanation of why the behavior was inappropriate. A public apology -may be requested. +Consequence: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. +A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +Community Impact: A violation through a single incident or series of actions. -**Consequence**: A warning with consequences for continued -behavior. No interaction with the people involved, including -unsolicited interaction with those enforcing the Code of Conduct, for -a specified period of time. This includes avoiding interactions in -community spaces as well as external channels like social -media. Violating these terms may lead to a temporary or permanent ban. +Consequence: A warning with consequences for continued behavior. +No interaction with the people involved, including unsolicited interaction with those enforcing this Community Code of Conduct, for a specified period of time. +This includes avoiding interactions in community spaces as well as external channels like social media. +Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban -**Community Impact**: A serious violation of community standards, -including sustained inappropriate behavior. +Community Impact: A serious violation of community standards, including sustained inappropriate behavior. -**Consequence**: A temporary ban from any sort of interaction or -public communication with the community for a specified period of -time. No public or private interaction with the people involved, -including unsolicited interaction with those enforcing the Code of -Conduct, is allowed during this period. Violating these terms may lead -to a permanent ban. +Consequence: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. +No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of -community standards, including sustained inappropriate behavior, -harassment of an individual, or aggression toward or disparagement of -classes of individuals. +Community Impact: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction -within the community. +Consequence: A permanent ban from any sort of public interaction within the community. + +## Acceptable Use Policy + +Violation of this Community Code of Conduct may also violate the Tailscale Acceptable Use Policy, which may result in suspension or termination of your Tailscale account. +For more information, please see the Tailscale Acceptable Use Policy, available at [tailscale.com/tailscale-aup](https://tailscale.com/tailscale-aup). + +## Privacy + +Please see the Tailscale [Privacy Policy](https://tailscale.com/privacy-policy) for more information about how Tailscale collects, uses, discloses and protects information. ## Attribution -This Code of Conduct is adapted from the [Contributor -Covenant][homepage], version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at . -Community Impact Guidelines were inspired by [Mozilla's code of -conduct enforcement ladder](https://github.com/mozilla/diversity). +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org -For answers to common questions about this code of conduct, see the -FAQ at https://www.contributor-covenant.org/faq. Translations are -available at https://www.contributor-covenant.org/translations. - +For answers to common questions about this code of conduct, see the FAQ at . +Translations are available at . diff --git a/Dockerfile b/Dockerfile index 015022e49fc28..ee12922f2d719 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause # Note that this Dockerfile is currently NOT used to build any of the published @@ -7,6 +7,15 @@ # Tailscale images are currently built using https://github.com/tailscale/mkctr, # and the build script can be found in ./build_docker.sh. # +# If you want to build local images for testing, you can use make. +# +# To build a Tailscale image and push to the local docker registry: +# +# $ REPO=local/tailscale TAGS=v0.0.1 PLATFORM=local make publishdevimage +# +# To build a Tailscale image and push to a remote docker registry: +# +# $ REPO=//tailscale TAGS=v0.0.1 make publishdevimage # # This Dockerfile includes all the tailscale binaries. # @@ -27,7 +36,7 @@ # $ docker exec tailscaled tailscale status -FROM golang:1.24-alpine AS build-env +FROM golang:1.26-alpine AS build-env WORKDIR /go/src/tailscale @@ -62,10 +71,15 @@ RUN GOARCH=$TARGETARCH go install -ldflags="\ -X tailscale.com/version.gitCommitStamp=$VERSION_GIT_HASH" \ -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot -FROM alpine:3.19 +FROM alpine:3.22 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables -RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables -RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables +# Alpine 3.19 replaced legacy iptables with nftables based implementation. +# Tailscale is used on some hosts that don't support nftables, such as Synology +# NAS, so link iptables back to legacy version. Hosts that don't require legacy +# iptables should be able to use Tailscale in nftables mode. See +# https://github.com/tailscale/tailscale/issues/17854 +RUN rm /usr/sbin/iptables && ln -s /usr/sbin/iptables-legacy /usr/sbin/iptables +RUN rm /usr/sbin/ip6tables && ln -s /usr/sbin/ip6tables-legacy /usr/sbin/ip6tables COPY --from=build-env /go/bin/* /usr/local/bin/ # For compat with the previous run.sh, although ideally you should be diff --git a/Dockerfile.base b/Dockerfile.base index b7e79a43c6fdf..295950c461339 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -1,12 +1,12 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause -FROM alpine:3.19 +FROM alpine:3.22 RUN apk add --no-cache ca-certificates iptables iptables-legacy iproute2 ip6tables iputils -# Alpine 3.19 replaces legacy iptables with nftables based implementation. We -# can't be certain that all hosts that run Tailscale containers currently -# suppport nftables, so link back to legacy for backwards compatibility reasons. -# TODO(irbekrm): add some way how to determine if we still run on nodes that -# don't support nftables, so that we can eventually remove these symlinks. -RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables -RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables +# Alpine 3.19 replaced legacy iptables with nftables based implementation. +# Tailscale is used on some hosts that don't support nftables, such as Synology +# NAS, so link iptables back to legacy version. Hosts that don't require legacy +# iptables should be able to use Tailscale in nftables mode. See +# https://github.com/tailscale/tailscale/issues/17854 +RUN rm /usr/sbin/iptables && ln -s /usr/sbin/iptables-legacy /usr/sbin/iptables +RUN rm /usr/sbin/ip6tables && ln -s /usr/sbin/ip6tables-legacy /usr/sbin/ip6tables diff --git a/LICENSE b/LICENSE index 394db19e4aa5c..ed6e4bb6d6ba6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2020 Tailscale Inc & AUTHORS. +Copyright (c) 2020 Tailscale Inc & contributors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Makefile b/Makefile index 41c67c711791d..0efd57fb486d6 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,9 @@ PLATFORM ?= "flyio" ## flyio==linux/amd64. Set to "" to build all platforms. vet: ## Run go vet ./tool/go vet ./... -tidy: ## Run go mod tidy +tidy: ## Run go mod tidy and update nix flake hashes ./tool/go mod tidy + ./tool/go run ./tool/updateflakes lint: ## Run golangci-lint ./tool/go run github.com/golangci/golangci-lint/cmd/golangci-lint run @@ -17,28 +18,36 @@ lint: ## Run golangci-lint updatedeps: ## Update depaware deps # depaware (via x/tools/go/packages) shells back to "go", so make sure the "go" # it finds in its $$PATH is the right one. - PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --internal \ + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --vendor --internal \ tailscale.com/cmd/tailscaled \ tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ tailscale.com/cmd/k8s-operator \ tailscale.com/cmd/stund \ tailscale.com/cmd/tsidp - PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update -goos=linux,darwin,windows,android,ios --internal \ + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --goos=linux,darwin,windows,android,ios --vendor --internal \ tailscale.com/tsnet + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --file=depaware-minbox.txt --goos=linux --tags="$$(./tool/go run ./cmd/featuretags --min --add=cli)" --vendor --internal \ + tailscale.com/cmd/tailscaled + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update --file=depaware-min.txt --goos=linux --tags="$$(./tool/go run ./cmd/featuretags --min)" --vendor --internal \ + tailscale.com/cmd/tailscaled depaware: ## Run depaware checks # depaware (via x/tools/go/packages) shells back to "go", so make sure the "go" # it finds in its $$PATH is the right one. - PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --internal \ + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --vendor --internal \ tailscale.com/cmd/tailscaled \ tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ tailscale.com/cmd/k8s-operator \ tailscale.com/cmd/stund \ tailscale.com/cmd/tsidp - PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --goos=linux,darwin,windows,android,ios --internal \ + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --goos=linux,darwin,windows,android,ios --vendor --internal \ tailscale.com/tsnet + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --file=depaware-minbox.txt --goos=linux --tags="$$(./tool/go run ./cmd/featuretags --min --add=cli)" --vendor --internal \ + tailscale.com/cmd/tailscaled + PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --file=depaware-min.txt --goos=linux --tags="$$(./tool/go run ./cmd/featuretags --min)" --vendor --internal \ + tailscale.com/cmd/tailscaled buildwindows: ## Build tailscale CLI for windows/amd64 GOOS=windows GOARCH=amd64 ./tool/go install tailscale.com/cmd/tailscale tailscale.com/cmd/tailscaled @@ -92,50 +101,62 @@ pushspk: spk ## Push and install synology package on ${SYNO_HOST} host scp tailscale.spk root@${SYNO_HOST}: ssh root@${SYNO_HOST} /usr/syno/bin/synopkg install tailscale.spk -publishdevimage: ## Build and publish tailscale image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +.PHONY: check-image-repo +check-image-repo: + @if [ -z "$(REPO)" ]; then \ + echo "REPO=... required; e.g. REPO=ghcr.io/$$USER/tailscale" >&2; \ + exit 1; \ + fi + @for repo in tailscale/tailscale ghcr.io/tailscale/tailscale \ + tailscale/k8s-operator ghcr.io/tailscale/k8s-operator \ + tailscale/k8s-nameserver ghcr.io/tailscale/k8s-nameserver \ + tailscale/tsidp ghcr.io/tailscale/tsidp \ + tailscale/k8s-proxy ghcr.io/tailscale/k8s-proxy; do \ + if [ "$(REPO)" = "$$repo" ]; then \ + echo "REPO=... must not be $$repo" >&2; \ + exit 1; \ + fi; \ + done + +publishdevimage: check-image-repo ## Build and publish tailscale image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=client ./build_docker.sh -publishdevoperator: ## Build and publish k8s-operator image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +publishdevoperator: check-image-repo ## Build and publish k8s-operator image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh -publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-nameserver" || (echo "REPO=... must not be tailscale/k8s-nameserver" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1) +publishdevnameserver: check-image-repo ## Build and publish k8s-nameserver image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh -publishdevtsidp: ## Build and publish tsidp image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tsidp" || (echo "REPO=... must not be tailscale/tsidp" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tsidp" || (echo "REPO=... must not be ghcr.io/tailscale/tsidp" && exit 1) +publishdevtsidp: check-image-repo ## Build and publish tsidp image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=tsidp ./build_docker.sh +publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to location specified by ${REPO} + TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-proxy ./build_docker.sh + .PHONY: sshintegrationtest sshintegrationtest: ## Run the SSH integration tests in various Docker containers - @GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ - GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ - echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \ - echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \ - echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \ - echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers + @GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ + echo "Testing on ubuntu:focal, ubuntu:jammy, ubuntu:noble, alpine:latest (in parallel)" && \ + docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers & \ + docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers & \ + docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers & \ + docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers & \ + wait + +.PHONY: generate +generate: ## Generate code + ./tool/go generate ./... + +.PHONY: pin-github-actions +pin-github-actions: + ./tool/go tool github.com/stacklok/frizbee actions .github/workflows help: ## Show this help - @echo "\nSpecify a command. The choices are:\n" - @grep -hE '^[0-9a-zA-Z_-]+:.*?## .*$$' ${MAKEFILE_LIST} | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-20s\033[m %s\n", $$1, $$2}' + @echo "" + @echo "Specify a command. The choices are:" + @echo "" + @grep -hE '^[0-9a-zA-Z_-]+:.*?## .*$$' ${MAKEFILE_LIST} | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-20s\033[m %s\n", $$1, $$2}' @echo "" .PHONY: help diff --git a/README.md b/README.md index 2c9713a6f339c..1d8208a867814 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ not open source. ## Building -We always require the latest Go release, currently Go 1.23. (While we build +We always require the latest Go release, currently Go 1.26. (While we build releases with our [Go fork](https://github.com/tailscale/go/), its use is not required.) diff --git a/VERSION.txt b/VERSION.txt index f288d11142d11..f19e66773133d 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.85.0 +1.101.0 diff --git a/appc/appconnector.go b/appc/appconnector.go index 89c6c9aeb9aa7..ee495bd10f100 100644 --- a/appc/appconnector.go +++ b/appc/appconnector.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package appc implements App Connectors. @@ -12,19 +12,20 @@ package appc import ( "context" "fmt" + "maps" "net/netip" "slices" "strings" - "sync" "time" - "golang.org/x/net/dns/dnsmessage" + "tailscale.com/syncs" + "tailscale.com/types/appctype" "tailscale.com/types/logger" "tailscale.com/types/views" "tailscale.com/util/clientmetric" "tailscale.com/util/dnsname" + "tailscale.com/util/eventbus" "tailscale.com/util/execqueue" - "tailscale.com/util/mak" "tailscale.com/util/slicesx" ) @@ -115,19 +116,6 @@ func metricStoreRoutes(rate, nRoutes int64) { recordMetric(nRoutes, metricStoreRoutesNBuckets, metricStoreRoutesN) } -// RouteInfo is a data structure used to persist the in memory state of an AppConnector -// so that we can know, even after a restart, which routes came from ACLs and which were -// learned from domains. -type RouteInfo struct { - // Control is the routes from the 'routes' section of an app connector acl. - Control []netip.Prefix `json:",omitempty"` - // Domains are the routes discovered by observing DNS lookups for configured domains. - Domains map[string][]netip.Addr `json:",omitempty"` - // Wildcards are the configured DNS lookup domains to observe. When a DNS query matches Wildcards, - // its result is added to Domains. - Wildcards []string `json:",omitempty"` -} - // AppConnector is an implementation of an AppConnector that performs // its function as a subsystem inside of a tailscale node. At the control plane // side App Connector routing is configured in terms of domains rather than IP @@ -138,14 +126,20 @@ type RouteInfo struct { // routes not yet served by the AppConnector the local node configuration is // updated to advertise the new route. type AppConnector struct { + // These fields are immutable after initialization. logf logger.Logf + eventBus *eventbus.Bus routeAdvertiser RouteAdvertiser + pubClient *eventbus.Client + updatePub *eventbus.Publisher[appctype.RouteUpdate] + storePub *eventbus.Publisher[appctype.RouteInfo] - // storeRoutesFunc will be called to persist routes if it is not nil. - storeRoutesFunc func(*RouteInfo) error + // hasStoredRoutes records whether the connector was initialized with + // persisted route information. + hasStoredRoutes bool // mu guards the fields that follow - mu sync.Mutex + mu syncs.Mutex // domains is a map of lower case domain names with no trailing dot, to an // ordered list of resolved IP addresses. @@ -164,53 +158,83 @@ type AppConnector struct { writeRateDay *rateLogger } +// Config carries the settings for an [AppConnector]. +type Config struct { + // Logf is the logger to which debug logs from the connector will be sent. + // It must be non-nil. + Logf logger.Logf + + // EventBus receives events when the collection of routes maintained by the + // connector is updated. It must be non-nil. + EventBus *eventbus.Bus + + // RouteAdvertiser allows the connector to update the set of advertised routes. + RouteAdvertiser RouteAdvertiser + + // RouteInfo, if non-nil, use used as the initial set of routes for the + // connector. If nil, the connector starts empty. + RouteInfo *appctype.RouteInfo + + // HasStoredRoutes indicates that the connector should assume stored routes. + HasStoredRoutes bool +} + // NewAppConnector creates a new AppConnector. -func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, routeInfo *RouteInfo, storeRoutesFunc func(*RouteInfo) error) *AppConnector { +func NewAppConnector(c Config) *AppConnector { + switch { + case c.Logf == nil: + panic("missing logger") + case c.EventBus == nil: + panic("missing event bus") + } + ec := c.EventBus.Client("appc.AppConnector") + ac := &AppConnector{ - logf: logger.WithPrefix(logf, "appc: "), - routeAdvertiser: routeAdvertiser, - storeRoutesFunc: storeRoutesFunc, + logf: logger.WithPrefix(c.Logf, "appc: "), + eventBus: c.EventBus, + pubClient: ec, + updatePub: eventbus.Publish[appctype.RouteUpdate](ec), + storePub: eventbus.Publish[appctype.RouteInfo](ec), + routeAdvertiser: c.RouteAdvertiser, + hasStoredRoutes: c.HasStoredRoutes, } - if routeInfo != nil { - ac.domains = routeInfo.Domains - ac.wildcards = routeInfo.Wildcards - ac.controlRoutes = routeInfo.Control + if c.RouteInfo != nil { + ac.domains = c.RouteInfo.Domains + ac.wildcards = c.RouteInfo.Wildcards + ac.controlRoutes = c.RouteInfo.Control } - ac.writeRateMinute = newRateLogger(time.Now, time.Minute, func(c int64, s time.Time, l int64) { - ac.logf("routeInfo write rate: %d in minute starting at %v (%d routes)", c, s, l) - metricStoreRoutes(c, l) + ac.writeRateMinute = newRateLogger(time.Now, time.Minute, func(c int64, s time.Time, ln int64) { + ac.logf("routeInfo write rate: %d in minute starting at %v (%d routes)", c, s, ln) + metricStoreRoutes(c, ln) }) - ac.writeRateDay = newRateLogger(time.Now, 24*time.Hour, func(c int64, s time.Time, l int64) { - ac.logf("routeInfo write rate: %d in 24 hours starting at %v (%d routes)", c, s, l) + ac.writeRateDay = newRateLogger(time.Now, 24*time.Hour, func(c int64, s time.Time, ln int64) { + ac.logf("routeInfo write rate: %d in 24 hours starting at %v (%d routes)", c, s, ln) }) return ac } // ShouldStoreRoutes returns true if the appconnector was created with the controlknob on // and is storing its discovered routes persistently. -func (e *AppConnector) ShouldStoreRoutes() bool { - return e.storeRoutesFunc != nil -} +func (e *AppConnector) ShouldStoreRoutes() bool { return e.hasStoredRoutes } // storeRoutesLocked takes the current state of the AppConnector and persists it -func (e *AppConnector) storeRoutesLocked() error { - if !e.ShouldStoreRoutes() { - return nil - } - - // log write rate and write size - numRoutes := int64(len(e.controlRoutes)) - for _, rs := range e.domains { - numRoutes += int64(len(rs)) +func (e *AppConnector) storeRoutesLocked() { + if e.storePub.ShouldPublish() { + // log write rate and write size + numRoutes := int64(len(e.controlRoutes)) + for _, rs := range e.domains { + numRoutes += int64(len(rs)) + } + e.writeRateMinute.update(numRoutes) + e.writeRateDay.update(numRoutes) + + e.storePub.Publish(appctype.RouteInfo{ + // Clone here, as the subscriber will handle these outside our lock. + Control: slices.Clone(e.controlRoutes), + Domains: maps.Clone(e.domains), + Wildcards: slices.Clone(e.wildcards), + }) } - e.writeRateMinute.update(numRoutes) - e.writeRateDay.update(numRoutes) - - return e.storeRoutesFunc(&RouteInfo{ - Control: e.controlRoutes, - Domains: e.domains, - Wildcards: e.wildcards, - }) } // ClearRoutes removes all route state from the AppConnector. @@ -220,7 +244,8 @@ func (e *AppConnector) ClearRoutes() error { e.controlRoutes = nil e.domains = nil e.wildcards = nil - return e.storeRoutesLocked() + e.storeRoutesLocked() + return nil } // UpdateDomainsAndRoutes starts an asynchronous update of the configuration @@ -249,6 +274,18 @@ func (e *AppConnector) Wait(ctx context.Context) { e.queue.Wait(ctx) } +// Close closes the connector and cleans up resources associated with it. +// It is safe (and a noop) to call Close on nil. +func (e *AppConnector) Close() { + if e == nil { + return + } + e.mu.Lock() + defer e.mu.Unlock() + e.queue.Shutdown() // TODO(creachadair): Should we wait for it too? + e.pubClient.Close() +} + func (e *AppConnector) updateDomains(domains []string) { e.mu.Lock() defer e.mu.Unlock() @@ -280,20 +317,26 @@ func (e *AppConnector) updateDomains(domains []string) { } } - // Everything left in oldDomains is a domain we're no longer tracking - // and if we are storing route info we can unadvertise the routes - if e.ShouldStoreRoutes() { + // Everything left in oldDomains is a domain we're no longer tracking and we + // can unadvertise the routes. + if e.hasStoredRoutes { toRemove := []netip.Prefix{} for _, addrs := range oldDomains { for _, a := range addrs { toRemove = append(toRemove, netip.PrefixFrom(a, a.BitLen())) } } - e.queue.Add(func() { - if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil { - e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err) + + if len(toRemove) != 0 { + if ra := e.routeAdvertiser; ra != nil { + e.queue.Add(func() { + if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil { + e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err) + } + }) } - }) + e.updatePub.Publish(appctype.RouteUpdate{Unadvertise: toRemove}) + } } e.logf("handling domains: %v and wildcards: %v", slicesx.MapKeys(e.domains), e.wildcards) @@ -314,11 +357,10 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) { var toRemove []netip.Prefix - // If we're storing routes and know e.controlRoutes is a good - // representation of what should be in AdvertisedRoutes we can stop - // advertising routes that used to be in e.controlRoutes but are not - // in routes. - if e.ShouldStoreRoutes() { + // If we know e.controlRoutes is a good representation of what should be in + // AdvertisedRoutes we can stop advertising routes that used to be in + // e.controlRoutes but are not in routes. + if e.hasStoredRoutes { toRemove = routesWithout(e.controlRoutes, routes) } @@ -335,19 +377,23 @@ nextRoute: } } - e.queue.Add(func() { - if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil { - e.logf("failed to advertise routes: %v: %v", routes, err) - } - if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil { - e.logf("failed to unadvertise routes: %v: %v", toRemove, err) - } + if e.routeAdvertiser != nil { + e.queue.Add(func() { + if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil { + e.logf("failed to advertise routes: %v: %v", routes, err) + } + if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil { + e.logf("failed to unadvertise routes: %v: %v", toRemove, err) + } + }) + } + e.updatePub.Publish(appctype.RouteUpdate{ + Advertise: routes, + Unadvertise: toRemove, }) e.controlRoutes = routes - if err := e.storeRoutesLocked(); err != nil { - e.logf("failed to store route info: %v", err) - } + e.storeRoutesLocked() } // Domains returns the currently configured domain list. @@ -372,124 +418,6 @@ func (e *AppConnector) DomainRoutes() map[string][]netip.Addr { return drCopy } -// ObserveDNSResponse is a callback invoked by the DNS resolver when a DNS -// response is being returned over the PeerAPI. The response is parsed and -// matched against the configured domains, if matched the routeAdvertiser is -// advised to advertise the discovered route. -func (e *AppConnector) ObserveDNSResponse(res []byte) error { - var p dnsmessage.Parser - if _, err := p.Start(res); err != nil { - return err - } - if err := p.SkipAllQuestions(); err != nil { - return err - } - - // cnameChain tracks a chain of CNAMEs for a given query in order to reverse - // a CNAME chain back to the original query for flattening. The keys are - // CNAME record targets, and the value is the name the record answers, so - // for www.example.com CNAME example.com, the map would contain - // ["example.com"] = "www.example.com". - var cnameChain map[string]string - - // addressRecords is a list of address records found in the response. - var addressRecords map[string][]netip.Addr - - for { - h, err := p.AnswerHeader() - if err == dnsmessage.ErrSectionDone { - break - } - if err != nil { - return err - } - - if h.Class != dnsmessage.ClassINET { - if err := p.SkipAnswer(); err != nil { - return err - } - continue - } - - switch h.Type { - case dnsmessage.TypeCNAME, dnsmessage.TypeA, dnsmessage.TypeAAAA: - default: - if err := p.SkipAnswer(); err != nil { - return err - } - continue - - } - - domain := strings.TrimSuffix(strings.ToLower(h.Name.String()), ".") - if len(domain) == 0 { - continue - } - - if h.Type == dnsmessage.TypeCNAME { - res, err := p.CNAMEResource() - if err != nil { - return err - } - cname := strings.TrimSuffix(strings.ToLower(res.CNAME.String()), ".") - if len(cname) == 0 { - continue - } - mak.Set(&cnameChain, cname, domain) - continue - } - - switch h.Type { - case dnsmessage.TypeA: - r, err := p.AResource() - if err != nil { - return err - } - addr := netip.AddrFrom4(r.A) - mak.Set(&addressRecords, domain, append(addressRecords[domain], addr)) - case dnsmessage.TypeAAAA: - r, err := p.AAAAResource() - if err != nil { - return err - } - addr := netip.AddrFrom16(r.AAAA) - mak.Set(&addressRecords, domain, append(addressRecords[domain], addr)) - default: - if err := p.SkipAnswer(); err != nil { - return err - } - continue - } - } - - e.mu.Lock() - defer e.mu.Unlock() - - for domain, addrs := range addressRecords { - domain, isRouted := e.findRoutedDomainLocked(domain, cnameChain) - - // domain and none of the CNAMEs in the chain are routed - if !isRouted { - continue - } - - // advertise each address we have learned for the routed domain, that - // was not already known. - var toAdvertise []netip.Prefix - for _, addr := range addrs { - if !e.isAddrKnownLocked(domain, addr) { - toAdvertise = append(toAdvertise, netip.PrefixFrom(addr, addr.BitLen())) - } - } - - if len(toAdvertise) > 0 { - e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise) - e.scheduleAdvertisement(domain, toAdvertise...) - } - } - return nil -} - // starting from the given domain that resolved to an address, find it, or any // of the domains in the CNAME chain toward resolving it, that are routed // domains, returning the routed domain name and a bool indicating whether a @@ -544,10 +472,13 @@ func (e *AppConnector) isAddrKnownLocked(domain string, addr netip.Addr) bool { // associated with the given domain. func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Prefix) { e.queue.Add(func() { - if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil { - e.logf("failed to advertise routes for %s: %v: %v", domain, routes, err) - return + if e.routeAdvertiser != nil { + if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil { + e.logf("failed to advertise routes for %s: %v: %v", domain, routes, err) + return + } } + e.updatePub.Publish(appctype.RouteUpdate{Advertise: routes}) e.mu.Lock() defer e.mu.Unlock() @@ -561,9 +492,7 @@ func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Pref e.logf("[v2] advertised route for %v: %v", domain, addr) } } - if err := e.storeRoutesLocked(); err != nil { - e.logf("failed to store route info: %v", err) - } + e.storeRoutesLocked() }) } @@ -581,8 +510,8 @@ func (e *AppConnector) addDomainAddrLocked(domain string, addr netip.Addr) { slices.SortFunc(e.domains[domain], compareAddr) } -func compareAddr(l, r netip.Addr) int { - return l.Compare(r) +func compareAddr(a, b netip.Addr) int { + return a.Compare(b) } // routesWithout returns a without b where a and b diff --git a/appc/appconnector_test.go b/appc/appconnector_test.go index c13835f39ed9a..c58aa80410869 100644 --- a/appc/appconnector_test.go +++ b/appc/appconnector_test.go @@ -1,10 +1,11 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package appc import ( - "context" + stdcmp "cmp" + "fmt" "net/netip" "reflect" "slices" @@ -12,28 +13,31 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "golang.org/x/net/dns/dnsmessage" "tailscale.com/appc/appctest" "tailscale.com/tstest" + "tailscale.com/types/appctype" "tailscale.com/util/clientmetric" + "tailscale.com/util/eventbus/eventbustest" "tailscale.com/util/mak" "tailscale.com/util/must" "tailscale.com/util/slicesx" ) -func fakeStoreRoutes(*RouteInfo) error { return nil } - func TestUpdateDomains(t *testing.T) { + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { - ctx := context.Background() - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, &appctest.RouteCollector{}, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, &appctest.RouteCollector{}, nil, nil) - } - a.UpdateDomains([]string{"example.com"}) + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) + a.UpdateDomains([]string{"example.com"}) a.Wait(ctx) if got, want := a.Domains().AsSlice(), []string{"example.com"}; !slices.Equal(got, want) { t.Errorf("got %v; want %v", got, want) @@ -58,15 +62,19 @@ func TestUpdateDomains(t *testing.T) { } func TestUpdateRoutes(t *testing.T) { + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { - ctx := context.Background() + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) + a.updateDomains([]string{"*.example.com"}) // This route should be collapsed into the range @@ -103,19 +111,37 @@ func TestUpdateRoutes(t *testing.T) { if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) { t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes()) } + + if err := eventbustest.Expect(w, + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.2.1/32")}), + eventbustest.Type[appctype.RouteInfo](), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.0.1/32")}), + eventbustest.Type[appctype.RouteInfo](), + eqUpdate(appctype.RouteUpdate{ + Advertise: prefixes("192.0.0.1/32", "192.0.2.0/24"), + Unadvertise: prefixes("192.0.2.1/32"), + }), + eventbustest.Type[appctype.RouteInfo](), + ); err != nil { + t.Error(err) + } } } func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) { - ctx := context.Background() + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) + mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")}) rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")} @@ -125,23 +151,36 @@ func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) { if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) { t.Fatalf("got %v, want %v", rc.Routes(), routes) } + + if err := eventbustest.ExpectExactly(w, + eqUpdate(appctype.RouteUpdate{ + Advertise: prefixes("192.0.2.0/24"), + Unadvertise: prefixes("192.0.2.1/32"), + }), + eventbustest.Type[appctype.RouteInfo](), + ); err != nil { + t.Error(err) + } } } func TestDomainRoutes(t *testing.T) { + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) a.updateDomains([]string{"example.com"}) if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil { t.Errorf("ObserveDNSResponse: %v", err) } - a.Wait(context.Background()) + a.Wait(t.Context()) want := map[string][]netip.Addr{ "example.com": {netip.MustParseAddr("192.0.0.8")}, @@ -150,19 +189,29 @@ func TestDomainRoutes(t *testing.T) { if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) { t.Fatalf("DomainRoutes: got %v, want %v", got, want) } + + if err := eventbustest.ExpectExactly(w, + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.0.8/32")}), + eventbustest.Type[appctype.RouteInfo](), + ); err != nil { + t.Error(err) + } } } func TestObserveDNSResponse(t *testing.T) { + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { - ctx := context.Background() + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) // a has no domains configured, so it should not advertise any routes if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil { @@ -239,19 +288,38 @@ func TestObserveDNSResponse(t *testing.T) { if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) { t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"]) } + + if err := eventbustest.ExpectExactly(w, + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.0.8/32")}), // from initial DNS response, via example.com + eventbustest.Type[appctype.RouteInfo](), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.0.9/32")}), // from CNAME response + eventbustest.Type[appctype.RouteInfo](), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.0.10/32")}), // from CNAME response, mid-chain + eventbustest.Type[appctype.RouteInfo](), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("2001:db8::1/128")}), // v6 DNS response + eventbustest.Type[appctype.RouteInfo](), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.2.0/24")}), // additional prefix + eventbustest.Type[appctype.RouteInfo](), + // N.B. no update for 192.0.2.1 as it is already covered + ); err != nil { + t.Error(err) + } } } func TestWildcardDomains(t *testing.T) { + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { - ctx := context.Background() + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) a.updateDomains([]string{"*.example.com"}) if err := a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8")); err != nil { @@ -278,6 +346,13 @@ func TestWildcardDomains(t *testing.T) { if len(a.wildcards) != 1 { t.Errorf("expected only one wildcard domain, got %v", a.wildcards) } + + if err := eventbustest.ExpectExactly(w, + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("192.0.0.8/32")}), + eventbustest.Type[appctype.RouteInfo](), + ); err != nil { + t.Error(err) + } } } @@ -393,8 +468,10 @@ func prefixes(in ...string) []netip.Prefix { } func TestUpdateRouteRouteRemoval(t *testing.T) { + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { - ctx := context.Background() + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) { @@ -406,12 +483,14 @@ func TestUpdateRouteRouteRemoval(t *testing.T) { } } - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) + // nothing has yet been advertised assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{}) @@ -434,12 +513,21 @@ func TestUpdateRouteRouteRemoval(t *testing.T) { wantRemovedRoutes = prefixes("1.2.3.2/32") } assertRoutes("removal", wantRoutes, wantRemovedRoutes) + + if err := eventbustest.Expect(w, + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.1/32", "1.2.3.2/32")}), // no duplicates here + eventbustest.Type[appctype.RouteInfo](), + ); err != nil { + t.Error(err) + } } } func TestUpdateDomainRouteRemoval(t *testing.T) { + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { - ctx := context.Background() + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) { @@ -451,12 +539,14 @@ func TestUpdateDomainRouteRemoval(t *testing.T) { } } - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) + assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{}) a.UpdateDomainsAndRoutes([]string{"a.example.com", "b.example.com"}, []netip.Prefix{}) @@ -489,12 +579,30 @@ func TestUpdateDomainRouteRemoval(t *testing.T) { wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32") } assertRoutes("removal", wantRoutes, wantRemovedRoutes) + + wantEvents := []any{ + // Each DNS record observed triggers an update. + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.1/32")}), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.2/32")}), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.3/32")}), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.4/32")}), + } + if shouldStore { + wantEvents = append(wantEvents, eqUpdate(appctype.RouteUpdate{ + Unadvertise: prefixes("1.2.3.3/32", "1.2.3.4/32"), + })) + } + if err := eventbustest.Expect(w, wantEvents...); err != nil { + t.Error(err) + } } } func TestUpdateWildcardRouteRemoval(t *testing.T) { + ctx := t.Context() + bus := eventbustest.NewBus(t) for _, shouldStore := range []bool{false, true} { - ctx := context.Background() + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} assertRoutes := func(prefix string, routes, removedRoutes []netip.Prefix) { @@ -506,12 +614,14 @@ func TestUpdateWildcardRouteRemoval(t *testing.T) { } } - var a *AppConnector - if shouldStore { - a = NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) - } else { - a = NewAppConnector(t.Logf, rc, nil, nil) - } + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: shouldStore, + }) + t.Cleanup(a.Close) + assertRoutes("appc init", []netip.Prefix{}, []netip.Prefix{}) a.UpdateDomainsAndRoutes([]string{"a.example.com", "*.b.example.com"}, []netip.Prefix{}) @@ -544,6 +654,22 @@ func TestUpdateWildcardRouteRemoval(t *testing.T) { wantRemovedRoutes = prefixes("1.2.3.3/32", "1.2.3.4/32") } assertRoutes("removal", wantRoutes, wantRemovedRoutes) + + wantEvents := []any{ + // Each DNS record observed triggers an update. + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.1/32")}), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.2/32")}), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.3/32")}), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("1.2.3.4/32")}), + } + if shouldStore { + wantEvents = append(wantEvents, eqUpdate(appctype.RouteUpdate{ + Unadvertise: prefixes("1.2.3.3/32", "1.2.3.4/32"), + })) + } + if err := eventbustest.Expect(w, wantEvents...); err != nil { + t.Error(err) + } } } @@ -572,7 +698,7 @@ func TestRateLogger(t *testing.T) { wasCalled = true }) - for i := 0; i < 3; i++ { + for range 3 { clock.Advance(1 * time.Millisecond) rl.update(0) if wasCalled { @@ -594,7 +720,7 @@ func TestRateLogger(t *testing.T) { wasCalled = true }) - for i := 0; i < 3; i++ { + for range 3 { clock.Advance(1 * time.Minute) rl.update(0) if wasCalled { @@ -610,6 +736,7 @@ func TestRateLogger(t *testing.T) { } func TestRouteStoreMetrics(t *testing.T) { + clientmetric.ResetForTest(t) metricStoreRoutes(1, 1) metricStoreRoutes(1, 1) // the 1 buckets value should be 2 metricStoreRoutes(5, 5) // the 5 buckets value should be 1 @@ -646,10 +773,22 @@ func TestMetricBucketsAreSorted(t *testing.T) { // routeAdvertiser, calls to Advertise/UnadvertiseRoutes can end up calling // back into AppConnector via authReconfig. If everything is called // synchronously, this results in a deadlock on AppConnector.mu. +// +// TODO(creachadair, 2025-09-18): Remove this along with the advertiser +// interface once the LocalBackend is switched to use the event bus and the +// tests have been updated not to need it. func TestUpdateRoutesDeadlock(t *testing.T) { - ctx := context.Background() + ctx := t.Context() + bus := eventbustest.NewBus(t) + w := eventbustest.NewWatcher(t, bus) rc := &appctest.RouteCollector{} - a := NewAppConnector(t.Logf, rc, &RouteInfo{}, fakeStoreRoutes) + a := NewAppConnector(Config{ + Logf: t.Logf, + EventBus: bus, + RouteAdvertiser: rc, + HasStoredRoutes: true, + }) + t.Cleanup(a.Close) advertiseCalled := new(atomic.Bool) unadvertiseCalled := new(atomic.Bool) @@ -693,4 +832,42 @@ func TestUpdateRoutesDeadlock(t *testing.T) { if want := []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}; !slices.Equal(slices.Compact(rc.Routes()), want) { t.Fatalf("got %v, want %v", rc.Routes(), want) } + + if err := eventbustest.ExpectExactly(w, + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("127.0.0.1/32", "127.0.0.2/32")}), + eventbustest.Type[appctype.RouteInfo](), + eqUpdate(appctype.RouteUpdate{Advertise: prefixes("127.0.0.1/32"), Unadvertise: prefixes("127.0.0.2/32")}), + eventbustest.Type[appctype.RouteInfo](), + ); err != nil { + t.Error(err) + } +} + +type textUpdate struct { + Advertise []string + Unadvertise []string +} + +func routeUpdateToText(u appctype.RouteUpdate) textUpdate { + var out textUpdate + for _, p := range u.Advertise { + out.Advertise = append(out.Advertise, p.String()) + } + for _, p := range u.Unadvertise { + out.Unadvertise = append(out.Unadvertise, p.String()) + } + return out +} + +// eqUpdate generates an eventbus test filter that matches a appctype.RouteUpdate +// message equal to want, or reports an error giving a human-readable diff. +func eqUpdate(want appctype.RouteUpdate) func(appctype.RouteUpdate) error { + return func(got appctype.RouteUpdate) error { + if diff := cmp.Diff(routeUpdateToText(got), routeUpdateToText(want), + cmpopts.SortSlices(stdcmp.Less[string]), + ); diff != "" { + return fmt.Errorf("wrong update (-got, +want):\n%s", diff) + } + return nil + } } diff --git a/appc/appctest/appctest.go b/appc/appctest/appctest.go index 9726a2b97d72b..c5eabf6761ec3 100644 --- a/appc/appctest/appctest.go +++ b/appc/appctest/appctest.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package appctest contains code to help test App Connectors. diff --git a/appc/conn25.go b/appc/conn25.go new file mode 100644 index 0000000000000..13a41e93ff79e --- /dev/null +++ b/appc/conn25.go @@ -0,0 +1,86 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package appc + +import ( + "cmp" + "fmt" + "slices" + "strings" + + "tailscale.com/ipn/ipnext" + "tailscale.com/tailcfg" + "tailscale.com/types/appctype" + "tailscale.com/types/dnstype" + "tailscale.com/util/set" +) + +const AppConnectorsExperimentalAttrName = "tailscale.com/app-connectors-experimental" + +func isPeerEligibleConnector(peer tailcfg.NodeView) bool { + if !peer.Valid() || !peer.Hostinfo().Valid() { + return false + } + isConn, _ := peer.Hostinfo().AppConnector().Get() + return isConn +} + +func sortByPreference(ns []tailcfg.NodeView) { + // The ordering of the nodes is semantic (callers use the first node they can + // get a peer api url for). We don't (currently 2026-02-27) have any + // preference over which node is chosen as long as it's consistent. In the + // future we anticipate integrating with traffic steering. + slices.SortFunc(ns, func(a, b tailcfg.NodeView) int { + return cmp.Compare(a.ID(), b.ID()) + }) +} + +// PickConnector returns peers the backend knows about that match the app, in order of preference to use as +// a connector. +func PickConnector(nb ipnext.NodeBackend, app appctype.Conn25Attr) []tailcfg.NodeView { + appTagsSet := set.SetOf(app.Connectors) + matches := nb.AppendMatchingPeers(nil, func(n tailcfg.NodeView) bool { + if !isPeerEligibleConnector(n) { + return false + } + for _, t := range n.Tags().All() { + if appTagsSet.Contains(t) { + return true + } + } + return false + }) + sortByPreference(matches) + return matches +} + +// DNSAddrScheme is the custom URI scheme used for conn25-managed split DNS +// entries to determine the destination at query time rather than configuration +// time. +const DNSAddrScheme = "tailscale-app" + +func AppDNSRoutes(hasCap func(c tailcfg.NodeCapability) bool, self tailcfg.NodeView) map[string][]*dnstype.Resolver { + if !hasCap(AppConnectorsExperimentalAttrName) { + return nil + } + apps, err := tailcfg.UnmarshalNodeCapViewJSON[appctype.AppConnectorAttr](self.CapMap(), AppConnectorsExperimentalAttrName) + if err != nil { + return nil + } + appNamesByDomain := map[string]string{} + for _, app := range apps { + for _, domain := range app.Domains { + domain, _ = strings.CutPrefix(domain, "*.") + domain = strings.ToLower(domain) + // in the case of multiple apps specifying the same domain (which is misconfiguration + // that should be validated at point of input) last write wins. + appNamesByDomain[domain] = app.Name + } + } + m := make(map[string][]*dnstype.Resolver, len(appNamesByDomain)) + for domain, appName := range appNamesByDomain { + m[domain] = []*dnstype.Resolver{{Addr: fmt.Sprintf("%s:%s", DNSAddrScheme, appName)}} + } + return m +} diff --git a/appc/conn25_test.go b/appc/conn25_test.go new file mode 100644 index 0000000000000..d9b4201a9d2bb --- /dev/null +++ b/appc/conn25_test.go @@ -0,0 +1,308 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package appc + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/ipn/ipnext" + "tailscale.com/tailcfg" + "tailscale.com/types/appctype" + "tailscale.com/types/dnstype" + "tailscale.com/types/opt" +) + +func TestAppDNSRoutes(t *testing.T) { + getBytesForAttr := func(name string, domains []string, tags []string) []byte { + attr := appctype.AppConnectorAttr{ + Name: name, + Domains: domains, + Connectors: tags, + } + bs, err := json.Marshal(attr) + if err != nil { + t.Fatalf("test setup: %v", err) + } + return bs + } + appOneBytes := getBytesForAttr("app1", []string{"example.com"}, []string{"tag:one"}) + appTwoBytes := getBytesForAttr("app2", []string{"a.example.com"}, []string{"tag:two"}) + appThreeBytes := getBytesForAttr("app3", []string{"woo.b.example.com", "hoo.b.example.com"}, []string{"tag:three1", "tag:three2"}) + appFourBytes := getBytesForAttr("app4", []string{"woo.b.example.com", "c.example.com"}, []string{"tag:four1", "tag:four2"}) + appFiveBytes := getBytesForAttr("app5", []string{"*.example.com", "example.com"}, []string{"tag:one"}) + appSixBytes := getBytesForAttr("app6", []string{"*.Example.com", "EXAMPLE.com", "EXAMPLE.COM"}, []string{"tag:one"}) + + resolver := func(appName string) []*dnstype.Resolver { + return []*dnstype.Resolver{{Addr: fmt.Sprintf("%s:%s", DNSAddrScheme, appName)}} + } + + for _, tt := range []struct { + name string + hasCap bool + config []tailcfg.RawMessage + want map[string][]*dnstype.Resolver + }{ + { + name: "no-capability", // hasCap false should return nil regardless of config. + hasCap: false, + }, + { + name: "no-apps", // hasCap true but no configured apps returns an empty map. + hasCap: true, + want: map[string][]*dnstype.Resolver{}, + }, + { + name: "bad-config", // bad config should return nil rather than error. + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(`hey`)}, + }, + { + name: "single-app", + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(appOneBytes)}, + want: map[string][]*dnstype.Resolver{ + "example.com": resolver("app1"), + }, + }, + { + name: "single-app-multi-domain", + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(appThreeBytes)}, + want: map[string][]*dnstype.Resolver{ + "woo.b.example.com": resolver("app3"), + "hoo.b.example.com": resolver("app3"), + }, + }, + { + name: "multi-app-no-overlap", + hasCap: true, + config: []tailcfg.RawMessage{ + tailcfg.RawMessage(appOneBytes), + tailcfg.RawMessage(appTwoBytes), + }, + want: map[string][]*dnstype.Resolver{ + "example.com": resolver("app1"), + "a.example.com": resolver("app2"), + }, + }, + { + name: "domain-collision-last-write-wins", + hasCap: true, + config: []tailcfg.RawMessage{ + tailcfg.RawMessage(appThreeBytes), // app3: woo.b.example.com, hoo.b.example.com + tailcfg.RawMessage(appFourBytes), // app4: woo.b.example.com, c.example.com + }, + want: map[string][]*dnstype.Resolver{ + // app4 overwrites app3 for the shared domain + "woo.b.example.com": resolver("app4"), + "hoo.b.example.com": resolver("app3"), + "c.example.com": resolver("app4"), + }, + }, + { + name: "wildcards-are-stripped-and-deduped", + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(appFiveBytes)}, + want: map[string][]*dnstype.Resolver{ + // *.example.com and example.com should both normalize to example.com. + "example.com": resolver("app5"), + }, + }, + { + name: "domains-are-normalized-and-deduped", + hasCap: true, + config: []tailcfg.RawMessage{tailcfg.RawMessage(appSixBytes)}, + want: map[string][]*dnstype.Resolver{ + // *.Example.com, EXAMPLE.com, EXAMPLE.COM should all normalize to example.com. + "example.com": resolver("app6"), + }, + }, + { + name: "sub-domains-and-top-domains-do-not-collide", + hasCap: true, + config: []tailcfg.RawMessage{ + tailcfg.RawMessage(appTwoBytes), + tailcfg.RawMessage(appFiveBytes), + }, + want: map[string][]*dnstype.Resolver{ + // *.example.com normalizes to example.com; a.example.com remains distinct. + "a.example.com": resolver("app2"), + "example.com": resolver("app5"), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + selfNode := &tailcfg.Node{} + if tt.config != nil { + selfNode.CapMap = tailcfg.NodeCapMap{ + tailcfg.NodeCapability(AppConnectorsExperimentalAttrName): tt.config, + } + } + selfView := selfNode.View() + got := AppDNSRoutes(func(_ tailcfg.NodeCapability) bool { + return tt.hasCap + }, selfView) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("AppDNSRoutes (-want, +got):\n%s", diff) + } + }) + } +} + +type testNodeBackend struct { + ipnext.NodeBackend + peers []tailcfg.NodeView +} + +func (nb *testNodeBackend) AppendMatchingPeers(base []tailcfg.NodeView, pred func(tailcfg.NodeView) bool) []tailcfg.NodeView { + for _, p := range nb.peers { + if pred(p) { + base = append(base, p) + } + } + return base +} + +func (nb *testNodeBackend) PeerHasPeerAPI(p tailcfg.NodeView) bool { + return true +} + +func TestPickConnector(t *testing.T) { + exampleApp := appctype.Conn25Attr{ + Name: "example", + Connectors: []string{"tag:example"}, + Domains: []string{"example.com"}, + } + + nvWithConnectorSet := func(id tailcfg.NodeID, isConnector bool, tags ...string) tailcfg.NodeView { + return (&tailcfg.Node{ + ID: id, + Tags: tags, + Hostinfo: (&tailcfg.Hostinfo{AppConnector: opt.NewBool(isConnector)}).View(), + }).View() + } + + nv := func(id tailcfg.NodeID, tags ...string) tailcfg.NodeView { + return nvWithConnectorSet(id, true, tags...) + } + + for _, tt := range []struct { + name string + candidates []tailcfg.NodeView + app appctype.Conn25Attr + want []tailcfg.NodeView + }{ + { + name: "empty-everything", + candidates: []tailcfg.NodeView{}, + app: appctype.Conn25Attr{}, + want: nil, + }, + { + name: "empty-candidates", + candidates: []tailcfg.NodeView{}, + app: exampleApp, + want: nil, + }, + { + name: "empty-app", + candidates: []tailcfg.NodeView{nv(1, "tag:example")}, + app: appctype.Conn25Attr{}, + want: nil, + }, + { + name: "one-matches", + candidates: []tailcfg.NodeView{nv(1, "tag:example")}, + app: exampleApp, + want: []tailcfg.NodeView{nv(1, "tag:example")}, + }, + { + name: "invalid-candidate", + candidates: []tailcfg.NodeView{ + {}, + nv(1, "tag:example"), + }, + app: exampleApp, + want: []tailcfg.NodeView{ + nv(1, "tag:example"), + }, + }, + { + name: "no-host-info", + candidates: []tailcfg.NodeView{ + (&tailcfg.Node{ + ID: 1, + Tags: []string{"tag:example"}, + }).View(), + nv(2, "tag:example"), + }, + app: exampleApp, + want: []tailcfg.NodeView{nv(2, "tag:example")}, + }, + { + name: "not-a-connector", + candidates: []tailcfg.NodeView{nvWithConnectorSet(1, false, "tag:example.com"), nv(2, "tag:example")}, + app: exampleApp, + want: []tailcfg.NodeView{nv(2, "tag:example")}, + }, + { + name: "without-matches", + candidates: []tailcfg.NodeView{nv(1, "tag:woo"), nv(2, "tag:example")}, + app: exampleApp, + want: []tailcfg.NodeView{nv(2, "tag:example")}, + }, + { + name: "multi-tags", + candidates: []tailcfg.NodeView{nv(1, "tag:woo", "tag:hoo"), nv(2, "tag:woo", "tag:example")}, + app: exampleApp, + want: []tailcfg.NodeView{nv(2, "tag:woo", "tag:example")}, + }, + { + name: "multi-matches", + candidates: []tailcfg.NodeView{nv(1, "tag:woo", "tag:hoo"), nv(2, "tag:woo", "tag:example"), nv(3, "tag:example1", "tag:example")}, + app: appctype.Conn25Attr{ + Name: "example2", + Connectors: []string{"tag:example1", "tag:example"}, + Domains: []string{"example.com"}, + }, + want: []tailcfg.NodeView{nv(2, "tag:woo", "tag:example"), nv(3, "tag:example1", "tag:example")}, + }, + { + name: "bit-of-everything", + candidates: []tailcfg.NodeView{ + nv(3, "tag:woo", "tag:hoo"), + {}, + nv(2, "tag:woo", "tag:example"), + nvWithConnectorSet(4, false, "tag:example"), + nv(1, "tag:example1", "tag:example"), + nv(7, "tag:example1", "tag:example"), + nvWithConnectorSet(5, false), + nv(6), + nvWithConnectorSet(8, false, "tag:example"), + nvWithConnectorSet(9, false), + nvWithConnectorSet(10, false), + }, + app: appctype.Conn25Attr{ + Name: "example2", + Connectors: []string{"tag:example1", "tag:example", "tag:example2"}, + Domains: []string{"example.com"}, + }, + want: []tailcfg.NodeView{ + nv(1, "tag:example1", "tag:example"), + nv(2, "tag:woo", "tag:example"), + nv(7, "tag:example1", "tag:example"), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got := PickConnector(&testNodeBackend{peers: tt.candidates}, tt.app) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("PickConnectors (-want, +got):\n%s", diff) + } + }) + } +} diff --git a/appc/observe.go b/appc/observe.go new file mode 100644 index 0000000000000..3cb2db662b564 --- /dev/null +++ b/appc/observe.go @@ -0,0 +1,132 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_appconnectors + +package appc + +import ( + "net/netip" + "strings" + + "golang.org/x/net/dns/dnsmessage" + "tailscale.com/util/mak" +) + +// ObserveDNSResponse is a callback invoked by the DNS resolver when a DNS +// response is being returned over the PeerAPI. The response is parsed and +// matched against the configured domains, if matched the routeAdvertiser is +// advised to advertise the discovered route. +func (e *AppConnector) ObserveDNSResponse(res []byte) error { + var p dnsmessage.Parser + if _, err := p.Start(res); err != nil { + return err + } + if err := p.SkipAllQuestions(); err != nil { + return err + } + + // cnameChain tracks a chain of CNAMEs for a given query in order to reverse + // a CNAME chain back to the original query for flattening. The keys are + // CNAME record targets, and the value is the name the record answers, so + // for www.example.com CNAME example.com, the map would contain + // ["example.com"] = "www.example.com". + var cnameChain map[string]string + + // addressRecords is a list of address records found in the response. + var addressRecords map[string][]netip.Addr + + for { + h, err := p.AnswerHeader() + if err == dnsmessage.ErrSectionDone { + break + } + if err != nil { + return err + } + + if h.Class != dnsmessage.ClassINET { + if err := p.SkipAnswer(); err != nil { + return err + } + continue + } + + switch h.Type { + case dnsmessage.TypeCNAME, dnsmessage.TypeA, dnsmessage.TypeAAAA: + default: + if err := p.SkipAnswer(); err != nil { + return err + } + continue + + } + + domain := strings.TrimSuffix(strings.ToLower(h.Name.String()), ".") + if len(domain) == 0 { + continue + } + + if h.Type == dnsmessage.TypeCNAME { + res, err := p.CNAMEResource() + if err != nil { + return err + } + cname := strings.TrimSuffix(strings.ToLower(res.CNAME.String()), ".") + if len(cname) == 0 { + continue + } + mak.Set(&cnameChain, cname, domain) + continue + } + + switch h.Type { + case dnsmessage.TypeA: + r, err := p.AResource() + if err != nil { + return err + } + addr := netip.AddrFrom4(r.A) + mak.Set(&addressRecords, domain, append(addressRecords[domain], addr)) + case dnsmessage.TypeAAAA: + r, err := p.AAAAResource() + if err != nil { + return err + } + addr := netip.AddrFrom16(r.AAAA) + mak.Set(&addressRecords, domain, append(addressRecords[domain], addr)) + default: + if err := p.SkipAnswer(); err != nil { + return err + } + continue + } + } + + e.mu.Lock() + defer e.mu.Unlock() + + for domain, addrs := range addressRecords { + domain, isRouted := e.findRoutedDomainLocked(domain, cnameChain) + + // domain and none of the CNAMEs in the chain are routed + if !isRouted { + continue + } + + // advertise each address we have learned for the routed domain, that + // was not already known. + var toAdvertise []netip.Prefix + for _, addr := range addrs { + if !e.isAddrKnownLocked(domain, addr) { + toAdvertise = append(toAdvertise, netip.PrefixFrom(addr, addr.BitLen())) + } + } + + if len(toAdvertise) > 0 { + e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise) + e.scheduleAdvertisement(domain, toAdvertise...) + } + } + return nil +} diff --git a/appc/observe_disabled.go b/appc/observe_disabled.go new file mode 100644 index 0000000000000..743c28a590f8e --- /dev/null +++ b/appc/observe_disabled.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_omit_appconnectors + +package appc + +func (e *AppConnector) ObserveDNSResponse(res []byte) error { return nil } diff --git a/assert_ts_toolchain_match.go b/assert_ts_toolchain_match.go index 40b24b334674f..4df0eeb1570d3 100644 --- a/assert_ts_toolchain_match.go +++ b/assert_ts_toolchain_match.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build tailscale_go @@ -17,7 +17,10 @@ func init() { panic("binary built with tailscale_go build tag but failed to read build info or find tailscale.toolchain.rev in build info") } want := strings.TrimSpace(GoToolchainRev) - if tsRev != want { + // Also permit the "next" toolchain rev, which is used in the main branch and will eventually become the new "current" rev. + // This allows building with TS_GO_NEXT=1 and then running the resulting binary without TS_GO_NEXT=1. + wantAlt := strings.TrimSpace(GoToolchainNextRev) + if tsRev != want && tsRev != wantAlt { if os.Getenv("TS_PERMIT_TOOLCHAIN_MISMATCH") == "1" { fmt.Fprintf(os.Stderr, "tailscale.toolchain.rev = %q, want %q; but ignoring due to TS_PERMIT_TOOLCHAIN_MISMATCH=1\n", tsRev, want) return diff --git a/atomicfile/atomicfile.go b/atomicfile/atomicfile.go index b3c8c93da2af9..1fa4c0641f74e 100644 --- a/atomicfile/atomicfile.go +++ b/atomicfile/atomicfile.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package atomicfile contains code related to writing to filesystems @@ -48,5 +48,9 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { if err := f.Close(); err != nil { return err } - return rename(tmpName, filename) + return Rename(tmpName, filename) } + +// Rename srcFile to dstFile, similar to [os.Rename] but preserving file +// attributes and ACLs on Windows. +func Rename(srcFile, dstFile string) error { return rename(srcFile, dstFile) } diff --git a/atomicfile/atomicfile_notwindows.go b/atomicfile/atomicfile_notwindows.go index 1ce2bb8acda7a..7104ddd5d9ff6 100644 --- a/atomicfile/atomicfile_notwindows.go +++ b/atomicfile/atomicfile_notwindows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !windows diff --git a/atomicfile/atomicfile_test.go b/atomicfile/atomicfile_test.go index 78c93e664f738..6dbf4eb430372 100644 --- a/atomicfile/atomicfile_test.go +++ b/atomicfile/atomicfile_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !js && !windows @@ -31,11 +31,11 @@ func TestDoesNotOverwriteIrregularFiles(t *testing.T) { // The least troublesome thing to make that is not a file is a unix socket. // Making a null device sadly requires root. - l, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) + ln, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) if err != nil { t.Fatal(err) } - defer l.Close() + defer ln.Close() err = WriteFile(path, []byte("hello"), 0644) if err == nil { diff --git a/atomicfile/atomicfile_windows.go b/atomicfile/atomicfile_windows.go index c67762df2b56c..d9c6ecf32ac5e 100644 --- a/atomicfile/atomicfile_windows.go +++ b/atomicfile/atomicfile_windows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package atomicfile diff --git a/atomicfile/atomicfile_windows_test.go b/atomicfile/atomicfile_windows_test.go index 4dec1493e0224..8748fc324f61a 100644 --- a/atomicfile/atomicfile_windows_test.go +++ b/atomicfile/atomicfile_windows_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package atomicfile diff --git a/atomicfile/mksyscall.go b/atomicfile/mksyscall.go index d8951a77c5ac6..2b0e4f9e58939 100644 --- a/atomicfile/mksyscall.go +++ b/atomicfile/mksyscall.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package atomicfile diff --git a/atomicfile/zsyscall_windows.go b/atomicfile/zsyscall_windows.go index f2f0b6d08cbb7..bd1bf8113ca2a 100644 --- a/atomicfile/zsyscall_windows.go +++ b/atomicfile/zsyscall_windows.go @@ -44,7 +44,7 @@ var ( ) func replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) { - r1, _, e1 := syscall.Syscall6(procReplaceFileW.Addr(), 6, uintptr(unsafe.Pointer(replaced)), uintptr(unsafe.Pointer(replacement)), uintptr(unsafe.Pointer(backup)), uintptr(flags), uintptr(exclude), uintptr(reserved)) + r1, _, e1 := syscall.SyscallN(procReplaceFileW.Addr(), uintptr(unsafe.Pointer(replaced)), uintptr(unsafe.Pointer(replacement)), uintptr(unsafe.Pointer(backup)), uintptr(flags), uintptr(exclude), uintptr(reserved)) if int32(r1) == 0 { err = errnoErr(e1) } diff --git a/build_dist.sh b/build_dist.sh index fed37c2646175..a32fe419e5739 100755 --- a/build_dist.sh +++ b/build_dist.sh @@ -18,7 +18,7 @@ fi eval `CGO_ENABLED=0 GOOS=$($go env GOHOSTOS) GOARCH=$($go env GOHOSTARCH) $go run ./cmd/mkversion` -if [ "$1" = "shellvars" ]; then +if [ "$#" -ge 1 ] && [ "$1" = "shellvars" ]; then cat <//tailscale TAGS=v0.0.1 make publishdevimage set -eu @@ -16,7 +26,7 @@ eval "$(./build_dist.sh shellvars)" DEFAULT_TARGET="client" DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}" -DEFAULT_BASE="tailscale/alpine-base:3.19" +DEFAULT_BASE="tailscale/alpine-base:3.22" # Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked # Github repo to find release notes for any new image tags. Note that for official Tailscale images the default # annotations defined here will be overriden by release scripts that call this script. @@ -28,6 +38,8 @@ TARGET="${TARGET:-${DEFAULT_TARGET}}" TAGS="${TAGS:-${DEFAULT_TAGS}}" BASE="${BASE:-${DEFAULT_BASE}}" PLATFORM="${PLATFORM:-}" # default to all platforms +GOARCH="${GOARCH:-arm,arm64,amd64,386,riscv64}" +FILES="${FILES:-}" # default to no extra files # OCI annotations that will be added to the image. # https://github.com/opencontainers/image-spec/blob/main/annotations.md ANNOTATIONS="${ANNOTATIONS:-${DEFAULT_ANNOTATIONS}}" @@ -51,7 +63,9 @@ case "$TARGET" in --repos="${REPOS}" \ --push="${PUSH}" \ --target="${PLATFORM}" \ + --goarch="${GOARCH}" \ --annotations="${ANNOTATIONS}" \ + --files="${FILES}" \ /usr/local/bin/containerboot ;; k8s-operator) @@ -69,7 +83,9 @@ case "$TARGET" in --repos="${REPOS}" \ --push="${PUSH}" \ --target="${PLATFORM}" \ + --goarch="${GOARCH}" \ --annotations="${ANNOTATIONS}" \ + --files="${FILES}" \ /usr/local/bin/operator ;; k8s-nameserver) @@ -87,7 +103,9 @@ case "$TARGET" in --repos="${REPOS}" \ --push="${PUSH}" \ --target="${PLATFORM}" \ + --goarch="${GOARCH}" \ --annotations="${ANNOTATIONS}" \ + --files="${FILES}" \ /usr/local/bin/k8s-nameserver ;; tsidp) @@ -105,9 +123,31 @@ case "$TARGET" in --repos="${REPOS}" \ --push="${PUSH}" \ --target="${PLATFORM}" \ + --goarch="${GOARCH}" \ --annotations="${ANNOTATIONS}" \ + --files="${FILES}" \ /usr/local/bin/tsidp ;; + k8s-proxy) + DEFAULT_REPOS="tailscale/k8s-proxy" + REPOS="${REPOS:-${DEFAULT_REPOS}}" + go run github.com/tailscale/mkctr \ + --gopaths="tailscale.com/cmd/k8s-proxy:/usr/local/bin/k8s-proxy" \ + --ldflags=" \ + -X tailscale.com/version.longStamp=${VERSION_LONG} \ + -X tailscale.com/version.shortStamp=${VERSION_SHORT} \ + -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \ + --base="${BASE}" \ + --tags="${TAGS}" \ + --gotags="ts_kube,ts_package_container" \ + --repos="${REPOS}" \ + --push="${PUSH}" \ + --target="${PLATFORM}" \ + --goarch="${GOARCH}" \ + --annotations="${ANNOTATIONS}" \ + --files="${FILES}" \ + /usr/local/bin/k8s-proxy + ;; *) echo "unknown target: $TARGET" exit 1 diff --git a/cache_key_test.go b/cache_key_test.go new file mode 100644 index 0000000000000..43de02e1328f3 --- /dev/null +++ b/cache_key_test.go @@ -0,0 +1,57 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package tailscaleroot + +import ( + "os" + "os/exec" + "strings" + "testing" + + "tailscale.com/util/cibuild" +) + +// TestTsgoRevInCacheKey verifies that the Tailscale Go toolchain's git +// revision (from go.toolchain.rev) is blended into Go build cache keys. +// Without this, bumping the toolchain to a new commit that doesn't change +// the Go version number would silently reuse stale cached build artifacts. +// +// See https://github.com/tailscale/tailscale/issues/36589. +func TestTsgoRevInCacheKey(t *testing.T) { + goRoot := goEnv(t, "GOROOT") + isTsgo := strings.Contains(goRoot, "/.cache/tsgo/") + if !cibuild.OnTailscaleCI() && !isTsgo { + t.Skip("skipping; not in Tailscale CI and not using the Tailscale Go toolchain") + } + + rev := strings.TrimSpace(GoToolchainRev) + if rev == "" { + t.Fatal("go.toolchain.rev is empty") + } + + // Build the small stdlib "errors" package with GODEBUG=gocachehash=1, + // which causes cmd/go to log its cache key computations to stderr. + cmd := exec.Command("go", "build", "errors") + cmd.Env = append(os.Environ(), "GODEBUG=gocachehash=1") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go build errors failed: %v\n%s", err, out) + } + + // The cache key output should contain the toolchain rev alongside the + // Go version, e.g.: + // HASH[moduleIndex]: "go1.26.2 dfe2a5fd8ee2e68b08ce5ff259269f50ecadf2f4" + if !strings.Contains(string(out), rev) { + t.Errorf("go.toolchain.rev %q not found in GODEBUG=gocachehash=1 output:\n%s", rev, out) + } +} + +func goEnv(t *testing.T, key string) string { + t.Helper() + out, err := exec.Command("go", "env", key).Output() + if err != nil { + t.Fatalf("go env %s: %v", key, err) + } + return strings.TrimSpace(string(out)) +} diff --git a/chirp/chirp.go b/chirp/chirp.go index 9653877221778..ed87542bc9a93 100644 --- a/chirp/chirp.go +++ b/chirp/chirp.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package chirp implements a client to communicate with the BIRD Internet diff --git a/chirp/chirp_test.go b/chirp/chirp_test.go index a57ef224b2c1b..eedc17f48afa9 100644 --- a/chirp/chirp_test.go +++ b/chirp/chirp_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package chirp @@ -24,7 +24,7 @@ type fakeBIRD struct { func newFakeBIRD(t *testing.T, protocols ...string) *fakeBIRD { sock := filepath.Join(t.TempDir(), "sock") - l, err := net.Listen("unix", sock) + ln, err := net.Listen("unix", sock) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func newFakeBIRD(t *testing.T, protocols ...string) *fakeBIRD { pe[p] = false } return &fakeBIRD{ - Listener: l, + Listener: ln, protocolsEnabled: pe, sock: sock, } @@ -123,12 +123,12 @@ type hangingListener struct { func newHangingListener(t *testing.T) *hangingListener { sock := filepath.Join(t.TempDir(), "sock") - l, err := net.Listen("unix", sock) + ln, err := net.Listen("unix", sock) if err != nil { t.Fatal(err) } return &hangingListener{ - Listener: l, + Listener: ln, t: t, done: make(chan struct{}), sock: sock, diff --git a/client/freedesktop/freedesktop.go b/client/freedesktop/freedesktop.go new file mode 100644 index 0000000000000..6ed1e8ccf88fc --- /dev/null +++ b/client/freedesktop/freedesktop.go @@ -0,0 +1,43 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package freedesktop provides helpers for freedesktop systems. +package freedesktop + +import "strings" + +const needsEscape = " \t\n\"'\\><~|&;$*?#()`" + +var escaper = strings.NewReplacer(`"`, `\"`, "`", "\\`", `$`, `\$`, `\`, `\\`) + +// Quote quotes according to the Desktop Entry Specification, as below: +// +// Arguments may be quoted in whole. If an argument contains a reserved +// character the argument must be quoted. The rules for quoting of arguments is +// also applicable to the executable name or path of the executable program as +// provided. +// +// Quoting must be done by enclosing the argument between double quotes and +// escaping the double quote character, backtick character ("`"), dollar sign +// ("$") and backslash character ("\") by preceding it with an additional +// backslash character. Implementations must undo quoting before expanding field +// codes and before passing the argument to the executable program. Reserved +// characters are space (" "), tab, newline, double quote, single quote ("'"), +// backslash character ("\"), greater-than sign (">"), less-than sign ("<"), +// tilde ("~"), vertical bar ("|"), ampersand ("&"), semicolon (";"), dollar +// sign ("$"), asterisk ("*"), question mark ("?"), hash mark ("#"), parenthesis +// ("(") and (")") and backtick character ("`"). +func Quote(s string) string { + if s == "" { + return `""` + } + if !strings.ContainsAny(s, needsEscape) { + return s + } + + var b strings.Builder + b.WriteString(`"`) + escaper.WriteString(&b, s) + b.WriteString(`"`) + return b.String() +} diff --git a/client/freedesktop/freedesktop_test.go b/client/freedesktop/freedesktop_test.go new file mode 100644 index 0000000000000..d02d1f67c286a --- /dev/null +++ b/client/freedesktop/freedesktop_test.go @@ -0,0 +1,145 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package freedesktop + +import ( + "strings" + "testing" +) + +func TestEscape(t *testing.T) { + tests := []struct { + name, input, want string + }{ + { + name: "no-illegal-chars", + input: "/home/user", + want: "/home/user", + }, + { + name: "empty-string", + input: "", + want: "\"\"", + }, + { + name: "space", + input: " ", + want: "\" \"", + }, + { + name: "tab", + input: "\t", + want: "\"\t\"", + }, + { + name: "newline", + input: "\n", + want: "\"\n\"", + }, + { + name: "double-quote", + input: "\"", + want: "\"\\\"\"", + }, + { + name: "single-quote", + input: "'", + want: "\"'\"", + }, + { + name: "backslash", + input: "\\", + want: "\"\\\\\"", + }, + { + name: "greater-than", + input: ">", + want: "\">\"", + }, + { + name: "less-than", + input: "<", + want: "\"<\"", + }, + { + name: "tilde", + input: "~", + want: "\"~\"", + }, + { + name: "pipe", + input: "|", + want: "\"|\"", + }, + { + name: "ampersand", + input: "&", + want: "\"&\"", + }, + { + name: "semicolon", + input: ";", + want: "\";\"", + }, + { + name: "dollar", + input: "$", + want: "\"\\$\"", + }, + { + name: "asterisk", + input: "*", + want: "\"*\"", + }, + { + name: "question-mark", + input: "?", + want: "\"?\"", + }, + { + name: "hash", + input: "#", + want: "\"#\"", + }, + { + name: "open-paren", + input: "(", + want: "\"(\"", + }, + { + name: "close-paren", + input: ")", + want: "\")\"", + }, + { + name: "backtick", + input: "`", + want: "\"\\`\"", + }, + { + name: "char-without-escape", + input: "/home/user\t", + want: "\"/home/user\t\"", + }, + { + name: "char-with-escape", + input: "/home/user\\", + want: "\"/home/user\\\\\"", + }, + { + name: "all-illegal-chars", + input: "/home/user" + needsEscape, + want: "\"/home/user \t\n\\\"'\\\\><~|&;\\$*?#()\\`\"", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := Quote(tt.input) + if strings.Compare(got, tt.want) != 0 { + t.Errorf("expected %s, got %s", tt.want, got) + } + }) + } +} diff --git a/client/local/cert.go b/client/local/cert.go new file mode 100644 index 0000000000000..701bfe026ceed --- /dev/null +++ b/client/local/cert.go @@ -0,0 +1,151 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !js && !ts_omit_acme + +package local + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "go4.org/mem" +) + +// SetDNS adds a DNS TXT record for the given domain name, containing +// the provided TXT value. The intended use case is answering +// LetsEncrypt/ACME dns-01 challenges. +// +// The control plane will only permit SetDNS requests with very +// specific names and values. The name should be +// "_acme-challenge." + your node's MagicDNS name. It's expected that +// clients cache the certs from LetsEncrypt (or whichever CA is +// providing them) and only request new ones as needed; the control plane +// rate limits SetDNS requests. +// +// This is a low-level interface; it's expected that most Tailscale +// users use a higher level interface to getting/using TLS +// certificates. +func (lc *Client) SetDNS(ctx context.Context, name, value string) error { + v := url.Values{} + v.Set("name", name) + v.Set("value", value) + _, err := lc.send(ctx, "POST", "/localapi/v0/set-dns?"+v.Encode(), 200, nil) + return err +} + +// CertPair returns a cert and private key for the provided DNS domain. +// +// It returns a cached certificate from disk if it's still valid. +// +// Deprecated: use [Client.CertPair]. +func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { + return defaultClient.CertPair(ctx, domain) +} + +// CertPair returns a cert and private key for the provided DNS domain. +// +// It returns a cached certificate from disk if it's still valid. +// +// API maturity: this is considered a stable API. +func (lc *Client) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { + return lc.CertPairWithValidity(ctx, domain, 0) +} + +// CertPairWithValidity returns a cert and private key for the provided DNS +// domain. +// +// It returns a cached certificate from disk if it's still valid. +// When minValidity is non-zero, the returned certificate will be valid for at +// least the given duration, if permitted by the CA. If the certificate is +// valid, but for less than minValidity, it will be synchronously renewed. +// +// API maturity: this is considered a stable API. +func (lc *Client) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) { + res, err := lc.send(ctx, "GET", fmt.Sprintf("/localapi/v0/cert/%s?type=pair&min_validity=%s", domain, minValidity), 200, nil) + if err != nil { + return nil, nil, err + } + // with ?type=pair, the response PEM is first the one private + // key PEM block, then the cert PEM blocks. + i := mem.Index(mem.B(res), mem.S("--\n--")) + if i == -1 { + return nil, nil, fmt.Errorf("unexpected output: no delimiter") + } + i += len("--\n") + keyPEM, certPEM = res[:i], res[i:] + if mem.Contains(mem.B(certPEM), mem.S(" PRIVATE KEY-----")) { + return nil, nil, fmt.Errorf("unexpected output: key in cert") + } + return certPEM, keyPEM, nil +} + +// GetCertificate fetches a TLS certificate for the TLS ClientHello in hi. +// +// It returns a cached certificate from disk if it's still valid. +// +// It's the right signature to use as the value of +// [tls.Config.GetCertificate]. +// +// Deprecated: use [Client.GetCertificate]. +func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { + return defaultClient.GetCertificate(hi) +} + +// GetCertificate fetches a TLS certificate for the TLS ClientHello in hi. +// +// It returns a cached certificate from disk if it's still valid. +// +// It's the right signature to use as the value of +// [tls.Config.GetCertificate]. +// +// API maturity: this is considered a stable API. +func (lc *Client) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { + if hi == nil || hi.ServerName == "" { + return nil, errors.New("no SNI ServerName") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + name := hi.ServerName + if !strings.Contains(name, ".") { + if v, ok := lc.ExpandSNIName(ctx, name); ok { + name = v + } + } + certPEM, keyPEM, err := lc.CertPair(ctx, name) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, err + } + return &cert, nil +} + +// ExpandSNIName expands bare label name into the most likely actual TLS cert name. +// +// Deprecated: use [Client.ExpandSNIName]. +func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) { + return defaultClient.ExpandSNIName(ctx, name) +} + +// ExpandSNIName expands bare label name into the most likely actual TLS cert name. +func (lc *Client) ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) { + st, err := lc.StatusWithoutPeers(ctx) + if err != nil { + return "", false + } + for _, d := range st.CertDomains { + if len(d) > len(name)+1 && strings.HasPrefix(d, name) && d[len(name)] == '.' { + return d, true + } + } + return "", false +} diff --git a/client/local/debugportmapper.go b/client/local/debugportmapper.go new file mode 100644 index 0000000000000..1cbb3ee0a303e --- /dev/null +++ b/client/local/debugportmapper.go @@ -0,0 +1,84 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_debugportmapper + +package local + +import ( + "cmp" + "context" + "fmt" + "io" + "net/http" + "net/netip" + "net/url" + "strconv" + "time" + + "tailscale.com/client/tailscale/apitype" +) + +// DebugPortmapOpts contains options for the [Client.DebugPortmap] command. +type DebugPortmapOpts struct { + // Duration is how long the mapping should be created for. It defaults + // to 5 seconds if not set. + Duration time.Duration + + // Type is the kind of portmap to debug. The empty string instructs the + // portmap client to perform all known types. Other valid options are + // "pmp", "pcp", and "upnp". + Type string + + // GatewayAddr specifies the gateway address used during portmapping. + // If set, SelfAddr must also be set. If unset, it will be + // autodetected. + GatewayAddr netip.Addr + + // SelfAddr specifies the gateway address used during portmapping. If + // set, GatewayAddr must also be set. If unset, it will be + // autodetected. + SelfAddr netip.Addr + + // LogHTTP instructs the debug-portmap endpoint to print all HTTP + // requests and responses made to the logs. + LogHTTP bool +} + +// DebugPortmap invokes the debug-portmap endpoint, and returns an +// io.ReadCloser that can be used to read the logs that are printed during this +// process. +// +// opts can be nil; if so, default values will be used. +func (lc *Client) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) { + vals := make(url.Values) + if opts == nil { + opts = &DebugPortmapOpts{} + } + + vals.Set("duration", cmp.Or(opts.Duration, 5*time.Second).String()) + vals.Set("type", opts.Type) + vals.Set("log_http", strconv.FormatBool(opts.LogHTTP)) + + if opts.GatewayAddr.IsValid() != opts.SelfAddr.IsValid() { + return nil, fmt.Errorf("both GatewayAddr and SelfAddr must be provided if one is") + } else if opts.GatewayAddr.IsValid() { + vals.Set("gateway_and_self", fmt.Sprintf("%s/%s", opts.GatewayAddr, opts.SelfAddr)) + } + + req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-portmap?"+vals.Encode(), nil) + if err != nil { + return nil, err + } + res, err := lc.doLocalRequestNiceError(req) + if err != nil { + return nil, err + } + if res.StatusCode != 200 { + body, _ := io.ReadAll(res.Body) + res.Body.Close() + return nil, fmt.Errorf("HTTP %s: %s", res.Status, body) + } + + return res.Body, nil +} diff --git a/client/local/local.go b/client/local/local.go index 12bf2f7d6fef3..50050eb1b5be1 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package local contains a Go client for the Tailscale LocalAPI. @@ -9,7 +9,6 @@ import ( "bytes" "cmp" "context" - "crypto/tls" "encoding/base64" "encoding/json" "errors" @@ -28,22 +27,24 @@ import ( "sync" "time" - "go4.org/mem" "tailscale.com/client/tailscale/apitype" "tailscale.com/drive" "tailscale.com/envknob" + "tailscale.com/feature" + "tailscale.com/feature/buildfeatures" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/net/netutil" + "tailscale.com/net/udprelay/status" "tailscale.com/paths" "tailscale.com/safesocket" + "tailscale.com/syncs" "tailscale.com/tailcfg" - "tailscale.com/tka" + "tailscale.com/types/appctype" "tailscale.com/types/dnstype" "tailscale.com/types/key" - "tailscale.com/types/tkatype" + "tailscale.com/util/clientmetric" "tailscale.com/util/eventbus" - "tailscale.com/util/syspolicy/setting" ) // defaultClient is the default Client when using the legacy @@ -191,8 +192,8 @@ func (e *AccessDeniedError) Unwrap() error { return e.err } // IsAccessDeniedError reports whether err is or wraps an AccessDeniedError. func IsAccessDeniedError(err error) bool { - var ae *AccessDeniedError - return errors.As(err, &ae) + _, ok := errors.AsType[*AccessDeniedError](err) + return ok } // PreconditionsFailedError is returned when the server responds @@ -209,8 +210,8 @@ func (e *PreconditionsFailedError) Unwrap() error { return e.err } // IsPreconditionsFailedError reports whether err is or wraps an PreconditionsFailedError. func IsPreconditionsFailedError(err error) bool { - var ae *PreconditionsFailedError - return errors.As(err, &ae) + _, ok := errors.AsType[*PreconditionsFailedError](err) + return ok } // bestError returns either err, or if body contains a valid JSON @@ -326,6 +327,35 @@ func (lc *Client) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsR return decodeJSON[*apitype.WhoIsResponse](body) } +// WhoIsForService is like [Client.WhoIs] but scopes the returned CapMap to +// capabilities that apply to the named VIP service. This enables per-service +// capability resolution on hosts that advertise multiple VIP services. +func (lc *Client) WhoIsForService(ctx context.Context, remoteAddr string, svcName tailcfg.ServiceName) (*apitype.WhoIsResponse, error) { + body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr)+"&svc_name="+url.QueryEscape(string(svcName))) + if err != nil { + if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound { + return nil, ErrPeerNotFound + } + return nil, err + } + return decodeJSON[*apitype.WhoIsResponse](body) +} + +// WhoIsForIP is like [Client.WhoIs] but scopes the returned CapMap to +// capabilities that apply to the given destination IP. The IP may be a +// VIP service address, the node's own tailnet address, or any other +// routable IP the node handles. +func (lc *Client) WhoIsForIP(ctx context.Context, remoteAddr string, dst netip.Addr) (*apitype.WhoIsResponse, error) { + body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr)+"&dst_ip="+url.QueryEscape(dst.String())) + if err != nil { + if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound { + return nil, ErrPeerNotFound + } + return nil, err + } + return decodeJSON[*apitype.WhoIsResponse](body) +} + // ErrPeerNotFound is returned by [Client.WhoIs], [Client.WhoIsNodeKey] and // [Client.WhoIsProto] when a peer is not found. var ErrPeerNotFound = errors.New("peer not found") @@ -382,18 +412,42 @@ func (lc *Client) UserMetrics(ctx context.Context) ([]byte, error) { // // IncrementCounter does not support gauge metrics or negative delta values. func (lc *Client) IncrementCounter(ctx context.Context, name string, delta int) error { - type metricUpdate struct { - Name string `json:"name"` - Type string `json:"type"` - Value int `json:"value"` // amount to increment by + if !buildfeatures.HasClientMetrics { + return nil } if delta < 0 { return errors.New("negative delta not allowed") } - _, err := lc.send(ctx, "POST", "/localapi/v0/upload-client-metrics", 200, jsonBody([]metricUpdate{{ + _, err := lc.send(ctx, "POST", "/localapi/v0/upload-client-metrics", 200, jsonBody([]clientmetric.MetricUpdate{{ Name: name, Type: "counter", Value: delta, + Op: "add", + }})) + return err +} + +// IncrementGauge increments the value of a Tailscale daemon's gauge +// metric by the given delta. If the metric has yet to exist, a new gauge +// metric is created and initialized to delta. The delta value can be negative. +func (lc *Client) IncrementGauge(ctx context.Context, name string, delta int) error { + _, err := lc.send(ctx, "POST", "/localapi/v0/upload-client-metrics", 200, jsonBody([]clientmetric.MetricUpdate{{ + Name: name, + Type: "gauge", + Value: delta, + Op: "add", + }})) + return err +} + +// SetGauge sets the value of a Tailscale daemon's gauge metric to the given value. +// If the metric has yet to exist, a new gauge metric is created and initialized to value. +func (lc *Client) SetGauge(ctx context.Context, name string, value int) error { + _, err := lc.send(ctx, "POST", "/localapi/v0/upload-client-metrics", 200, jsonBody([]clientmetric.MetricUpdate{{ + Name: name, + Type: "gauge", + Value: value, + Op: "set", }})) return err } @@ -415,6 +469,17 @@ func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { return res.Body, nil } +// EventBusGraph returns a graph of active publishers and subscribers in the eventbus +// as a [eventbus.DebugTopics] +func (lc *Client) EventBusGraph(ctx context.Context) ([]byte, error) { + return lc.get200(ctx, "/localapi/v0/debug-bus-graph") +} + +// EventBusQueues returns a JSON snapshot of event bus queue depths per client. +func (lc *Client) EventBusQueues(ctx context.Context) ([]byte, error) { + return lc.get200(ctx, "/localapi/v0/debug-bus-queues") +} + // StreamBusEvents returns an iterator of Tailscale bus events as they arrive. // Each pair is a valid event and a nil error, or a zero event a non-nil error. // In case of error, the iterator ends after the pair reporting the error. @@ -571,68 +636,35 @@ func (lc *Client) DebugResultJSON(ctx context.Context, action string) (any, erro return x, nil } -// DebugPortmapOpts contains options for the [Client.DebugPortmap] command. -type DebugPortmapOpts struct { - // Duration is how long the mapping should be created for. It defaults - // to 5 seconds if not set. - Duration time.Duration - - // Type is the kind of portmap to debug. The empty string instructs the - // portmap client to perform all known types. Other valid options are - // "pmp", "pcp", and "upnp". - Type string - - // GatewayAddr specifies the gateway address used during portmapping. - // If set, SelfAddr must also be set. If unset, it will be - // autodetected. - GatewayAddr netip.Addr - - // SelfAddr specifies the gateway address used during portmapping. If - // set, GatewayAddr must also be set. If unset, it will be - // autodetected. - SelfAddr netip.Addr - - // LogHTTP instructs the debug-portmap endpoint to print all HTTP - // requests and responses made to the logs. - LogHTTP bool -} - -// DebugPortmap invokes the debug-portmap endpoint, and returns an -// io.ReadCloser that can be used to read the logs that are printed during this -// process. +// GetDebugResultJSON invokes a debug action and decodes the JSON response +// into a value of type T. It avoids the marshal/unmarshal roundtrip that +// callers of [Client.DebugResultJSON] otherwise need to do to get a typed +// value. // -// opts can be nil; if so, default values will be used. -func (lc *Client) DebugPortmap(ctx context.Context, opts *DebugPortmapOpts) (io.ReadCloser, error) { - vals := make(url.Values) - if opts == nil { - opts = &DebugPortmapOpts{} +// These are development tools and subject to change or removal over time. +func GetDebugResultJSON[T any](ctx context.Context, lc *Client, action string) (T, error) { + var v T + body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, nil) + if err != nil { + return v, fmt.Errorf("error %w: %s", err, body) } - - vals.Set("duration", cmp.Or(opts.Duration, 5*time.Second).String()) - vals.Set("type", opts.Type) - vals.Set("log_http", strconv.FormatBool(opts.LogHTTP)) - - if opts.GatewayAddr.IsValid() != opts.SelfAddr.IsValid() { - return nil, fmt.Errorf("both GatewayAddr and SelfAddr must be provided if one is") - } else if opts.GatewayAddr.IsValid() { - vals.Set("gateway_and_self", fmt.Sprintf("%s/%s", opts.GatewayAddr, opts.SelfAddr)) + if err := json.Unmarshal(body, &v); err != nil { + return v, err } + return v, nil +} - req, err := http.NewRequestWithContext(ctx, "GET", "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-portmap?"+vals.Encode(), nil) +// QueryOptionalFeatures queries the optional features supported by the Tailscale daemon. +func (lc *Client) QueryOptionalFeatures(ctx context.Context) (*apitype.OptionalFeatures, error) { + body, err := lc.send(ctx, "POST", "/localapi/v0/debug-optional-features", 200, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("error %w: %s", err, body) } - res, err := lc.doLocalRequestNiceError(req) - if err != nil { + var x apitype.OptionalFeatures + if err := json.Unmarshal(body, &x); err != nil { return nil, err } - if res.StatusCode != 200 { - body, _ := io.ReadAll(res.Body) - res.Body.Close() - return nil, fmt.Errorf("HTTP %s: %s", res.Status, body) - } - - return res.Body, nil + return &x, nil } // SetDevStoreKeyValue set a statestore key/value. It's only meant for development. @@ -652,6 +684,9 @@ func (lc *Client) SetDevStoreKeyValue(ctx context.Context, key, value string) er // the provided duration. If the duration is in the past, the debug logging // is disabled. func (lc *Client) SetComponentDebugLogging(ctx context.Context, component string, d time.Duration) error { + if !buildfeatures.HasDebug { + return feature.ErrUnavailable + } body, err := lc.send(ctx, "POST", fmt.Sprintf("/localapi/v0/component-debug-logging?component=%s&secs=%d", url.QueryEscape(component), int64(d.Seconds())), 200, nil) @@ -791,6 +826,9 @@ func (lc *Client) PushFile(ctx context.Context, target tailcfg.StableNodeID, siz // machine is properly configured to forward IP packets as a subnet router // or exit node. func (lc *Client) CheckIPForwarding(ctx context.Context) error { + if !buildfeatures.HasAdvertiseRoutes { + return nil + } body, err := lc.get200(ctx, "/localapi/v0/check-ip-forwarding") if err != nil { return err @@ -827,25 +865,6 @@ func (lc *Client) CheckUDPGROForwarding(ctx context.Context) error { return nil } -// CheckReversePathFiltering asks the local Tailscale daemon whether strict -// reverse path filtering is enabled, which would break exit node usage on Linux. -func (lc *Client) CheckReversePathFiltering(ctx context.Context) error { - body, err := lc.get200(ctx, "/localapi/v0/check-reverse-path-filtering") - if err != nil { - return err - } - var jres struct { - Warning string - } - if err := json.Unmarshal(body, &jres); err != nil { - return fmt.Errorf("invalid JSON from check-reverse-path-filtering: %w", err) - } - if jres.Warning != "" { - return errors.New(jres.Warning) - } - return nil -} - // SetUDPGROForwarding enables UDP GRO forwarding for the main interface of this // node. This can be done to improve performance of tailnet nodes acting as exit // nodes or subnet routers. @@ -903,36 +922,12 @@ func (lc *Client) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Pref return decodeJSON[*ipn.Prefs](body) } -// GetEffectivePolicy returns the effective policy for the specified scope. -func (lc *Client) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) { - scopeID, err := scope.MarshalText() - if err != nil { - return nil, err - } - body, err := lc.get200(ctx, "/localapi/v0/policy/"+string(scopeID)) - if err != nil { - return nil, err - } - return decodeJSON[*setting.Snapshot](body) -} - -// ReloadEffectivePolicy reloads the effective policy for the specified scope -// by reading and merging policy settings from all applicable policy sources. -func (lc *Client) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) { - scopeID, err := scope.MarshalText() - if err != nil { - return nil, err - } - body, err := lc.send(ctx, "POST", "/localapi/v0/policy/"+string(scopeID), 200, http.NoBody) - if err != nil { - return nil, err - } - return decodeJSON[*setting.Snapshot](body) -} - // GetDNSOSConfig returns the system DNS configuration for the current device. // That is, it returns the DNS configuration that the system would use if Tailscale weren't being used. func (lc *Client) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, error) { + if !buildfeatures.HasDNS { + return nil, feature.ErrUnavailable + } body, err := lc.get200(ctx, "/localapi/v0/dns-osconfig") if err != nil { return nil, err @@ -948,6 +943,9 @@ func (lc *Client) GetDNSOSConfig(ctx context.Context) (*apitype.DNSOSConfig, err // It returns the raw DNS response bytes and the resolvers that were used to answer the query // (often just one, but can be more if we raced multiple resolvers). func (lc *Client) QueryDNS(ctx context.Context, name string, queryType string) (bytes []byte, resolvers []*dnstype.Resolver, err error) { + if !buildfeatures.HasDNS { + return nil, nil, feature.ErrUnavailable + } body, err := lc.get200(ctx, fmt.Sprintf("/localapi/v0/dns-query?name=%s&type=%s", url.QueryEscape(name), queryType)) if err != nil { return nil, nil, err @@ -978,28 +976,6 @@ func (lc *Client) Logout(ctx context.Context) error { return err } -// SetDNS adds a DNS TXT record for the given domain name, containing -// the provided TXT value. The intended use case is answering -// LetsEncrypt/ACME dns-01 challenges. -// -// The control plane will only permit SetDNS requests with very -// specific names and values. The name should be -// "_acme-challenge." + your node's MagicDNS name. It's expected that -// clients cache the certs from LetsEncrypt (or whichever CA is -// providing them) and only request new ones as needed; the control plane -// rate limits SetDNS requests. -// -// This is a low-level interface; it's expected that most Tailscale -// users use a higher level interface to getting/using TLS -// certificates. -func (lc *Client) SetDNS(ctx context.Context, name, value string) error { - v := url.Values{} - v.Set("name", name) - v.Set("value", value) - _, err := lc.send(ctx, "POST", "/localapi/v0/set-dns?"+v.Encode(), 200, nil) - return err -} - // DialTCP connects to the host's port via Tailscale. // // The host may be a base DNS name (resolved from the netmap inside @@ -1043,6 +1019,19 @@ func (lc *Client) UserDial(ctx context.Context, network, host string, port uint1 if res.StatusCode != http.StatusSwitchingProtocols { body, _ := io.ReadAll(res.Body) res.Body.Close() + if res.StatusCode == http.StatusOK && res.Header.Get("Dial-Self") == "true" { + // Server told us to dial the address ourselves rather than + // proxying through the daemon. This happens for non-Tailscale + // addresses where the daemon shouldn't dial as root on the + // client's behalf. The server provides the resolved address + // to avoid a TOCTOU race with DNS re-resolution. + addr := res.Header.Get("Dial-Addr") + if addr == "" { + return nil, errors.New("server returned Dial-Self without Dial-Addr") + } + var d net.Dialer + return d.DialContext(ctx, network, addr) + } return nil, fmt.Errorf("unexpected HTTP response: %s, %s", res.Status, body) } // From here on, the underlying net.Conn is ours to use, but there @@ -1080,115 +1069,60 @@ func (lc *Client) CurrentDERPMap(ctx context.Context) (*tailcfg.DERPMap, error) return &derpMap, nil } -// CertPair returns a cert and private key for the provided DNS domain. -// -// It returns a cached certificate from disk if it's still valid. -// -// Deprecated: use [Client.CertPair]. -func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { - return defaultClient.CertPair(ctx, domain) -} - -// CertPair returns a cert and private key for the provided DNS domain. -// -// It returns a cached certificate from disk if it's still valid. -// -// API maturity: this is considered a stable API. -func (lc *Client) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { - return lc.CertPairWithValidity(ctx, domain, 0) -} - -// CertPairWithValidity returns a cert and private key for the provided DNS -// domain. -// -// It returns a cached certificate from disk if it's still valid. -// When minValidity is non-zero, the returned certificate will be valid for at -// least the given duration, if permitted by the CA. If the certificate is -// valid, but for less than minValidity, it will be synchronously renewed. -// -// API maturity: this is considered a stable API. -func (lc *Client) CertPairWithValidity(ctx context.Context, domain string, minValidity time.Duration) (certPEM, keyPEM []byte, err error) { - res, err := lc.send(ctx, "GET", fmt.Sprintf("/localapi/v0/cert/%s?type=pair&min_validity=%s", domain, minValidity), 200, nil) +// CertDomains returns the list of domains for which the local tailscaled can +// fetch TLS certificates, equivalent to the DNS.CertDomains field of the +// current netmap. The returned list is sorted in ascending order, and is +// empty if no netmap has been received yet. +func (lc *Client) CertDomains(ctx context.Context) ([]string, error) { + body, err := lc.get200(ctx, "/localapi/v0/cert-domains") if err != nil { - return nil, nil, err - } - // with ?type=pair, the response PEM is first the one private - // key PEM block, then the cert PEM blocks. - i := mem.Index(mem.B(res), mem.S("--\n--")) - if i == -1 { - return nil, nil, fmt.Errorf("unexpected output: no delimiter") - } - i += len("--\n") - keyPEM, certPEM = res[:i], res[i:] - if mem.Contains(mem.B(certPEM), mem.S(" PRIVATE KEY-----")) { - return nil, nil, fmt.Errorf("unexpected output: key in cert") + return nil, err } - return certPEM, keyPEM, nil + return decodeJSON[[]string](body) } -// GetCertificate fetches a TLS certificate for the TLS ClientHello in hi. -// -// It returns a cached certificate from disk if it's still valid. -// -// It's the right signature to use as the value of -// [tls.Config.GetCertificate]. -// -// Deprecated: use [Client.GetCertificate]. -func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { - return defaultClient.GetCertificate(hi) -} - -// GetCertificate fetches a TLS certificate for the TLS ClientHello in hi. -// -// It returns a cached certificate from disk if it's still valid. -// -// It's the right signature to use as the value of -// [tls.Config.GetCertificate]. -// -// API maturity: this is considered a stable API. -func (lc *Client) GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { - if hi == nil || hi.ServerName == "" { - return nil, errors.New("no SNI ServerName") - } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - name := hi.ServerName - if !strings.Contains(name, ".") { - if v, ok := lc.ExpandSNIName(ctx, name); ok { - name = v - } - } - certPEM, keyPEM, err := lc.CertPair(ctx, name) +// DNSConfig returns the [tailcfg.DNSConfig] from the current netmap. +// It returns an error if no netmap has been received yet. +// It is intended for callers that need fields like ExtraRecords or CertDomains +// without pulling the rest of the netmap. +func (lc *Client) DNSConfig(ctx context.Context) (*tailcfg.DNSConfig, error) { + body, err := lc.get200(ctx, "/localapi/v0/dns-config") if err != nil { return nil, err } - cert, err := tls.X509KeyPair(certPEM, keyPEM) + return decodeJSON[*tailcfg.DNSConfig](body) +} + +// PeerByID returns a peer's current full [tailcfg.Node] looked up by its +// [tailcfg.NodeID]. It returns an error if no peer with that NodeID is in the +// current netmap. +// +// It is intended for callers that observed a peer-mutation signal (e.g. +// [ipn.Notify.PeerChangedPatch] or [ipn.Notify.PeersChanged]) and want the +// latest state of the affected node without having to apply the patch +// themselves. +func (lc *Client) PeerByID(ctx context.Context, id tailcfg.NodeID) (*tailcfg.Node, error) { + body, err := lc.get200(ctx, "/localapi/v0/peer-by-id?id="+strconv.FormatInt(int64(id), 10)) if err != nil { return nil, err } - return &cert, nil + return decodeJSON[*tailcfg.Node](body) } -// ExpandSNIName expands bare label name into the most likely actual TLS cert name. +// UserProfile returns the current [tailcfg.UserProfile] for the given +// [tailcfg.UserID]. It returns an error if no user with that UserID is in the +// current netmap. // -// Deprecated: use [Client.ExpandSNIName]. -func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) { - return defaultClient.ExpandSNIName(ctx, name) -} - -// ExpandSNIName expands bare label name into the most likely actual TLS cert name. -func (lc *Client) ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) { - st, err := lc.StatusWithoutPeers(ctx) +// It is the LocalAPI fallback for IPN-bus consumers that see a UserID +// referenced by a peer Node and want to resolve it to a UserProfile. Sessions +// opted in to [ipn.NotifyPeerChanges] / [ipn.NotifyPeerPatches] also receive +// UserProfiles automatically via [ipn.Notify.UserProfiles]. +func (lc *Client) UserProfile(ctx context.Context, id tailcfg.UserID) (*tailcfg.UserProfile, error) { + body, err := lc.get200(ctx, "/localapi/v0/user-profile?id="+strconv.FormatInt(int64(id), 10)) if err != nil { - return "", false - } - for _, d := range st.CertDomains { - if len(d) > len(name)+1 && strings.HasPrefix(d, name) && d[len(name)] == '.' { - return d, true - } + return nil, err } - return "", false + return decodeJSON[*tailcfg.UserProfile](body) } // PingOpts contains options for the ping request. @@ -1224,197 +1158,6 @@ func (lc *Client) Ping(ctx context.Context, ip netip.Addr, pingtype tailcfg.Ping return lc.PingWithOpts(ctx, ip, pingtype, PingOpts{}) } -// NetworkLockStatus fetches information about the tailnet key authority, if one is configured. -func (lc *Client) NetworkLockStatus(ctx context.Context) (*ipnstate.NetworkLockStatus, error) { - body, err := lc.send(ctx, "GET", "/localapi/v0/tka/status", 200, nil) - if err != nil { - return nil, fmt.Errorf("error: %w", err) - } - return decodeJSON[*ipnstate.NetworkLockStatus](body) -} - -// NetworkLockInit initializes the tailnet key authority. -// -// TODO(tom): Plumb through disablement secrets. -func (lc *Client) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.NetworkLockStatus, error) { - var b bytes.Buffer - type initRequest struct { - Keys []tka.Key - DisablementValues [][]byte - SupportDisablement []byte - } - - if err := json.NewEncoder(&b).Encode(initRequest{Keys: keys, DisablementValues: disablementValues, SupportDisablement: supportDisablement}); err != nil { - return nil, err - } - - body, err := lc.send(ctx, "POST", "/localapi/v0/tka/init", 200, &b) - if err != nil { - return nil, fmt.Errorf("error: %w", err) - } - return decodeJSON[*ipnstate.NetworkLockStatus](body) -} - -// NetworkLockWrapPreauthKey wraps a pre-auth key with information to -// enable unattended bringup in the locked tailnet. -func (lc *Client) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) { - encodedPrivate, err := tkaKey.MarshalText() - if err != nil { - return "", err - } - - var b bytes.Buffer - type wrapRequest struct { - TSKey string - TKAKey string // key.NLPrivate.MarshalText - } - if err := json.NewEncoder(&b).Encode(wrapRequest{TSKey: preauthKey, TKAKey: string(encodedPrivate)}); err != nil { - return "", err - } - - body, err := lc.send(ctx, "POST", "/localapi/v0/tka/wrap-preauth-key", 200, &b) - if err != nil { - return "", fmt.Errorf("error: %w", err) - } - return string(body), nil -} - -// NetworkLockModify adds and/or removes key(s) to the tailnet key authority. -func (lc *Client) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error { - var b bytes.Buffer - type modifyRequest struct { - AddKeys []tka.Key - RemoveKeys []tka.Key - } - - if err := json.NewEncoder(&b).Encode(modifyRequest{AddKeys: addKeys, RemoveKeys: removeKeys}); err != nil { - return err - } - - if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/modify", 204, &b); err != nil { - return fmt.Errorf("error: %w", err) - } - return nil -} - -// NetworkLockSign signs the specified node-key and transmits that signature to the control plane. -// rotationPublic, if specified, must be an ed25519 public key. -func (lc *Client) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error { - var b bytes.Buffer - type signRequest struct { - NodeKey key.NodePublic - RotationPublic []byte - } - - if err := json.NewEncoder(&b).Encode(signRequest{NodeKey: nodeKey, RotationPublic: rotationPublic}); err != nil { - return err - } - - if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/sign", 200, &b); err != nil { - return fmt.Errorf("error: %w", err) - } - return nil -} - -// NetworkLockAffectedSigs returns all signatures signed by the specified keyID. -func (lc *Client) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) { - body, err := lc.send(ctx, "POST", "/localapi/v0/tka/affected-sigs", 200, bytes.NewReader(keyID)) - if err != nil { - return nil, fmt.Errorf("error: %w", err) - } - return decodeJSON[[]tkatype.MarshaledSignature](body) -} - -// NetworkLockLog returns up to maxEntries number of changes to network-lock state. -func (lc *Client) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.NetworkLockUpdate, error) { - v := url.Values{} - v.Set("limit", fmt.Sprint(maxEntries)) - body, err := lc.send(ctx, "GET", "/localapi/v0/tka/log?"+v.Encode(), 200, nil) - if err != nil { - return nil, fmt.Errorf("error %w: %s", err, body) - } - return decodeJSON[[]ipnstate.NetworkLockUpdate](body) -} - -// NetworkLockForceLocalDisable forcibly shuts down network lock on this node. -func (lc *Client) NetworkLockForceLocalDisable(ctx context.Context) error { - // This endpoint expects an empty JSON stanza as the payload. - var b bytes.Buffer - if err := json.NewEncoder(&b).Encode(struct{}{}); err != nil { - return err - } - - if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/force-local-disable", 200, &b); err != nil { - return fmt.Errorf("error: %w", err) - } - return nil -} - -// NetworkLockVerifySigningDeeplink verifies the network lock deeplink contained -// in url and returns information extracted from it. -func (lc *Client) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) { - vr := struct { - URL string - }{url} - - body, err := lc.send(ctx, "POST", "/localapi/v0/tka/verify-deeplink", 200, jsonBody(vr)) - if err != nil { - return nil, fmt.Errorf("sending verify-deeplink: %w", err) - } - - return decodeJSON[*tka.DeeplinkValidationResult](body) -} - -// NetworkLockGenRecoveryAUM generates an AUM for recovering from a tailnet-lock key compromise. -func (lc *Client) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) { - vr := struct { - Keys []tkatype.KeyID - ForkFrom string - }{removeKeys, forkFrom.String()} - - body, err := lc.send(ctx, "POST", "/localapi/v0/tka/generate-recovery-aum", 200, jsonBody(vr)) - if err != nil { - return nil, fmt.Errorf("sending generate-recovery-aum: %w", err) - } - - return body, nil -} - -// NetworkLockCosignRecoveryAUM co-signs a recovery AUM using the node's tailnet lock key. -func (lc *Client) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) { - r := bytes.NewReader(aum.Serialize()) - body, err := lc.send(ctx, "POST", "/localapi/v0/tka/cosign-recovery-aum", 200, r) - if err != nil { - return nil, fmt.Errorf("sending cosign-recovery-aum: %w", err) - } - - return body, nil -} - -// NetworkLockSubmitRecoveryAUM submits a recovery AUM to the control plane. -func (lc *Client) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error { - r := bytes.NewReader(aum.Serialize()) - _, err := lc.send(ctx, "POST", "/localapi/v0/tka/submit-recovery-aum", 200, r) - if err != nil { - return fmt.Errorf("sending cosign-recovery-aum: %w", err) - } - return nil -} - -// SetServeConfig sets or replaces the serving settings. -// If config is nil, settings are cleared and serving is disabled. -func (lc *Client) SetServeConfig(ctx context.Context, config *ipn.ServeConfig) error { - h := make(http.Header) - if config != nil { - h.Set("If-Match", config.ETag) - } - _, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/serve-config", 200, jsonBody(config), h) - if err != nil { - return fmt.Errorf("sending serve config: %w", err) - } - return nil -} - // DisconnectControl shuts down all connections to control, thus making control consider this node inactive. This can be // run on HA subnet router or app connector replicas before shutting them down to ensure peers get told to switch over // to another replica whilst there is still some grace period for the existing connections to terminate. @@ -1426,40 +1169,6 @@ func (lc *Client) DisconnectControl(ctx context.Context) error { return nil } -// NetworkLockDisable shuts down network-lock across the tailnet. -func (lc *Client) NetworkLockDisable(ctx context.Context, secret []byte) error { - if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil { - return fmt.Errorf("error: %w", err) - } - return nil -} - -// GetServeConfig return the current serve config. -// -// If the serve config is empty, it returns (nil, nil). -func (lc *Client) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) { - body, h, err := lc.sendWithHeaders(ctx, "GET", "/localapi/v0/serve-config", 200, nil, nil) - if err != nil { - return nil, fmt.Errorf("getting serve config: %w", err) - } - sc, err := getServeConfigFromJSON(body) - if err != nil { - return nil, err - } - if sc == nil { - sc = new(ipn.ServeConfig) - } - sc.ETag = h.Get("Etag") - return sc, nil -} - -func getServeConfigFromJSON(body []byte) (sc *ipn.ServeConfig, err error) { - if err := json.Unmarshal(body, &sc); err != nil { - return nil, err - } - return sc, nil -} - // tailscaledConnectHint gives a little thing about why tailscaled (or // platform equivalent) is not answering localapi connections. // @@ -1478,7 +1187,7 @@ func tailscaledConnectHint() string { // ActiveState=inactive // SubState=dead st := map[string]string{} - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if k, v, ok := strings.Cut(line, "="); ok { st[k] = strings.TrimSpace(v) } @@ -1615,6 +1324,16 @@ func (lc *Client) DebugSetExpireIn(ctx context.Context, d time.Duration) error { return err } +// DebugPeerRelaySessions returns debug information about the current peer +// relay sessions running through this node. +func (lc *Client) DebugPeerRelaySessions(ctx context.Context) (*status.ServerStatus, error) { + body, err := lc.send(ctx, "GET", "/localapi/v0/debug-peer-relay-sessions", 200, nil) + if err != nil { + return nil, fmt.Errorf("error %w: %s", err, body) + } + return decodeJSON[*status.ServerStatus](body) +} + // StreamDebugCapture streams a pcap-formatted packet capture. // // The provided context does not determine the lifetime of the @@ -1752,7 +1471,7 @@ type IPNBusWatcher struct { httpRes *http.Response dec *json.Decoder - mu sync.Mutex + mu syncs.Mutex closed bool } @@ -1788,3 +1507,44 @@ func (lc *Client) SuggestExitNode(ctx context.Context) (apitype.ExitNodeSuggesti } return decodeJSON[apitype.ExitNodeSuggestionResponse](body) } + +// CheckSOMarkInUse reports whether the socket mark option is in use. This will only +// be true if tailscale is running on Linux and tailscaled uses SO_MARK. +func (lc *Client) CheckSOMarkInUse(ctx context.Context) (bool, error) { + body, err := lc.get200(ctx, "/localapi/v0/check-so-mark-in-use") + if err != nil { + return false, err + } + var res struct { + UseSOMark bool `json:"useSoMark"` + } + + if err := json.Unmarshal(body, &res); err != nil { + return false, fmt.Errorf("invalid JSON from check-so-mark-in-use: %w", err) + } + return res.UseSOMark, nil +} + +// ShutdownTailscaled requests a graceful shutdown of tailscaled. +func (lc *Client) ShutdownTailscaled(ctx context.Context) error { + _, err := lc.send(ctx, "POST", "/localapi/v0/shutdown", 200, nil) + return err +} + +func (lc *Client) GetAppConnectorRouteInfo(ctx context.Context) (appctype.RouteInfo, error) { + body, err := lc.get200(ctx, "/localapi/v0/appc-route-info") + if err != nil { + return appctype.RouteInfo{}, err + } + return decodeJSON[appctype.RouteInfo](body) +} + +// GetServices returns the Services visible to this node, +// including their names, IP addresses, and ports, keyed by service name. +func (lc *Client) GetServices(ctx context.Context) (map[tailcfg.ServiceName]tailcfg.ServiceDetails, error) { + body, err := lc.get200(ctx, "/localapi/v0/services") + if err != nil { + return nil, err + } + return decodeJSON[map[tailcfg.ServiceName]tailcfg.ServiceDetails](body) +} diff --git a/client/local/local_test.go b/client/local/local_test.go index 0e01e74cd1813..58a87b224564b 100644 --- a/client/local/local_test.go +++ b/client/local/local_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 @@ -61,6 +61,57 @@ func TestWhoIsPeerNotFound(t *testing.T) { } } +func TestUserDialSelf(t *testing.T) { + // Start a real TCP listener that the client should dial directly + // when the server tells it to dial-self. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Write([]byte("hello")) + c.Close() + } + }() + targetAddr := ln.Addr().(*net.TCPAddr) + + // Mock LocalAPI server that returns Dial-Self response. + nw := nettest.GetNetwork(t) + ts := nettest.NewHTTPServer(nw, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Dial-Self", "true") + w.Header().Set("Dial-Addr", targetAddr.String()) + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + lc := &Client{ + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + return nw.Dial(ctx, network, ts.Listener.Addr().String()) + }, + } + + conn, err := lc.UserDial(context.Background(), "tcp", targetAddr.IP.String(), uint16(targetAddr.Port)) + if err != nil { + t.Fatalf("UserDial: %v", err) + } + defer conn.Close() + + buf := make([]byte, 5) + n, err := conn.Read(buf) + if err != nil { + t.Fatalf("Read: %v", err) + } + if got := string(buf[:n]); got != "hello" { + t.Errorf("got %q, want %q", got, "hello") + } +} + func TestDeps(t *testing.T) { deptest.DepChecker{ BadDeps: map[string]string{ diff --git a/client/local/routecheck.go b/client/local/routecheck.go new file mode 100644 index 0000000000000..bf64842f70b47 --- /dev/null +++ b/client/local/routecheck.go @@ -0,0 +1,43 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_routecheck + +package local + +import ( + "context" + "errors" + "fmt" + "net/http" + + "tailscale.com/net/routecheck" +) + +// ErrReportPending is returned by [Client.RouteCheck] and [Client.RouteCheckProbe] +// when the report is pending. +var ErrRouteCheckReportUnavailable = errors.New("report pending") + +// RouteCheckProbe performs a routecheck probe and waits for its report. +func (lc *Client) RouteCheckProbe(ctx context.Context) (*routecheck.Report, error) { + body, err := lc.send(ctx, "POST", "/localapi/v0/routecheck?probe=true", http.StatusOK, nil) + if err != nil { + if hs, ok := errors.AsType[httpStatusError](err); ok && hs.HTTPStatus == http.StatusNoContent { + return nil, ErrRouteCheckReportUnavailable + } + return nil, fmt.Errorf("error %w: %s", err, body) + } + return decodeJSON[*routecheck.Report](body) +} + +// RouteCheck requests the report compiled by the latest routecheck probe. +func (lc *Client) RouteCheck(ctx context.Context) (*routecheck.Report, error) { + body, err := lc.send(ctx, "POST", "/localapi/v0/routecheck", http.StatusOK, nil) + if err != nil { + if hs, ok := errors.AsType[httpStatusError](err); ok && hs.HTTPStatus == http.StatusNoContent { + return nil, ErrRouteCheckReportUnavailable + } + return nil, fmt.Errorf("error %w: %s", err, body) + } + return decodeJSON[*routecheck.Report](body) +} diff --git a/client/local/serve.go b/client/local/serve.go new file mode 100644 index 0000000000000..7f9a16a03f825 --- /dev/null +++ b/client/local/serve.go @@ -0,0 +1,55 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_serve + +package local + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "tailscale.com/ipn" +) + +// GetServeConfig return the current serve config. +// +// If the serve config is empty, it returns (nil, nil). +func (lc *Client) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) { + body, h, err := lc.sendWithHeaders(ctx, "GET", "/localapi/v0/serve-config", 200, nil, nil) + if err != nil { + return nil, fmt.Errorf("getting serve config: %w", err) + } + sc, err := getServeConfigFromJSON(body) + if err != nil { + return nil, err + } + if sc == nil { + sc = new(ipn.ServeConfig) + } + sc.ETag = h.Get("Etag") + return sc, nil +} + +func getServeConfigFromJSON(body []byte) (sc *ipn.ServeConfig, err error) { + if err := json.Unmarshal(body, &sc); err != nil { + return nil, err + } + return sc, nil +} + +// SetServeConfig sets or replaces the serving settings. +// If config is nil, settings are cleared and serving is disabled. +func (lc *Client) SetServeConfig(ctx context.Context, config *ipn.ServeConfig) error { + h := make(http.Header) + if config != nil { + h.Set("If-Match", config.ETag) + } + _, _, err := lc.sendWithHeaders(ctx, "POST", "/localapi/v0/serve-config", 200, jsonBody(config), h) + if err != nil { + return fmt.Errorf("sending serve config: %w", err) + } + return nil +} diff --git a/client/local/syspolicy.go b/client/local/syspolicy.go new file mode 100644 index 0000000000000..49708fa154d9a --- /dev/null +++ b/client/local/syspolicy.go @@ -0,0 +1,40 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_syspolicy + +package local + +import ( + "context" + "net/http" + + "tailscale.com/util/syspolicy/setting" +) + +// GetEffectivePolicy returns the effective policy for the specified scope. +func (lc *Client) GetEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) { + scopeID, err := scope.MarshalText() + if err != nil { + return nil, err + } + body, err := lc.get200(ctx, "/localapi/v0/policy/"+string(scopeID)) + if err != nil { + return nil, err + } + return decodeJSON[*setting.Snapshot](body) +} + +// ReloadEffectivePolicy reloads the effective policy for the specified scope +// by reading and merging policy settings from all applicable policy sources. +func (lc *Client) ReloadEffectivePolicy(ctx context.Context, scope setting.PolicyScope) (*setting.Snapshot, error) { + scopeID, err := scope.MarshalText() + if err != nil { + return nil, err + } + body, err := lc.send(ctx, "POST", "/localapi/v0/policy/"+string(scopeID), 200, http.NoBody) + if err != nil { + return nil, err + } + return decodeJSON[*setting.Snapshot](body) +} diff --git a/client/local/tailnetlock.go b/client/local/tailnetlock.go new file mode 100644 index 0000000000000..8445d9bd00ab8 --- /dev/null +++ b/client/local/tailnetlock.go @@ -0,0 +1,267 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_tailnetlock + +package local + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tka" + "tailscale.com/types/key" + "tailscale.com/types/tkatype" +) + +// TailnetLockStatus fetches information about the tailnet key authority, if one is configured. +func (lc *Client) TailnetLockStatus(ctx context.Context) (*ipnstate.TailnetLockStatus, error) { + body, err := lc.send(ctx, "GET", "/localapi/v0/tka/status", 200, nil) + if err != nil { + return nil, fmt.Errorf("error: %w", err) + } + return decodeJSON[*ipnstate.TailnetLockStatus](body) +} + +// Deprecated: use [Client.TailnetLockStatus] instead. +func (lc *Client) NetworkLockStatus(ctx context.Context) (*ipnstate.TailnetLockStatus, error) { + return lc.TailnetLockStatus(ctx) +} + +// TailnetLockInit initializes the tailnet key authority. +func (lc *Client) TailnetLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.TailnetLockStatus, error) { + var b bytes.Buffer + type initRequest struct { + Keys []tka.Key + DisablementValues [][]byte + SupportDisablement []byte + } + + if err := json.NewEncoder(&b).Encode(initRequest{Keys: keys, DisablementValues: disablementValues, SupportDisablement: supportDisablement}); err != nil { + return nil, err + } + + body, err := lc.send(ctx, "POST", "/localapi/v0/tka/init", 200, &b) + if err != nil { + return nil, fmt.Errorf("error: %w", err) + } + return decodeJSON[*ipnstate.TailnetLockStatus](body) +} + +// Deprecated: use [Client.TailnetLockInit] instead. +func (lc *Client) NetworkLockInit(ctx context.Context, keys []tka.Key, disablementValues [][]byte, supportDisablement []byte) (*ipnstate.TailnetLockStatus, error) { + return lc.TailnetLockInit(ctx, keys, disablementValues, supportDisablement) +} + +// TailnetLockWrapPreauthKey wraps a pre-auth key with information to +// enable unattended bringup in the locked tailnet. +func (lc *Client) TailnetLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) { + encodedPrivate, err := tkaKey.MarshalText() + if err != nil { + return "", err + } + + var b bytes.Buffer + type wrapRequest struct { + TSKey string + TKAKey string // key.NLPrivate.MarshalText + } + if err := json.NewEncoder(&b).Encode(wrapRequest{TSKey: preauthKey, TKAKey: string(encodedPrivate)}); err != nil { + return "", err + } + + body, err := lc.send(ctx, "POST", "/localapi/v0/tka/wrap-preauth-key", 200, &b) + if err != nil { + return "", fmt.Errorf("error: %w", err) + } + return string(body), nil +} + +// Deprecated: use [Client.TailnetLockWrapPreauthKey] instead. +func (lc *Client) NetworkLockWrapPreauthKey(ctx context.Context, preauthKey string, tkaKey key.NLPrivate) (string, error) { + return lc.TailnetLockWrapPreauthKey(ctx, preauthKey, tkaKey) +} + +// TailnetLockModify adds and/or removes key(s) to the tailnet key authority. +func (lc *Client) TailnetLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error { + var b bytes.Buffer + type modifyRequest struct { + AddKeys []tka.Key + RemoveKeys []tka.Key + } + + if err := json.NewEncoder(&b).Encode(modifyRequest{AddKeys: addKeys, RemoveKeys: removeKeys}); err != nil { + return err + } + + if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/modify", 204, &b); err != nil { + return fmt.Errorf("error: %w", err) + } + return nil +} + +// Deprecated: use [Client.TailnetLockModify] instead. +func (lc *Client) NetworkLockModify(ctx context.Context, addKeys, removeKeys []tka.Key) error { + return lc.TailnetLockModify(ctx, addKeys, removeKeys) +} + +// TailnetLockSign signs the specified node-key and transmits that signature to the control plane. +// rotationPublic, if specified, must be an ed25519 public key. +func (lc *Client) TailnetLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error { + var b bytes.Buffer + type signRequest struct { + NodeKey key.NodePublic + RotationPublic []byte + } + + if err := json.NewEncoder(&b).Encode(signRequest{NodeKey: nodeKey, RotationPublic: rotationPublic}); err != nil { + return err + } + + if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/sign", 200, &b); err != nil { + return fmt.Errorf("error: %w", err) + } + return nil +} + +// Deprecated: use [Client.TailnetLockSign] instead. +func (lc *Client) NetworkLockSign(ctx context.Context, nodeKey key.NodePublic, rotationPublic []byte) error { + return lc.TailnetLockSign(ctx, nodeKey, rotationPublic) +} + +// TailnetLockAffectedSigs returns all signatures signed by the specified keyID. +func (lc *Client) TailnetLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) { + body, err := lc.send(ctx, "POST", "/localapi/v0/tka/affected-sigs", 200, bytes.NewReader(keyID)) + if err != nil { + return nil, fmt.Errorf("error: %w", err) + } + return decodeJSON[[]tkatype.MarshaledSignature](body) +} + +// Deprecated: use [Client.TailnetLockAffectedSigs] instead. +func (lc *Client) NetworkLockAffectedSigs(ctx context.Context, keyID tkatype.KeyID) ([]tkatype.MarshaledSignature, error) { + return lc.TailnetLockAffectedSigs(ctx, keyID) +} + +// TailnetLockLog returns up to maxEntries number of changes to tailnet-lock state. +func (lc *Client) TailnetLockLog(ctx context.Context, maxEntries int) ([]ipnstate.TailnetLockUpdate, error) { + v := url.Values{} + v.Set("limit", fmt.Sprint(maxEntries)) + body, err := lc.send(ctx, "GET", "/localapi/v0/tka/log?"+v.Encode(), 200, nil) + if err != nil { + return nil, fmt.Errorf("error %w: %s", err, body) + } + return decodeJSON[[]ipnstate.TailnetLockUpdate](body) +} + +// Deprecated: use [Client.TailnetLockLog] instead. +func (lc *Client) NetworkLockLog(ctx context.Context, maxEntries int) ([]ipnstate.TailnetLockUpdate, error) { + return lc.TailnetLockLog(ctx, maxEntries) +} + +// TailnetLockForceLocalDisable forcibly shuts down tailnet lock on this node. +func (lc *Client) TailnetLockForceLocalDisable(ctx context.Context) error { + // This endpoint expects an empty JSON stanza as the payload. + var b bytes.Buffer + if err := json.NewEncoder(&b).Encode(struct{}{}); err != nil { + return err + } + + if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/force-local-disable", 200, &b); err != nil { + return fmt.Errorf("error: %w", err) + } + return nil +} + +// Deprecated: use [Client.TailnetLockForceLocalDisable] instead. +func (lc *Client) NetworkLockForceLocalDisable(ctx context.Context) error { + return lc.TailnetLockForceLocalDisable(ctx) +} + +// TailnetLockVerifySigningDeeplink verifies the tailnet lock deeplink contained +// in url and returns information extracted from it. +func (lc *Client) TailnetLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) { + vr := struct { + URL string + }{url} + + body, err := lc.send(ctx, "POST", "/localapi/v0/tka/verify-deeplink", 200, jsonBody(vr)) + if err != nil { + return nil, fmt.Errorf("sending verify-deeplink: %w", err) + } + + return decodeJSON[*tka.DeeplinkValidationResult](body) +} + +// Deprecated: use [Client.TailnetLockVerifySigningDeeplink] instead. +func (lc *Client) NetworkLockVerifySigningDeeplink(ctx context.Context, url string) (*tka.DeeplinkValidationResult, error) { + return lc.TailnetLockVerifySigningDeeplink(ctx, url) +} + +// TailnetLockGenRecoveryAUM generates an AUM for recovering from a tailnet-lock key compromise. +func (lc *Client) TailnetLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) { + vr := struct { + Keys []tkatype.KeyID + ForkFrom string + }{removeKeys, forkFrom.String()} + + body, err := lc.send(ctx, "POST", "/localapi/v0/tka/generate-recovery-aum", 200, jsonBody(vr)) + if err != nil { + return nil, fmt.Errorf("sending generate-recovery-aum: %w", err) + } + + return body, nil +} + +// Deprecated: use [Client.TailnetLockGenRecoveryAUM] instead. +func (lc *Client) NetworkLockGenRecoveryAUM(ctx context.Context, removeKeys []tkatype.KeyID, forkFrom tka.AUMHash) ([]byte, error) { + return lc.TailnetLockGenRecoveryAUM(ctx, removeKeys, forkFrom) +} + +// TailnetLockCosignRecoveryAUM co-signs a recovery AUM using the node's tailnet lock key. +func (lc *Client) TailnetLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) { + r := bytes.NewReader(aum.Serialize()) + body, err := lc.send(ctx, "POST", "/localapi/v0/tka/cosign-recovery-aum", 200, r) + if err != nil { + return nil, fmt.Errorf("sending cosign-recovery-aum: %w", err) + } + + return body, nil +} + +// Deprecated: use [Client.TailnetLockCosignRecoveryAUM] instead. +func (lc *Client) NetworkLockCosignRecoveryAUM(ctx context.Context, aum tka.AUM) ([]byte, error) { + return lc.TailnetLockCosignRecoveryAUM(ctx, aum) +} + +// TailnetLockSubmitRecoveryAUM submits a recovery AUM to the control plane. +func (lc *Client) TailnetLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error { + r := bytes.NewReader(aum.Serialize()) + _, err := lc.send(ctx, "POST", "/localapi/v0/tka/submit-recovery-aum", 200, r) + if err != nil { + return fmt.Errorf("sending cosign-recovery-aum: %w", err) + } + return nil +} + +// Deprecated: use [Client.TailnetLockSubmitRecoveryAUM] instead. +func (lc *Client) NetworkLockSubmitRecoveryAUM(ctx context.Context, aum tka.AUM) error { + return lc.TailnetLockSubmitRecoveryAUM(ctx, aum) +} + +// TailnetLockDisable shuts down tailnet-lock across the tailnet. +func (lc *Client) TailnetLockDisable(ctx context.Context, secret []byte) error { + if _, err := lc.send(ctx, "POST", "/localapi/v0/tka/disable", 200, bytes.NewReader(secret)); err != nil { + return fmt.Errorf("error: %w", err) + } + return nil +} + +// Deprecated: use [Client.TailnetLockDisable] instead. +func (lc *Client) NetworkLockDisable(ctx context.Context, secret []byte) error { + return lc.TailnetLockDisable(ctx, secret) +} diff --git a/client/systray/logo.go b/client/systray/logo.go index 3467d1b741f93..334cd7917cf59 100644 --- a/client/systray/logo.go +++ b/client/systray/logo.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build cgo || !darwin @@ -11,6 +11,7 @@ import ( "image" "image/color" "image/png" + "log" "runtime" "sync" "time" @@ -204,12 +205,49 @@ var ( ) var ( - bg = color.NRGBA{0, 0, 0, 255} - fg = color.NRGBA{255, 255, 255, 255} - gray = color.NRGBA{255, 255, 255, 102} - red = color.NRGBA{229, 111, 74, 255} + black = color.NRGBA{0, 0, 0, 255} + white = color.NRGBA{255, 255, 255, 255} + darkGray = color.NRGBA{102, 102, 102, 255} + lightGray = color.NRGBA{153, 153, 153, 255} + red = color.NRGBA{229, 111, 74, 255} + transparent = color.NRGBA{} + + // default values to dark theme + bg = black + fg = white + gray = darkGray ) +// SetTheme sets the color theme of the systray icon. +// +// Supported themes are: +// - dark - white and gray dots over black background +// - dark:nobg - white and grey dots over transparent background +// - light - black and gray dots over white background +// - light:nobg - black and grey dots over transparent background +func SetTheme(theme string) { + switch theme { + case "dark": + bg = black + fg = white + gray = darkGray + case "dark:nobg": + bg = transparent + fg = white + gray = darkGray + case "light": + bg = white + fg = black + gray = lightGray + case "light:nobg": + bg = transparent + fg = black + gray = lightGray + default: + log.Printf("unknown theme: %q", theme) + } +} + // render returns a PNG image of the logo. func (logo tsLogo) render() *bytes.Buffer { const borderUnits = 1 @@ -233,8 +271,8 @@ func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer { dc.InvertMask() } - for y := 0; y < 3; y++ { - for x := 0; x < 3; x++ { + for y := range 3 { + for x := range 3 { px := (borderUnits + 1 + 3*x) * radius py := (borderUnits + 1 + 3*y) * radius col := fg diff --git a/client/systray/startup-creator.go b/client/systray/startup-creator.go new file mode 100644 index 0000000000000..02a01809945e1 --- /dev/null +++ b/client/systray/startup-creator.go @@ -0,0 +1,215 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build cgo || !darwin + +package systray + +import ( + "bufio" + "bytes" + _ "embed" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "tailscale.com/client/freedesktop" +) + +//go:embed tailscale-systray.service +var embedSystemd string + +//go:embed tailscale-systray.desktop +var embedFreedesktop string + +//go:embed tailscale.svg +var embedLogoSvg string + +//go:embed tailscale.png +var embedLogoPng string + +func InstallStartupScript(initSystem string) error { + switch initSystem { + case "systemd": + return installSystemd() + case "freedesktop": + return installFreedesktop() + default: + return fmt.Errorf("unsupported init system '%s'", initSystem) + } +} + +func installSystemd() error { + // Find the path to tailscale, just in case it's not where the example file + // has it placed, and replace that before writing the file. + tailscaleBin, err := exec.LookPath("tailscale") + if err != nil { + return fmt.Errorf("failed to find tailscale binary %w", err) + } + + var output bytes.Buffer + scanner := bufio.NewScanner(strings.NewReader(embedSystemd)) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "ExecStart=") { + line = fmt.Sprintf("ExecStart=%s systray", tailscaleBin) + } + output.WriteString(line + "\n") + } + + configDir, err := os.UserConfigDir() + if err != nil { + homeDir, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("unable to locate user home: %w", err) + } + configDir = filepath.Join(homeDir, ".config") + } + + systemdDir := filepath.Join(configDir, "systemd", "user") + if err := os.MkdirAll(systemdDir, 0o755); err != nil { + return fmt.Errorf("failed creating systemd user dir: %w", err) + } + + serviceFile := filepath.Join(systemdDir, "tailscale-systray.service") + + if err := os.WriteFile(serviceFile, output.Bytes(), 0o755); err != nil { + return fmt.Errorf("failed writing systemd user service: %w", err) + } + + fmt.Printf("Successfully installed systemd service to: %s\n", serviceFile) + fmt.Println("To enable and start the service, run:") + fmt.Println(" systemctl --user daemon-reload") + fmt.Println(" systemctl --user enable --now tailscale-systray") + + return nil +} + +func installFreedesktop() error { + tmpDir, err := os.MkdirTemp("", "tailscale-systray") + if err != nil { + return fmt.Errorf("unable to make tmpDir: %w", err) + } + defer os.RemoveAll(tmpDir) + + // Install icon, and use it if it works, and if not change to some generic + // network/vpn icon. + iconName := "tailscale" + if err := installIcon(tmpDir); err != nil { + iconName = "network-transmit" + fmt.Printf("unable to install icon, continuing without: %s\n", err.Error()) + } + + // Create desktop file in a tmp dir + desktopTmpPath := filepath.Join(tmpDir, "tailscale-systray.desktop") + if err := os.WriteFile(desktopTmpPath, []byte(embedFreedesktop), + 0o0755); err != nil { + return fmt.Errorf("unable to create desktop file: %w", err) + } + + // Ensure autostart dir exists and install the desktop file + configDir, err := os.UserConfigDir() + if err != nil { + homeDir, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("unable to locate user home: %w", err) + } + configDir = filepath.Join(homeDir, ".config") + } + + autostartDir := filepath.Join(configDir, "autostart") + if err := os.MkdirAll(autostartDir, 0o644); err != nil { + return fmt.Errorf("failed creating freedesktop autostart dir: %w", err) + } + + desktopCmd := exec.Command("desktop-file-install", "--dir", autostartDir, + desktopTmpPath) + if output, err := desktopCmd.Output(); err != nil { + return fmt.Errorf("unable to install desktop file: %w - %s", err, output) + } + + // Find the path to tailscale, just in case it's not where the example file + // has it placed, and replace that before writing the file. + tailscaleBin, err := os.Executable() + if err != nil { + return fmt.Errorf("failed to find tailscale binary %w", err) + } + tailscaleBin = freedesktop.Quote(tailscaleBin) + + // Make possible changes to the desktop file + runEdit := func(args ...string) error { + cmd := exec.Command("desktop-file-edit", args...) + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("cmd: %s: %w\n%s", cmd.String(), err, out) + } + return nil + } + + edits := [][]string{ + {"--set-key=Exec", "--set-value=" + tailscaleBin + " systray"}, + {"--set-key=TryExec", "--set-value=" + tailscaleBin}, + {"--set-icon=" + iconName}, + } + + var errs []error + desktopFile := filepath.Join(autostartDir, "tailscale-systray.desktop") + for _, args := range edits { + args = append(args, desktopFile) + if err := runEdit(args...); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return fmt.Errorf( + "failed changing autostart file, try rebooting: %w", errors.Join(errs...)) + } + + fmt.Printf("Successfully installed freedesktop autostart service to: %s\n", desktopFile) + fmt.Println("The service will run upon logging in.") + + return nil +} + +// installIcon installs an icon using the freedesktop tools. SVG support +// is still on its way for some distros, notably missing on Ubuntu 25.10 as of +// 2026-02-19. Try to install both icons and let the DE decide from what is +// available. +// Reference: https://gitlab.freedesktop.org/xdg/xdg-utils/-/merge_requests/116 +func installIcon(tmpDir string) error { + svgPath := filepath.Join(tmpDir, "tailscale.svg") + if err := os.WriteFile(svgPath, []byte(embedLogoSvg), 0o0644); err != nil { + return fmt.Errorf("unable to create svg: %w", err) + } + + pngPath := filepath.Join(tmpDir, "tailscale.png") + if err := os.WriteFile(pngPath, []byte(embedLogoPng), 0o0644); err != nil { + return fmt.Errorf("unable to create png: %w", err) + } + + var errs []error + installed := false + svgCmd := exec.Command("xdg-icon-resource", "install", "--size", "scalable", + "--novendor", svgPath, "tailscale") + if output, err := svgCmd.Output(); err != nil { + errs = append(errs, fmt.Errorf("unable to install svg: %s - %s", err, output)) + } else { + installed = true + } + pngCmd := exec.Command("xdg-icon-resource", "install", "--size", "512", + "--novendor", pngPath, "tailscale") + if output, err := pngCmd.Output(); err != nil { + errs = append(errs, fmt.Errorf("unable to install png: %s - %s", err, output)) + } else { + installed = true + } + + if !installed { + return errors.Join(errs...) + } + return nil +} diff --git a/client/systray/systray.go b/client/systray/systray.go index 195a157fb1386..eb550fde49f93 100644 --- a/client/systray/systray.go +++ b/client/systray/systray.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build cgo || !darwin @@ -48,7 +48,12 @@ var ( ) // Run starts the systray menu and blocks until the menu exits. -func (menu *Menu) Run() { +// If client is nil, a default local.Client is used. +func (menu *Menu) Run(client *local.Client) { + if client == nil { + client = &local.Client{} + } + menu.lc = client menu.updateState() // exit cleanly on SIGINT and SIGTERM @@ -61,7 +66,13 @@ func (menu *Menu) Run() { case <-menu.bgCtx.Done(): } }() - go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1) + go menu.lc.SetGauge(menu.bgCtx, "systray_running", 1) + defer menu.lc.SetGauge(menu.bgCtx, "systray_running", 0) + + // set initial title, which is used by the systray package as the ID of the StatusNotifierItem. + // This value will get overwritten later as the client status changes. + // This must be called before systray.Run. + systray.SetTitle("tailscale") systray.Run(menu.onReady, menu.onExit) } @@ -70,7 +81,7 @@ func (menu *Menu) Run() { type Menu struct { mu sync.Mutex // protects the entire Menu - lc local.Client + lc *local.Client status *ipnstate.Status curProfile ipn.LoginProfile allProfiles []ipn.LoginProfile @@ -127,7 +138,7 @@ func init() { desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP")) switch desktop { - case "gnome": + case "gnome", "ubuntu:gnome": // GNOME expands submenus downward in the main menu, rather than flyouts to the side. // Either as a result of that or another limitation, there seems to be a maximum depth of submenus. // Mullvad countries that have a city submenu are not being rendered, and so can't be selected. @@ -152,8 +163,32 @@ func init() { // onReady is called by the systray package when the menu is ready to be built. func (menu *Menu) onReady() { log.Printf("starting") + if os.Getuid() == 0 || os.Getuid() != os.Geteuid() || os.Getenv("SUDO_USER") != "" || os.Getenv("DOAS_USER") != "" { + fmt.Fprintln(os.Stderr, ` +It appears that you might be running the systray with sudo/doas. +This can lead to issues with D-Bus, and should be avoided. + +The systray application should be run with the same user as your desktop session. +This usually means that you should run the application like: + +tailscale systray + +See https://tailscale.com/kb/1597/linux-systray for more information.`) + } setAppIcon(disconnected) + menu.rebuild() + + menu.mu.Lock() + if menu.readonly { + fmt.Fprintln(os.Stderr, ` +No permission to manage Tailscale. Set operator by running: + +sudo tailscale set --operator=$USER + +See https://tailscale.com/s/cli-operator for more information.`) + } + menu.mu.Unlock() } // updateState updates the Menu state from the Tailscale local client. @@ -258,21 +293,23 @@ func (menu *Menu) rebuild() { accounts := systray.AddMenuItem(account, "") setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL) time.Sleep(newMenuDelay) - for _, profile := range menu.allProfiles { - title := profileTitle(profile) - var item *systray.MenuItem - if profile.ID == menu.curProfile.ID { - item = accounts.AddSubMenuItemCheckbox(title, "", true) - } else { - item = accounts.AddSubMenuItem(title, "") - } - setRemoteIcon(item, profile.UserProfile.ProfilePicURL) - onClick(ctx, item, func(ctx context.Context) { - select { - case <-ctx.Done(): - case menu.accountsCh <- profile.ID: + if len(menu.allProfiles) > 1 { + for _, profile := range menu.allProfiles { + title := profileTitle(profile) + var item *systray.MenuItem + if profile.ID == menu.curProfile.ID { + item = accounts.AddSubMenuItemCheckbox(title, "", true) + } else { + item = accounts.AddSubMenuItem(title, "") } - }) + setRemoteIcon(item, profile.UserProfile.ProfilePicURL) + onClick(ctx, item, func(ctx context.Context) { + select { + case <-ctx.Done(): + case menu.accountsCh <- profile.ID: + } + }) + } } } @@ -289,11 +326,14 @@ func (menu *Menu) rebuild() { menu.rebuildExitNodeMenu(ctx) } - if menu.status != nil { - menu.more = systray.AddMenuItem("More settings", "") + menu.more = systray.AddMenuItem("More settings", "") + if menu.status != nil && menu.status.BackendState == "Running" { + // web client is only available if backend is running onClick(ctx, menu.more, func(_ context.Context) { webbrowser.Open("http://100.100.100.100/") }) + } else { + menu.more.Disable() } // TODO(#15528): this menu item shouldn't be necessary at all, @@ -315,16 +355,27 @@ func (menu *Menu) rebuild() { // profileTitle returns the title string for a profile menu item. func profileTitle(profile ipn.LoginProfile) string { - title := profile.Name + tailnet := "" if profile.NetworkProfile.DomainName != "" { - if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { - // windows and mac don't support multi-line menu - title += " (" + profile.NetworkProfile.DomainName + ")" - } else { - title += "\n" + profile.NetworkProfile.DomainName - } + tailnet = profile.NetworkProfile.DisplayNameOrDefault() } - return title + // windows and mac don't support multi-line menu items. + multiline := runtime.GOOS != "windows" && runtime.GOOS != "darwin" + + return formatProfileTitle(profile.Name, tailnet, multiline) +} + +// formatProfileTitle builds a profile menu label from a login name and an +// optional tailnet name. The tailnet portion is omitted when it matches the +// login name, so single-user tailnets don't show the same string twice. +func formatProfileTitle(name, tailnet string, multiline bool) string { + if tailnet == "" || strings.EqualFold(name, tailnet) { + return name + } + if multiline { + return name + "\n" + tailnet + } + return name + " (" + tailnet + ")" } var ( @@ -340,6 +391,7 @@ func setRemoteIcon(menu *systray.MenuItem, urlStr string) { } cacheMu.Lock() + defer cacheMu.Unlock() b, ok := httpCache[urlStr] if !ok { resp, err := http.Get(urlStr) @@ -363,7 +415,6 @@ func setRemoteIcon(menu *systray.MenuItem, urlStr string) { resp.Body.Close() } } - cacheMu.Unlock() if len(b) > 0 { menu.SetIcon(b) @@ -480,7 +531,7 @@ func (menu *Menu) watchIPNBus() { } func (menu *Menu) watchIPNBusInner() error { - watcher, err := menu.lc.WatchIPNBus(menu.bgCtx, ipn.NotifyNoPrivateKeys) + watcher, err := menu.lc.WatchIPNBus(menu.bgCtx, 0) if err != nil { return fmt.Errorf("watching ipn bus: %w", err) } @@ -494,6 +545,15 @@ func (menu *Menu) watchIPNBusInner() error { if err != nil { return fmt.Errorf("ipnbus error: %w", err) } + if url := n.BrowseToURL; url != nil { + // Avoid opening the browser when running as root, just in case. + runningAsRoot := os.Getuid() == 0 + if !runningAsRoot { + if err := webbrowser.Open(*url); err != nil { + log.Printf("failed to open BrowseToURL: %v", err) + } + } + } var rebuild bool if n.State != nil { log.Printf("new state: %v", n.State) @@ -520,9 +580,9 @@ func (menu *Menu) copyTailscaleIP(device *ipnstate.PeerStatus) { err := clipboard.WriteAll(ip) if err != nil { log.Printf("clipboard error: %v", err) + } else { + menu.sendNotification(fmt.Sprintf("Copied Address for %v", name), ip) } - - menu.sendNotification(fmt.Sprintf("Copied Address for %v", name), ip) } // sendNotification sends a desktop notification with the given title and content. @@ -575,11 +635,9 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { title += strings.Split(sugg.Name, ".")[0] } menu.exitNodes.AddSeparator() - rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false) + active := recommendedIsActive(status, sugg.ID, sugg.Location.CountryCode(), sugg.Location.City()) + rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", active) setExitNodeOnClick(rm, sugg.ID) - if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID { - rm.Check() - } } } @@ -601,13 +659,11 @@ func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) { if !ps.Online { name += " (offline)" } - sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", false) + active := status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID + sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", active) if !ps.Online { sm.Disable() } - if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID { - sm.Check() - } setExitNodeOnClick(sm, ps.ID) } } @@ -697,6 +753,30 @@ func (mc *mvCountry) sortedCities() []*mvCity { return cities } +// recommendedIsActive reports whether the suggested exit node corresponds to +// the currently active exit node in status. +func recommendedIsActive(status *ipnstate.Status, suggID tailcfg.StableNodeID, suggCountry, suggCity string) bool { + if status == nil || status.ExitNodeStatus == nil || status.ExitNodeStatus.ID.IsZero() { + return false + } + if suggID == status.ExitNodeStatus.ID { + return true + } + if suggCountry == "" || suggCity == "" { + return false + } + for _, p := range status.Peer { + if p.ID != status.ExitNodeStatus.ID { + continue + } + if loc := p.Location; loc != nil && loc.CountryCode == suggCountry && loc.City == suggCity { + return true + } + return false + } + return false +} + // countryFlag takes a 2-character ASCII string and returns the corresponding emoji flag. // It returns the empty string on error. func countryFlag(code string) string { diff --git a/client/systray/systray_test.go b/client/systray/systray_test.go new file mode 100644 index 0000000000000..6bb2bfee3e185 --- /dev/null +++ b/client/systray/systray_test.go @@ -0,0 +1,147 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build cgo || !darwin + +package systray + +import ( + "testing" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/key" +) + +func TestProfileTitleMultiline(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + login string + tailnet string + multiline bool + want string + }{ + {"no_tailnet", "alice@example.com", "", true, "alice@example.com"}, + {"dup_exact", "example.com", "example.com", true, "example.com"}, + {"dup_casefold", "Example.com", "example.com", false, "Example.com"}, + {"distinct_multiline", "alice@example.com", "example.com", true, "alice@example.com\nexample.com"}, + {"distinct_singleline", "alice@example.com", "example.com", false, "alice@example.com (example.com)"}, + {"empty", "", "", true, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := formatProfileTitle(tt.login, tt.tailnet, tt.multiline); got != tt.want { + t.Errorf("profileTitleMultiline; got %v, want %v", got, tt.want) + } + }) + } +} + +func TestRecommendedIsActive(t *testing.T) { + t.Parallel() + + const ( + activeID = tailcfg.StableNodeID("active") + suggID = tailcfg.StableNodeID("suggestion") + ) + usNYC := &tailcfg.Location{CountryCode: "US", City: "New York"} + usCHI := &tailcfg.Location{CountryCode: "US", City: "Chicago"} + seSTO := &tailcfg.Location{CountryCode: "SE", City: "Stockholm"} + + statusWith := func(activePeer *ipnstate.PeerStatus) *ipnstate.Status { + s := &ipnstate.Status{ + ExitNodeStatus: &ipnstate.ExitNodeStatus{ID: activeID}, + } + if activePeer != nil { + s.Peer = map[key.NodePublic]*ipnstate.PeerStatus{{}: activePeer} + } + return s + } + + tests := []struct { + name string + status *ipnstate.Status + suggID tailcfg.StableNodeID + suggCountry string + suggCity string + isActive bool + }{ + { + name: "nil_status", + status: nil, + suggID: suggID, + }, + { + name: "no_exit_node", + status: &ipnstate.Status{}, + suggID: suggID, + }, + { + name: "exit_node_id_is_zero", + status: &ipnstate.Status{ExitNodeStatus: &ipnstate.ExitNodeStatus{}}, + suggID: suggID, + }, + { + name: "exact_id_match_short-circuits", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usCHI}), + suggID: activeID, + suggCountry: "US", + suggCity: "New York", + isActive: true, + }, + { + name: "id_mismatch_but_same_city", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usNYC}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + isActive: true, + }, + { + name: "different_city", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usCHI}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + { + name: "different_country", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: seSTO}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + { + name: "id_mismatch_suggestion_has_no_location", + status: statusWith(&ipnstate.PeerStatus{ID: activeID, Location: usNYC}), + suggID: suggID, + }, + { + name: "id_mismatch_active_peer_has_no_location", + status: statusWith(&ipnstate.PeerStatus{ID: activeID}), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + { + name: "active_peer_not_in_status", + status: statusWith(nil), + suggID: suggID, + suggCountry: "US", + suggCity: "New York", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + isExitNodeActive := recommendedIsActive(tt.status, tt.suggID, tt.suggCountry, tt.suggCity) + if isExitNodeActive != tt.isActive { + t.Errorf("recommendedIsActive; got %v, want %v", isExitNodeActive, tt.isActive) + } + }) + } +} diff --git a/client/systray/tailscale-systray.desktop b/client/systray/tailscale-systray.desktop new file mode 100644 index 0000000000000..b79b72d181ddc --- /dev/null +++ b/client/systray/tailscale-systray.desktop @@ -0,0 +1,13 @@ +[Desktop Entry] +Type=Application +Version=1.0 +Name=Tailscale System Tray +Comment=Tailscale system tray applet for managing Tailscale +Exec=/usr/bin/tailscale systray +TryExec=/usr/bin/tailscale +Terminal=false +NoDisplay=true +StartupNotify=false +Icon=tailscale +Categories=Network;System; +X-GNOME-Autostart-enabled=true diff --git a/client/systray/tailscale-systray.service b/client/systray/tailscale-systray.service new file mode 100644 index 0000000000000..bc4b470043bf7 --- /dev/null +++ b/client/systray/tailscale-systray.service @@ -0,0 +1,13 @@ +[Unit] +Description=Tailscale System Tray +Documentation=https://tailscale.com/kb/1597/linux-systray +Requires=dbus.service +After=dbus.service +PartOf=default.target + +[Service] +Type=simple +ExecStart=/usr/bin/tailscale systray + +[Install] +WantedBy=default.target diff --git a/client/systray/tailscale.png b/client/systray/tailscale.png new file mode 100644 index 0000000000000..d476e88fc262f Binary files /dev/null and b/client/systray/tailscale.png differ diff --git a/client/systray/tailscale.svg b/client/systray/tailscale.svg new file mode 100644 index 0000000000000..9e6990271e472 --- /dev/null +++ b/client/systray/tailscale.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/client/tailscale/acl.go b/client/tailscale/acl.go index 929ec2b3b1ca9..e69d45a2bff5d 100644 --- a/client/tailscale/acl.go +++ b/client/tailscale/acl.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 diff --git a/client/tailscale/apitype/apitype.go b/client/tailscale/apitype/apitype.go index 58cdcecc78d4f..d7d1440be9f8a 100644 --- a/client/tailscale/apitype/apitype.go +++ b/client/tailscale/apitype/apitype.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package apitype contains types for the Tailscale LocalAPI and control plane API. @@ -94,3 +94,13 @@ type DNSQueryResponse struct { // Resolvers is the list of resolvers that the forwarder deemed able to resolve the query. Resolvers []*dnstype.Resolver } + +// OptionalFeatures describes which optional features are enabled in the build. +type OptionalFeatures struct { + // Features is the map of optional feature names to whether they are + // enabled. + // + // Disabled features may be absent from the map. (That is, false values + // are not guaranteed to be present.) + Features map[string]bool +} diff --git a/client/tailscale/apitype/controltype.go b/client/tailscale/apitype/controltype.go index 9a623be319606..2259bb8861aad 100644 --- a/client/tailscale/apitype/controltype.go +++ b/client/tailscale/apitype/controltype.go @@ -1,19 +1,52 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package apitype +// DNSConfig is the DNS configuration for a tailnet +// used in /tailnet/{tailnet}/dns/config. type DNSConfig struct { - Resolvers []DNSResolver `json:"resolvers"` - FallbackResolvers []DNSResolver `json:"fallbackResolvers"` - Routes map[string][]DNSResolver `json:"routes"` - Domains []string `json:"domains"` - Nameservers []string `json:"nameservers"` - Proxied bool `json:"proxied"` - TempCorpIssue13969 string `json:"TempCorpIssue13969,omitempty"` + // Resolvers are the global DNS resolvers to use + // overriding the local OS configuration. + Resolvers []DNSResolver `json:"resolvers"` + + // FallbackResolvers are used as global resolvers when + // the client is unable to determine the OS's preferred DNS servers. + FallbackResolvers []DNSResolver `json:"fallbackResolvers"` + + // Routes map DNS name suffixes to a set of DNS resolvers, + // used for Split DNS and other advanced routing overlays. + Routes map[string][]DNSResolver `json:"routes"` + + // Domains are the search domains to use. + Domains []string `json:"domains"` + + // Proxied means MagicDNS is enabled. + Proxied bool `json:"proxied"` + + // TempCorpIssue13969 is from an internal hack day prototype, + // See tailscale/corp#13969. + TempCorpIssue13969 string `json:"TempCorpIssue13969,omitempty"` + + // Nameservers are the IP addresses of global nameservers to use. + // This is a deprecated format but may still be found in tailnets + // that were configured a long time ago. When making updates, + // set Resolvers and leave Nameservers empty. + Nameservers []string `json:"nameservers"` } +// DNSResolver is a DNS resolver in a DNS configuration. type DNSResolver struct { - Addr string `json:"addr"` + // Addr is the address of the DNS resolver. + // It is usually an IP address or a DoH URL. + // See dnstype.Resolver.Addr for full details. + Addr string `json:"addr"` + + // BootstrapResolution is an optional suggested resolution for + // the DoT/DoH resolver. BootstrapResolution []string `json:"bootstrapResolution,omitempty"` + + // UseWithExitNode signals this resolver should be used + // even when a tailscale exit node is configured on a device. + UseWithExitNode bool `json:"useWithExitNode,omitempty"` } diff --git a/client/tailscale/cert.go b/client/tailscale/cert.go new file mode 100644 index 0000000000000..797c5535d17f5 --- /dev/null +++ b/client/tailscale/cert.go @@ -0,0 +1,34 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !js && !ts_omit_acme + +package tailscale + +import ( + "context" + "crypto/tls" + + "tailscale.com/client/local" +) + +// GetCertificate is an alias for [tailscale.com/client/local.GetCertificate]. +// +// Deprecated: import [tailscale.com/client/local] instead and use [local.Client.GetCertificate]. +func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { + return local.GetCertificate(hi) +} + +// CertPair is an alias for [tailscale.com/client/local.CertPair]. +// +// Deprecated: import [tailscale.com/client/local] instead and use [local.Client.CertPair]. +func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { + return local.CertPair(ctx, domain) +} + +// ExpandSNIName is an alias for [tailscale.com/client/local.ExpandSNIName]. +// +// Deprecated: import [tailscale.com/client/local] instead and use [local.Client.ExpandSNIName]. +func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) { + return local.ExpandSNIName(ctx, name) +} diff --git a/client/tailscale/devices.go b/client/tailscale/devices.go index 0664f9e63edb1..2b2cf7a0cd049 100644 --- a/client/tailscale/devices.go +++ b/client/tailscale/devices.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 diff --git a/client/tailscale/dns.go b/client/tailscale/dns.go index bbdc7c56c65f7..427caea0fc593 100644 --- a/client/tailscale/dns.go +++ b/client/tailscale/dns.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 diff --git a/client/tailscale/example/servetls/servetls.go b/client/tailscale/example/servetls/servetls.go index 0ade420887634..864dafd07b242 100644 --- a/client/tailscale/example/servetls/servetls.go +++ b/client/tailscale/example/servetls/servetls.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The servetls program shows how to run an HTTPS server diff --git a/client/tailscale/keys.go b/client/tailscale/keys.go index 79e19e99880f7..6edbae034a759 100644 --- a/client/tailscale/keys.go +++ b/client/tailscale/keys.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package tailscale diff --git a/client/tailscale/localclient_aliases.go b/client/tailscale/localclient_aliases.go index 2b53906b71ae4..98a72068a5eba 100644 --- a/client/tailscale/localclient_aliases.go +++ b/client/tailscale/localclient_aliases.go @@ -1,11 +1,10 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package tailscale import ( "context" - "crypto/tls" "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" @@ -32,23 +31,11 @@ type IPNBusWatcher = local.IPNBusWatcher // Deprecated: import [tailscale.com/client/local] instead. type BugReportOpts = local.BugReportOpts -// DebugPortmapOpts is an alias for [tailscale.com/client/local.DebugPortmapOpts]. -// -// Deprecated: import [tailscale.com/client/local] instead. -type DebugPortmapOpts = local.DebugPortmapOpts - // PingOpts is an alias for [tailscale.com/client/local.PingOpts]. // // Deprecated: import [tailscale.com/client/local] instead. type PingOpts = local.PingOpts -// GetCertificate is an alias for [tailscale.com/client/local.GetCertificate]. -// -// Deprecated: import [tailscale.com/client/local] instead and use [local.Client.GetCertificate]. -func GetCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { - return local.GetCertificate(hi) -} - // SetVersionMismatchHandler is an alias for [tailscale.com/client/local.SetVersionMismatchHandler]. // // Deprecated: import [tailscale.com/client/local] instead. @@ -90,17 +77,3 @@ func Status(ctx context.Context) (*ipnstate.Status, error) { func StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { return local.StatusWithoutPeers(ctx) } - -// CertPair is an alias for [tailscale.com/client/local.CertPair]. -// -// Deprecated: import [tailscale.com/client/local] instead and use [local.Client.CertPair]. -func CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { - return local.CertPair(ctx, domain) -} - -// ExpandSNIName is an alias for [tailscale.com/client/local.ExpandSNIName]. -// -// Deprecated: import [tailscale.com/client/local] instead and use [local.Client.ExpandSNIName]. -func ExpandSNIName(ctx context.Context, name string) (fqdn string, ok bool) { - return local.ExpandSNIName(ctx, name) -} diff --git a/client/tailscale/required_version.go b/client/tailscale/required_version.go index d6bca1c6d8ff9..fb994e55fb604 100644 --- a/client/tailscale/required_version.go +++ b/client/tailscale/required_version.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !go1.23 diff --git a/client/tailscale/routes.go b/client/tailscale/routes.go index b72f2743ff9fb..aa6e49e3b7fc2 100644 --- a/client/tailscale/routes.go +++ b/client/tailscale/routes.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 diff --git a/client/tailscale/tailnet.go b/client/tailscale/tailnet.go index 9453962c908c8..75ca7dfd60a33 100644 --- a/client/tailscale/tailnet.go +++ b/client/tailscale/tailnet.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 diff --git a/client/tailscale/tailscale.go b/client/tailscale/tailscale.go index 76e44454b2fc2..d5585a052bb99 100644 --- a/client/tailscale/tailscale.go +++ b/client/tailscale/tailscale.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 diff --git a/client/tailscale/tailscale_test.go b/client/tailscale/tailscale_test.go index 67379293bd580..342a2d7872026 100644 --- a/client/tailscale/tailscale_test.go +++ b/client/tailscale/tailscale_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package tailscale @@ -31,7 +31,7 @@ func TestClientBuildURL(t *testing.T) { want: `http://127.0.0.1:1234/api/v2/tailnet/example%20dot%20com%3Ffoo=bar`, }, { - desc: "url.Values", + desc: "url-Values", elements: []any{"tailnet", "example.com", "acl", url.Values{"details": {"1"}}}, want: `http://127.0.0.1:1234/api/v2/tailnet/example.com/acl?details=1`, }, @@ -71,7 +71,7 @@ func TestClientBuildTailnetURL(t *testing.T) { want: `http://127.0.0.1:1234/api/v2/tailnet/example.com/foo%20bar%3Fbaz=qux`, }, { - desc: "url.Values", + desc: "url-Values", elements: []any{"acl", url.Values{"details": {"1"}}}, want: `http://127.0.0.1:1234/api/v2/tailnet/example.com/acl?details=1`, }, diff --git a/client/web/assets.go b/client/web/assets.go index c4f4e9e3bcf66..b9e4226299dd1 100644 --- a/client/web/assets.go +++ b/client/web/assets.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package web diff --git a/client/web/auth.go b/client/web/auth.go index 8b195a417f415..916f24782d55a 100644 --- a/client/web/auth.go +++ b/client/web/auth.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package web @@ -37,6 +37,7 @@ type browserSession struct { AuthURL string // from tailcfg.WebClientAuthResponse Created time.Time Authenticated bool + PendingAuth bool } // isAuthorized reports true if the given session is authorized @@ -172,12 +173,14 @@ func (s *Server) newSession(ctx context.Context, src *apitype.WhoIsResponse) (*b } session.AuthID = a.ID session.AuthURL = a.URL + session.PendingAuth = true } else { // control does not support check mode, so there is no additional auth we can do. session.Authenticated = true } s.browserSessions.Store(sid, session) + return session, nil } @@ -192,7 +195,7 @@ func (s *Server) controlSupportsCheckMode(ctx context.Context) bool { if err != nil { return true } - controlURL, err := url.Parse(prefs.ControlURLOrDefault()) + controlURL, err := url.Parse(prefs.ControlURLOrDefault(s.polc)) if err != nil { return true } @@ -206,16 +209,24 @@ func (s *Server) awaitUserAuth(ctx context.Context, session *browserSession) err if session.isAuthorized(s.timeNow()) { return nil // already authorized } + a, err := s.waitAuthURL(ctx, session.AuthID, session.SrcNode) if err != nil { - // Clean up the session. Doing this on any error from control - // server to avoid the user getting stuck with a bad session - // cookie. + // Don't delete the session on context cancellation, as this is expected + // when users navigate away or refresh the page. + if errors.Is(err, context.Canceled) { + return err + } + + // Clean up the session for non-cancellation errors from control server + // to avoid the user getting stuck with a bad session cookie. s.browserSessions.Delete(session.ID) return err } + if a.Complete { session.Authenticated = a.Complete + session.PendingAuth = false s.browserSessions.Store(session.ID, session) } return nil diff --git a/client/web/package.json b/client/web/package.json index c45f7d6a867ec..c733b0de06b97 100644 --- a/client/web/package.json +++ b/client/web/package.json @@ -34,10 +34,10 @@ "prettier-plugin-organize-imports": "^3.2.2", "tailwindcss": "^3.3.3", "typescript": "^5.3.3", - "vite": "^5.1.7", + "vite": "^5.4.21", "vite-plugin-svgr": "^4.2.0", "vite-tsconfig-paths": "^3.5.0", - "vitest": "^1.3.1" + "vitest": "^1.6.1" }, "resolutions": { "@typescript-eslint/eslint-plugin": "^6.2.1", diff --git a/client/web/qnap.go b/client/web/qnap.go index 9bde64bf5885b..132b95aed086d 100644 --- a/client/web/qnap.go +++ b/client/web/qnap.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // qnap.go contains handlers and logic, such as authentication, diff --git a/client/web/src/api.ts b/client/web/src/api.ts index e780c76459dfd..ea64742cdd339 100644 --- a/client/web/src/api.ts +++ b/client/web/src/api.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { useCallback } from "react" @@ -123,7 +123,10 @@ export function useAPI() { return apiFetch<{ url?: string }>("/up", "POST", t.data) .then((d) => d.url && window.open(d.url, "_blank")) // "up" login step .then(() => incrementMetric("web_client_node_connect")) - .then(() => mutate("/data")) + .then(() => { + mutate("/data") + mutate("/auth") + }) .catch(handlePostError("Failed to login")) /** @@ -134,9 +137,9 @@ export function useAPI() { // For logout, must increment metric before running api call, // as tailscaled will be unreachable after the call completes. incrementMetric("web_client_node_disconnect") - return apiFetch("/local/v0/logout", "POST").catch( - handlePostError("Failed to logout") - ) + return apiFetch("/local/v0/logout", "POST") + .then(() => mutate("/auth")) + .catch(handlePostError("Failed to logout")) /** * "new-auth-session" handles creating a new check mode session to diff --git a/client/web/src/components/acl-tag.tsx b/client/web/src/components/acl-tag.tsx index cb34375ed293c..95ab764c4a56d 100644 --- a/client/web/src/components/acl-tag.tsx +++ b/client/web/src/components/acl-tag.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/address-copy-card.tsx b/client/web/src/components/address-copy-card.tsx index 6b4f25bed73f4..697086f15c58d 100644 --- a/client/web/src/components/address-copy-card.tsx +++ b/client/web/src/components/address-copy-card.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import * as Primitive from "@radix-ui/react-popover" diff --git a/client/web/src/components/app.tsx b/client/web/src/components/app.tsx index 981dd8889c4b2..b885125b7f278 100644 --- a/client/web/src/components/app.tsx +++ b/client/web/src/components/app.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import React from "react" diff --git a/client/web/src/components/control-components.tsx b/client/web/src/components/control-components.tsx index ffb0a2999558f..42ed25107c986 100644 --- a/client/web/src/components/control-components.tsx +++ b/client/web/src/components/control-components.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import React from "react" diff --git a/client/web/src/components/exit-node-selector.tsx b/client/web/src/components/exit-node-selector.tsx index c0fd5e730b04c..a564ebbfc56b1 100644 --- a/client/web/src/components/exit-node-selector.tsx +++ b/client/web/src/components/exit-node-selector.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/login-toggle.tsx b/client/web/src/components/login-toggle.tsx index f5c4efe3ce2ac..397cb2ee1f8f1 100644 --- a/client/web/src/components/login-toggle.tsx +++ b/client/web/src/components/login-toggle.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/nice-ip.tsx b/client/web/src/components/nice-ip.tsx index f00d763f96db9..1f90d1cd73802 100644 --- a/client/web/src/components/nice-ip.tsx +++ b/client/web/src/components/nice-ip.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/update-available.tsx b/client/web/src/components/update-available.tsx index 763007de889c7..9d678d9073aa7 100644 --- a/client/web/src/components/update-available.tsx +++ b/client/web/src/components/update-available.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import React from "react" diff --git a/client/web/src/components/views/device-details-view.tsx b/client/web/src/components/views/device-details-view.tsx index fa58e52aea473..e24aacf520743 100644 --- a/client/web/src/components/views/device-details-view.tsx +++ b/client/web/src/components/views/device-details-view.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/views/disconnected-view.tsx b/client/web/src/components/views/disconnected-view.tsx index 492c69e469ef1..5ec86aae4643b 100644 --- a/client/web/src/components/views/disconnected-view.tsx +++ b/client/web/src/components/views/disconnected-view.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import React from "react" diff --git a/client/web/src/components/views/home-view.tsx b/client/web/src/components/views/home-view.tsx index 8073823466b34..e9051f22ba1cd 100644 --- a/client/web/src/components/views/home-view.tsx +++ b/client/web/src/components/views/home-view.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/views/login-view.tsx b/client/web/src/components/views/login-view.tsx index f8c15b16dbcaa..a6c4a9ae2c7ab 100644 --- a/client/web/src/components/views/login-view.tsx +++ b/client/web/src/components/views/login-view.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import React from "react" diff --git a/client/web/src/components/views/ssh-view.tsx b/client/web/src/components/views/ssh-view.tsx index 1337b9fdd10fd..67c324fa53563 100644 --- a/client/web/src/components/views/ssh-view.tsx +++ b/client/web/src/components/views/ssh-view.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/views/subnet-router-view.tsx b/client/web/src/components/views/subnet-router-view.tsx index 26369112c1cbf..7f4c682996033 100644 --- a/client/web/src/components/views/subnet-router-view.tsx +++ b/client/web/src/components/views/subnet-router-view.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/components/views/updating-view.tsx b/client/web/src/components/views/updating-view.tsx index 0c844c7d09faa..d39dc5c63fd27 100644 --- a/client/web/src/components/views/updating-view.tsx +++ b/client/web/src/components/views/updating-view.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import React from "react" diff --git a/client/web/src/hooks/auth.ts b/client/web/src/hooks/auth.ts index 51eb0c400bae9..c676647ca0b7e 100644 --- a/client/web/src/hooks/auth.ts +++ b/client/web/src/hooks/auth.ts @@ -1,8 +1,9 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { useCallback, useEffect, useState } from "react" import { apiFetch, setSynoToken } from "src/api" +import useSWR from "swr" export type AuthResponse = { serverMode: AuthServerMode @@ -49,33 +50,26 @@ export function hasAnyEditCapabilities(auth: AuthResponse): boolean { * useAuth reports and refreshes Tailscale auth status for the web client. */ export default function useAuth() { - const [data, setData] = useState() - const [loading, setLoading] = useState(true) + const { data, error, mutate } = useSWR("/auth") const [ranSynoAuth, setRanSynoAuth] = useState(false) - const loadAuth = useCallback(() => { - setLoading(true) - return apiFetch("/auth", "GET") - .then((d) => { - setData(d) - if (d.needsSynoAuth) { - fetch("/webman/login.cgi") - .then((r) => r.json()) - .then((a) => { - setSynoToken(a.SynoToken) - setRanSynoAuth(true) - setLoading(false) - }) - } else { - setLoading(false) - } - return d - }) - .catch((error) => { - setLoading(false) - console.error(error) - }) - }, []) + const loading = !data && !error + + // Start Synology auth flow if needed. + useEffect(() => { + if (data?.needsSynoAuth && !ranSynoAuth) { + fetch("/webman/login.cgi") + .then((r) => r.json()) + .then((a) => { + setSynoToken(a.SynoToken) + setRanSynoAuth(true) + mutate() + }) + .catch((error) => { + console.error("Synology auth error:", error) + }) + } + }, [data?.needsSynoAuth, ranSynoAuth, mutate]) const newSession = useCallback(() => { return apiFetch<{ authUrl?: string }>("/auth/session/new", "GET") @@ -86,34 +80,26 @@ export default function useAuth() { } }) .then(() => { - loadAuth() + mutate() }) .catch((error) => { console.error(error) }) - }, [loadAuth]) + }, [mutate]) + // Start regular auth flow. useEffect(() => { - loadAuth().then((d) => { - if (!d) { - return - } - if ( - !d.authorized && - hasAnyEditCapabilities(d) && - // Start auth flow immediately if browser has requested it. - new URLSearchParams(window.location.search).get("check") === "now" - ) { - newSession() - } - }) - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) + const needsAuth = + data && + !loading && + !data.authorized && + hasAnyEditCapabilities(data) && + new URLSearchParams(window.location.search).get("check") === "now" - useEffect(() => { - loadAuth() // Refresh auth state after syno auth runs - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ranSynoAuth]) + if (needsAuth) { + newSession() + } + }, [data, loading, newSession]) return { data, diff --git a/client/web/src/hooks/exit-nodes.ts b/client/web/src/hooks/exit-nodes.ts index b3ce0a9fa12ec..78f8a383dbb00 100644 --- a/client/web/src/hooks/exit-nodes.ts +++ b/client/web/src/hooks/exit-nodes.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { useMemo } from "react" @@ -66,7 +66,7 @@ export default function useExitNodes(node: NodeData, filter?: string) { // match from a list of exit node `options` to `nodes`. const addBestMatchNode = ( options: ExitNode[], - name: (l: ExitNodeLocation) => string + name: (loc: ExitNodeLocation) => string ) => { const bestNode = highestPriorityNode(options) if (!bestNode || !bestNode.Location) { @@ -86,7 +86,7 @@ export default function useExitNodes(node: NodeData, filter?: string) { locationNodesMap.forEach( // add one node per country (countryNodes) => - addBestMatchNode(flattenMap(countryNodes), (l) => l.Country) + addBestMatchNode(flattenMap(countryNodes), (loc) => loc.Country) ) } else { // Otherwise, show the best match on a city-level, @@ -97,12 +97,12 @@ export default function useExitNodes(node: NodeData, filter?: string) { countryNodes.forEach( // add one node per city (cityNodes) => - addBestMatchNode(cityNodes, (l) => `${l.Country}: ${l.City}`) + addBestMatchNode(cityNodes, (loc) => `${loc.Country}: ${loc.City}`) ) // add the "Country: Best Match" node addBestMatchNode( flattenMap(countryNodes), - (l) => `${l.Country}: Best Match` + (loc) => `${loc.Country}: Best Match` ) }) } diff --git a/client/web/src/hooks/self-update.ts b/client/web/src/hooks/self-update.ts index eb10463c1abe1..e63d6eddaeebf 100644 --- a/client/web/src/hooks/self-update.ts +++ b/client/web/src/hooks/self-update.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { useCallback, useEffect, useState } from "react" diff --git a/client/web/src/hooks/toaster.ts b/client/web/src/hooks/toaster.ts index 41fb4f42d0918..8c30cab58a6a5 100644 --- a/client/web/src/hooks/toaster.ts +++ b/client/web/src/hooks/toaster.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { useRawToasterForHook } from "src/ui/toaster" diff --git a/client/web/src/hooks/ts-web-connected.ts b/client/web/src/hooks/ts-web-connected.ts index 3145663d7654d..bd020c9e9b595 100644 --- a/client/web/src/hooks/ts-web-connected.ts +++ b/client/web/src/hooks/ts-web-connected.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { useCallback, useEffect, useState } from "react" diff --git a/client/web/src/index.tsx b/client/web/src/index.tsx index 31ac7890f45f2..2b970ebca8ed7 100644 --- a/client/web/src/index.tsx +++ b/client/web/src/index.tsx @@ -1,10 +1,10 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Preserved js license comment for web client app. /** * @license - * Copyright (c) Tailscale Inc & AUTHORS + * Copyright (c) Tailscale Inc & contributors * SPDX-License-Identifier: BSD-3-Clause */ diff --git a/client/web/src/types.ts b/client/web/src/types.ts index 62fa4c59f1fbf..ebf11d442fc52 100644 --- a/client/web/src/types.ts +++ b/client/web/src/types.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { assertNever } from "src/utils/util" diff --git a/client/web/src/ui/badge.tsx b/client/web/src/ui/badge.tsx index c0caa6403b37e..de8c21e3568ab 100644 --- a/client/web/src/ui/badge.tsx +++ b/client/web/src/ui/badge.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/button.tsx b/client/web/src/ui/button.tsx index 18dc2939f1889..e38f58f02bd2e 100644 --- a/client/web/src/ui/button.tsx +++ b/client/web/src/ui/button.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/card.tsx b/client/web/src/ui/card.tsx index 4e17c3df6d29e..7d3c9b89e8202 100644 --- a/client/web/src/ui/card.tsx +++ b/client/web/src/ui/card.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/collapsible.tsx b/client/web/src/ui/collapsible.tsx index 6aa8c0b9f5ca1..bd0b0eedad84a 100644 --- a/client/web/src/ui/collapsible.tsx +++ b/client/web/src/ui/collapsible.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import * as Primitive from "@radix-ui/react-collapsible" diff --git a/client/web/src/ui/dialog.tsx b/client/web/src/ui/dialog.tsx index d5af834ce05d2..6b3bb792b8565 100644 --- a/client/web/src/ui/dialog.tsx +++ b/client/web/src/ui/dialog.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import * as DialogPrimitive from "@radix-ui/react-dialog" diff --git a/client/web/src/ui/empty-state.tsx b/client/web/src/ui/empty-state.tsx index 6ac7fd4fa87e6..3964a55590ab4 100644 --- a/client/web/src/ui/empty-state.tsx +++ b/client/web/src/ui/empty-state.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/input.tsx b/client/web/src/ui/input.tsx index 756e0fc2ea4d6..7cff6bf5bf074 100644 --- a/client/web/src/ui/input.tsx +++ b/client/web/src/ui/input.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/loading-dots.tsx b/client/web/src/ui/loading-dots.tsx index 6b47552a95844..83c60da93a937 100644 --- a/client/web/src/ui/loading-dots.tsx +++ b/client/web/src/ui/loading-dots.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/popover.tsx b/client/web/src/ui/popover.tsx index c0f01c833f465..0139894bb5e4a 100644 --- a/client/web/src/ui/popover.tsx +++ b/client/web/src/ui/popover.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import * as PopoverPrimitive from "@radix-ui/react-popover" diff --git a/client/web/src/ui/portal-container-context.tsx b/client/web/src/ui/portal-container-context.tsx index d25b30bae1731..922cd0d14ea52 100644 --- a/client/web/src/ui/portal-container-context.tsx +++ b/client/web/src/ui/portal-container-context.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import React from "react" diff --git a/client/web/src/ui/profile-pic.tsx b/client/web/src/ui/profile-pic.tsx index 343fb29b490e0..4bbdad878ab4a 100644 --- a/client/web/src/ui/profile-pic.tsx +++ b/client/web/src/ui/profile-pic.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/quick-copy.tsx b/client/web/src/ui/quick-copy.tsx index bc8f916c84144..0c51f820ccf33 100644 --- a/client/web/src/ui/quick-copy.tsx +++ b/client/web/src/ui/quick-copy.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/search-input.tsx b/client/web/src/ui/search-input.tsx index debba371caec6..9b99984acc014 100644 --- a/client/web/src/ui/search-input.tsx +++ b/client/web/src/ui/search-input.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/spinner.tsx b/client/web/src/ui/spinner.tsx index 51f6e887b836d..be3dc8d5b4fa5 100644 --- a/client/web/src/ui/spinner.tsx +++ b/client/web/src/ui/spinner.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/toaster.tsx b/client/web/src/ui/toaster.tsx index 18f491f3b2552..677ccde4d5d9b 100644 --- a/client/web/src/ui/toaster.tsx +++ b/client/web/src/ui/toaster.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/ui/toggle.tsx b/client/web/src/ui/toggle.tsx index 4922830058f16..83ca92608a4dc 100644 --- a/client/web/src/ui/toggle.tsx +++ b/client/web/src/ui/toggle.tsx @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import cx from "classnames" diff --git a/client/web/src/utils/clipboard.ts b/client/web/src/utils/clipboard.ts index f003bc24079ab..3ca5f281ebb93 100644 --- a/client/web/src/utils/clipboard.ts +++ b/client/web/src/utils/clipboard.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { isPromise } from "src/utils/util" diff --git a/client/web/src/utils/util.test.ts b/client/web/src/utils/util.test.ts index 148f6cc365589..2a598d6505654 100644 --- a/client/web/src/utils/util.test.ts +++ b/client/web/src/utils/util.test.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause import { isTailscaleIPv6, pluralize } from "src/utils/util" diff --git a/client/web/src/utils/util.ts b/client/web/src/utils/util.ts index 5f8eda7b77572..81fc904034c08 100644 --- a/client/web/src/utils/util.ts +++ b/client/web/src/utils/util.ts @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause /** diff --git a/client/web/synology.go b/client/web/synology.go index 922489d78af16..e39cbc9c5c82e 100644 --- a/client/web/synology.go +++ b/client/web/synology.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // synology.go contains handlers and logic, such as authentication, diff --git a/client/web/web.go b/client/web/web.go index f3158cd1f6ff5..95259ef1a9039 100644 --- a/client/web/web.go +++ b/client/web/web.go @@ -1,10 +1,11 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package web provides the Tailscale client for web. package web import ( + "cmp" "context" "encoding/json" "errors" @@ -23,9 +24,10 @@ import ( "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" - "tailscale.com/clientupdate" "tailscale.com/envknob" "tailscale.com/envknob/featureknob" + "tailscale.com/feature" + "tailscale.com/feature/buildfeatures" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" @@ -33,9 +35,12 @@ import ( "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" "tailscale.com/tailcfg" + "tailscale.com/tsweb" "tailscale.com/types/logger" "tailscale.com/types/views" + "tailscale.com/util/ctxkey" "tailscale.com/util/httpm" + "tailscale.com/util/syspolicy/policyclient" "tailscale.com/version" "tailscale.com/version/distro" ) @@ -49,6 +54,7 @@ type Server struct { mode ServerMode logf logger.Logf + polc policyclient.Client // must be non-nil lc *local.Client timeNow func() time.Time @@ -139,9 +145,13 @@ type ServerOpts struct { TimeNow func() time.Time // Logf optionally provides a logger function. - // log.Printf is used as default. + // If nil, log.Printf is used as default. Logf logger.Logf + // PolicyClient, if non-nil, will be used to fetch policy settings. + // If nil, the default policy client will be used. + PolicyClient policyclient.Client + // The following two fields are required and used exclusively // in ManageServerMode to facilitate the control server login // check step for authorizing browser sessions. @@ -178,6 +188,7 @@ func NewServer(opts ServerOpts) (s *Server, err error) { } s = &Server{ mode: opts.Mode, + polc: cmp.Or(opts.PolicyClient, policyclient.Get()), logf: opts.Logf, devMode: envknob.Bool("TS_DEBUG_WEB_CLIENT_DEV"), lc: opts.LocalClient, @@ -488,6 +499,10 @@ func (s *Server) authorizeRequest(w http.ResponseWriter, r *http.Request) (ok bo // Client using system-specific auth. switch distro.Get() { case distro.Synology: + if !buildfeatures.HasSynology { + // Synology support not built in. + return false + } authorized, _ := authorizeSynology(r) return authorized case distro.QNAP: @@ -514,45 +529,40 @@ func (s *Server) serveLoginAPI(w http.ResponseWriter, r *http.Request) { } } -type apiHandler[data any] struct { - s *Server - w http.ResponseWriter - r *http.Request - - // permissionCheck allows for defining whether a requesting peer's - // capabilities grant them access to make the given data update. - // If permissionCheck reports false, the request fails as unauthorized. - permissionCheck func(data data, peer peerCapabilities) bool -} - -// newHandler constructs a new api handler which restricts the given request -// to the specified permission check. If the permission check fails for -// the peer associated with the request, an unauthorized error is returned -// to the client. -func newHandler[data any](s *Server, w http.ResponseWriter, r *http.Request, permissionCheck func(data data, peer peerCapabilities) bool) *apiHandler[data] { - return &apiHandler[data]{ - s: s, - w: w, - r: r, - permissionCheck: permissionCheck, +// handleJSON manages decoding the request's body JSON as data and passing it +// on to the provided handler function. +func handleJSON[data any](h func(ctx context.Context, data data) error) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var body data + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := h(r.Context(), body); err != nil { + if httpErr, ok := errors.AsType[tsweb.HTTPError](err); ok { + tsweb.WriteHTTPError(w, r, httpErr) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + w.WriteHeader(http.StatusOK) } } -// alwaysAllowed can be passed as the permissionCheck argument to newHandler -// for requests that are always allowed to complete regardless of a peer's -// capabilities. -func alwaysAllowed[data any](_ data, _ peerCapabilities) bool { return true } +var contextKeyPeer = ctxkey.New("peer-capabilities", peerCapabilities{}) -func (a *apiHandler[data]) getPeer() (peerCapabilities, error) { +func (s *Server) setPeer(r *http.Request) (*http.Request, error) { // TODO(tailscale/corp#16695,sonia): We also call StatusWithoutPeers and // WhoIs when originally checking for a session from authorizeRequest. // Would be nice if we could pipe those through to here so we don't end // up having to re-call them to grab the peer capabilities. - status, err := a.s.lc.StatusWithoutPeers(a.r.Context()) + status, err := s.lc.StatusWithoutPeers(r.Context()) if err != nil { return nil, err } - whois, err := a.s.lc.WhoIs(a.r.Context(), a.r.RemoteAddr) + whois, err := s.lc.WhoIs(r.Context(), r.RemoteAddr) if err != nil { return nil, err } @@ -560,56 +570,11 @@ func (a *apiHandler[data]) getPeer() (peerCapabilities, error) { if err != nil { return nil, err } - return peer, nil + return r.WithContext(contextKeyPeer.WithValue(r.Context(), peer)), nil } -type noBodyData any // empty type, for use from serveAPI for endpoints with empty body - -// handle runs the given handler if the source peer satisfies the -// constraints for running this request. -// -// handle is expected for use when `data` type is empty, or set to -// `noBodyData` in practice. For requests that expect JSON body data -// to be attached, use handleJSON instead. -func (a *apiHandler[data]) handle(h http.HandlerFunc) { - peer, err := a.getPeer() - if err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - var body data // not used - if !a.permissionCheck(body, peer) { - http.Error(a.w, "not allowed", http.StatusUnauthorized) - return - } - h(a.w, a.r) -} - -// handleJSON manages decoding the request's body JSON and passing -// it on to the provided function if the source peer satisfies the -// constraints for running this request. -func (a *apiHandler[data]) handleJSON(h func(ctx context.Context, data data) error) { - defer a.r.Body.Close() - var body data - if err := json.NewDecoder(a.r.Body).Decode(&body); err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - peer, err := a.getPeer() - if err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - if !a.permissionCheck(body, peer) { - http.Error(a.w, "not allowed", http.StatusUnauthorized) - return - } - - if err := h(a.r.Context(), body); err != nil { - http.Error(a.w, err.Error(), http.StatusInternalServerError) - return - } - a.w.WriteHeader(http.StatusOK) +func (s *Server) getPeer(ctx context.Context) peerCapabilities { + return contextKeyPeer.Value(ctx) } // serveAPI serves requests for the web client api. @@ -624,67 +589,44 @@ func (s *Server) serveAPI(w http.ResponseWriter, r *http.Request) { } } + var err error + r, err = s.setPeer(r) + if err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + path := strings.TrimPrefix(r.URL.Path, "/api") switch { case path == "/data" && r.Method == httpm.GET: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.serveGetNodeData) + s.serveGetNodeData(w, r) return case path == "/exit-nodes" && r.Method == httpm.GET: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.serveGetExitNodes) + s.serveGetExitNodes(w, r) return case path == "/routes" && r.Method == httpm.POST: - peerAllowed := func(d postRoutesRequest, p peerCapabilities) bool { - if d.SetExitNode && !p.canEdit(capFeatureExitNodes) { - return false - } else if d.SetRoutes && !p.canEdit(capFeatureSubnets) { - return false - } - return true - } - newHandler[postRoutesRequest](s, w, r, peerAllowed). - handleJSON(s.servePostRoutes) + handleJSON[postRoutesRequest](s.servePostRoutes)(w, r) return case path == "/device-details-click" && r.Method == httpm.POST: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.serveDeviceDetailsClick) + s.serveDeviceDetailsClick(w, r) return case path == "/local/v0/logout" && r.Method == httpm.POST: - peerAllowed := func(_ noBodyData, peer peerCapabilities) bool { - return peer.canEdit(capFeatureAccount) - } - newHandler[noBodyData](s, w, r, peerAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/prefs" && r.Method == httpm.PATCH: - peerAllowed := func(data maskedPrefs, peer peerCapabilities) bool { - if data.RunSSHSet && !peer.canEdit(capFeatureSSH) { - return false - } - return true - } - newHandler[maskedPrefs](s, w, r, peerAllowed). - handleJSON(s.serveUpdatePrefs) + handleJSON[maskedPrefs](s.serveUpdatePrefs)(w, r) return case path == "/local/v0/update/check" && r.Method == httpm.GET: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/update/check" && r.Method == httpm.POST: - peerAllowed := func(_ noBodyData, peer peerCapabilities) bool { - return peer.canEdit(capFeatureAccount) - } - newHandler[noBodyData](s, w, r, peerAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/update/progress" && r.Method == httpm.POST: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return case path == "/local/v0/upload-client-metrics" && r.Method == httpm.POST: - newHandler[noBodyData](s, w, r, alwaysAllowed). - handle(s.proxyRequestToLocalAPI) + s.proxyRequestToLocalAPI(w, r) return } http.Error(w, "invalid endpoint", http.StatusNotFound) @@ -758,6 +700,19 @@ func (s *Server) serveAPIAuth(w http.ResponseWriter, r *http.Request) { } } + // We might have a session for which we haven't awaited the result yet. + // This can happen when the AuthURL opens in the same browser tab instead + // of a new one due to browser settings. + // (See https://github.com/tailscale/tailscale/issues/11905) + // We therefore set a PendingAuth flag when creating a new session, check + // it here and call awaitUserAuth if we find it to be true. Once the auth + // wait completes, awaitUserAuth will set PendingAuth to false. + if sErr == nil && session.PendingAuth == true { + if err := s.awaitUserAuth(r.Context(), session); err != nil { + sErr = err + } + } + switch { case sErr != nil && errors.Is(sErr, errNotUsingTailscale): s.lc.IncrementCounter(r.Context(), "web_client_viewing_local", 1) @@ -950,7 +905,7 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) { UnraidToken: os.Getenv("UNRAID_CSRF_TOKEN"), RunningSSHServer: prefs.RunSSH, URLPrefix: strings.TrimSuffix(s.pathPrefix, "/"), - ControlAdminURL: prefs.AdminPageURL(), + ControlAdminURL: prefs.AdminPageURL(s.polc), LicensesURL: licenses.LicensesURL(), Features: availableFeatures(), @@ -970,9 +925,18 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) { data.ClientVersion = cv } - if st.CurrentTailnet != nil { - data.TailnetName = st.CurrentTailnet.MagicDNSSuffix - data.DomainName = st.CurrentTailnet.Name + profile, _, err := s.lc.ProfileStatus(r.Context()) + if err != nil { + s.logf("error fetching profiles: %v", err) + // If for some reason we can't fetch profiles, + // continue to use st.CurrentTailnet if set. + if st.CurrentTailnet != nil { + data.TailnetName = st.CurrentTailnet.MagicDNSSuffix + data.DomainName = st.CurrentTailnet.Name + } + } else { + data.TailnetName = profile.NetworkProfile.MagicDNSName + data.DomainName = profile.NetworkProfile.DisplayNameOrDefault() } if st.Self.Tags != nil { data.Tags = st.Self.Tags.AsSlice() @@ -1032,7 +996,7 @@ func availableFeatures() map[string]bool { "advertise-routes": true, // available on all platforms "use-exit-node": featureknob.CanUseExitNode() == nil, "ssh": featureknob.CanRunTailscaleSSH() == nil, - "auto-update": version.IsUnstableBuild() && clientupdate.CanAutoUpdate(), + "auto-update": version.IsUnstableBuild() && feature.CanAutoUpdate(), } return features } @@ -1087,6 +1051,11 @@ type maskedPrefs struct { } func (s *Server) serveUpdatePrefs(ctx context.Context, prefs maskedPrefs) error { + peer := s.getPeer(ctx) + if prefs.RunSSHSet && !peer.canEdit(capFeatureSSH) { + return tsweb.Error(http.StatusUnauthorized, "RunSSHSet not allowed", nil) + } + _, err := s.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ RunSSHSet: prefs.RunSSHSet, Prefs: ipn.Prefs{ @@ -1105,6 +1074,17 @@ type postRoutesRequest struct { } func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) error { + if !data.SetExitNode && !data.SetRoutes { + return tsweb.Error(http.StatusBadRequest, "must specify SetExitNode or SetRoutes", nil) + } + peer := s.getPeer(ctx) + if data.SetExitNode && !peer.canEdit(capFeatureExitNodes) { + return tsweb.Error(http.StatusUnauthorized, "SetExitNode not allowed", nil) + } + if data.SetRoutes && !peer.canEdit(capFeatureSubnets) { + return tsweb.Error(http.StatusUnauthorized, "SetRoutes not allowed", nil) + } + prefs, err := s.lc.GetPrefs(ctx) if err != nil { return err @@ -1118,13 +1098,14 @@ func (s *Server) servePostRoutes(ctx context.Context, data postRoutesRequest) er } currNonExitRoutes = append(currNonExitRoutes, r.String()) } - // Set non-edited fields to their current values. - if data.SetExitNode { - data.AdvertiseRoutes = currNonExitRoutes - } else if data.SetRoutes { + // For each group of fields not being set, preserve the current prefs. + if !data.SetExitNode { data.AdvertiseExitNode = currAdvertisingExitNode data.UseExitNode = prefs.ExitNodeID } + if !data.SetRoutes { + data.AdvertiseRoutes = currNonExitRoutes + } // Calculate routes. routesStr := strings.Join(data.AdvertiseRoutes, ",") @@ -1301,6 +1282,19 @@ func (s *Server) proxyRequestToLocalAPI(w http.ResponseWriter, r *http.Request) return } + switch path { + case "/v0/logout": + if !s.getPeer(r.Context()).canEdit(capFeatureAccount) { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + case "/v0/update/check": + if r.Method == httpm.POST && !s.getPeer(r.Context()).canEdit(capFeatureAccount) { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + } + localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi" + path req, err := http.NewRequestWithContext(r.Context(), r.Method, localAPIURL, r.Body) if err != nil { diff --git a/client/web/web_test.go b/client/web/web_test.go index 12dbb5c79b13a..51b6a8ac58781 100644 --- a/client/web/web_test.go +++ b/client/web/web_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package web @@ -28,6 +28,7 @@ import ( "tailscale.com/tailcfg" "tailscale.com/types/views" "tailscale.com/util/httpm" + "tailscale.com/util/syspolicy/policyclient" ) func TestQnapAuthnURL(t *testing.T) { @@ -40,37 +41,37 @@ func TestQnapAuthnURL(t *testing.T) { want string }{ { - name: "localhost http", + name: "localhost-http", in: "http://localhost:8088/", want: "http://localhost:8088/cgi-bin/authLogin.cgi?qtoken=token", }, { - name: "localhost https", + name: "localhost-https", in: "https://localhost:5000/", want: "https://localhost:5000/cgi-bin/authLogin.cgi?qtoken=token", }, { - name: "IP http", + name: "IP-http", in: "http://10.1.20.4:80/", want: "http://10.1.20.4:80/cgi-bin/authLogin.cgi?qtoken=token", }, { - name: "IP6 https", + name: "IP6-https", in: "https://[ff7d:0:1:2::1]/", want: "https://[ff7d:0:1:2::1]/cgi-bin/authLogin.cgi?qtoken=token", }, { - name: "hostname https", + name: "hostname-https", in: "https://qnap.example.com/", want: "https://qnap.example.com/cgi-bin/authLogin.cgi?qtoken=token", }, { - name: "invalid URL", + name: "invalid-URL", in: "This is not a URL, it is a really really really really really really really really really really really really long string to exercise the URL truncation code in the error path.", want: "http://localhost/cgi-bin/authLogin.cgi?qtoken=token", }, { - name: "err != nil", + name: "err-not-nil", in: "http://192.168.0.%31/", want: "http://localhost/cgi-bin/authLogin.cgi?qtoken=token", }, @@ -190,7 +191,7 @@ func TestServeAPI(t *testing.T) { reqBody: "{\"setExitNode\":true}", tests: []requestTest{{ remoteIP: remoteIPWithNoCapabilities, - wantResponse: "not allowed", + wantResponse: "SetExitNode not allowed", wantStatus: http.StatusUnauthorized, }, { remoteIP: remoteIPWithAllCapabilities, @@ -203,7 +204,7 @@ func TestServeAPI(t *testing.T) { reqContentType: "application/json", tests: []requestTest{{ remoteIP: remoteIPWithNoCapabilities, - wantResponse: "not allowed", + wantResponse: "RunSSHSet not allowed", wantStatus: http.StatusUnauthorized, }, { remoteIP: remoteIPWithAllCapabilities, @@ -576,16 +577,28 @@ func TestServeAuth(t *testing.T) { timeNow: func() time.Time { return timeNow }, newAuthURL: mockNewAuthURL, waitAuthURL: mockWaitAuthURL, + polc: policyclient.NoPolicyClient{}, } successCookie := "ts-cookie-success" s.browserSessions.Store(successCookie, &browserSession{ - ID: successCookie, - SrcNode: remoteNode.Node.ID, - SrcUser: user.ID, - Created: oneHourAgo, - AuthID: testAuthPathSuccess, - AuthURL: *testControlURL + testAuthPathSuccess, + ID: successCookie, + SrcNode: remoteNode.Node.ID, + SrcUser: user.ID, + Created: oneHourAgo, + AuthID: testAuthPathSuccess, + AuthURL: *testControlURL + testAuthPathSuccess, + PendingAuth: true, + }) + successCookie2 := "ts-cookie-success-2" + s.browserSessions.Store(successCookie2, &browserSession{ + ID: successCookie2, + SrcNode: remoteNode.Node.ID, + SrcUser: user.ID, + Created: oneHourAgo, + AuthID: testAuthPathSuccess, + AuthURL: *testControlURL + testAuthPathSuccess, + PendingAuth: true, }) failureCookie := "ts-cookie-failure" s.browserSessions.Store(failureCookie, &browserSession{ @@ -640,14 +653,15 @@ func TestServeAuth(t *testing.T) { AuthID: testAuthPath, AuthURL: *testControlURL + testAuthPath, Authenticated: false, + PendingAuth: true, }, }, { - name: "query-existing-incomplete-session", - path: "/api/auth", + name: "existing-session-used", + path: "/api/auth/session/new", // should not create new session cookie: successCookie, wantStatus: http.StatusOK, - wantResp: &authResponse{ViewerIdentity: vi, ServerMode: ManageServerMode}, + wantResp: &newSessionAuthResponse{AuthURL: *testControlURL + testAuthPathSuccess}, wantSession: &browserSession{ ID: successCookie, SrcNode: remoteNode.Node.ID, @@ -656,14 +670,15 @@ func TestServeAuth(t *testing.T) { AuthID: testAuthPathSuccess, AuthURL: *testControlURL + testAuthPathSuccess, Authenticated: false, + PendingAuth: true, }, }, { - name: "existing-session-used", - path: "/api/auth/session/new", // should not create new session + name: "transition-to-successful-session-via-api-auth-session-wait", + path: "/api/auth/session/wait", cookie: successCookie, wantStatus: http.StatusOK, - wantResp: &newSessionAuthResponse{AuthURL: *testControlURL + testAuthPathSuccess}, + wantResp: nil, wantSession: &browserSession{ ID: successCookie, SrcNode: remoteNode.Node.ID, @@ -671,17 +686,17 @@ func TestServeAuth(t *testing.T) { Created: oneHourAgo, AuthID: testAuthPathSuccess, AuthURL: *testControlURL + testAuthPathSuccess, - Authenticated: false, + Authenticated: true, }, }, { - name: "transition-to-successful-session", - path: "/api/auth/session/wait", - cookie: successCookie, + name: "transition-to-successful-session-via-api-auth", + path: "/api/auth", + cookie: successCookie2, wantStatus: http.StatusOK, - wantResp: nil, + wantResp: &authResponse{Authorized: true, ViewerIdentity: vi, ServerMode: ManageServerMode}, wantSession: &browserSession{ - ID: successCookie, + ID: successCookie2, SrcNode: remoteNode.Node.ID, SrcUser: user.ID, Created: oneHourAgo, @@ -729,6 +744,7 @@ func TestServeAuth(t *testing.T) { AuthID: testAuthPath, AuthURL: *testControlURL + testAuthPath, Authenticated: false, + PendingAuth: true, }, }, { @@ -746,6 +762,7 @@ func TestServeAuth(t *testing.T) { AuthID: testAuthPath, AuthURL: *testControlURL + testAuthPath, Authenticated: false, + PendingAuth: true, }, }, { @@ -1460,7 +1477,9 @@ func mockLocalAPI(t *testing.T, whoIs map[string]*apitype.WhoIsResponse, self fu http.Error(w, "invalid JSON body", http.StatusBadRequest) return } - metricCapture(metricNames[0].Name) + if metricCapture != nil && len(metricNames) > 0 { + metricCapture(metricNames[0].Name) + } writeJSON(w, struct{}{}) return case "/localapi/v0/logout": @@ -1499,47 +1518,47 @@ func TestCSRFProtect(t *testing.T) { wantError bool }{ { - name: "GET requests with no header are allowed", + name: "GET-no-header-allowed", // GET requests with no header are allowed method: "GET", }, { - name: "POST requests with same-origin are allowed", + name: "POST-same-origin-allowed", method: "POST", secFetchSite: "same-origin", }, { - name: "POST requests with cross-site are not allowed", + name: "POST-cross-site-rejected", method: "POST", secFetchSite: "cross-site", wantError: true, }, { - name: "POST requests with unknown sec-fetch-site values are not allowed", + name: "POST-unknown-sec-fetch-site-rejected", method: "POST", secFetchSite: "new-unknown-value", wantError: true, }, { - name: "POST requests with none are not allowed", + name: "POST-sec-fetch-none-rejected", method: "POST", secFetchSite: "none", wantError: true, }, { - name: "POST requests with no sec-fetch-site header but matching host and origin are allowed", + name: "POST-no-sec-fetch-site-matching-host-origin", // no sec-fetch-site header but matching host and origin are allowed method: "POST", host: "example.com", origin: "https://example.com", }, { - name: "POST requests with no sec-fetch-site and non-matching host and origin are not allowed", + name: "POST-no-sec-fetch-site-mismatched-host-origin-rejected", method: "POST", host: "example.com", origin: "https://example.net", wantError: true, }, { - name: "POST requests with no sec-fetch-site and and origin that matches the override are allowed", + name: "POST-no-sec-fetch-site-origin-override-allowed", method: "POST", originOverride: "example.net", host: "internal.example.foo", // Host can be changed by reverse proxies @@ -1585,3 +1604,149 @@ func TestCSRFProtect(t *testing.T) { }) } } + +func TestServePostRoutes(t *testing.T) { + existingExitNodeID := tailcfg.StableNodeID("existing-exit-node") + existingRoute := netip.MustParsePrefix("192.168.1.0/24") + + existingPrefs := &ipn.Prefs{ + ExitNodeID: existingExitNodeID, + AdvertiseRoutes: []netip.Prefix{existingRoute}, + } + + tests := []struct { + name string + data postRoutesRequest + peerCaps peerCapabilities + wantErr bool + wantEditPrefs bool // whether EditPrefs (PATCH /prefs) should be called + wantExitNodeID tailcfg.StableNodeID + wantRoutes []netip.Prefix + }{ + { + name: "empty-request", + data: postRoutesRequest{}, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantErr: true, + wantEditPrefs: false, + }, + { + name: "SetExitNode-only", + data: postRoutesRequest{ + SetExitNode: true, + UseExitNode: "new-exit-node", + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantEditPrefs: true, + wantExitNodeID: "new-exit-node", + wantRoutes: []netip.Prefix{existingRoute}, + }, + { + name: "SetExitNode-not-allowed", + data: postRoutesRequest{ + SetExitNode: true, + UseExitNode: "new-exit-node", + }, + peerCaps: peerCapabilities{capFeatureSubnets: true}, + wantErr: true, + }, + { + name: "SetRoutes-only", + data: postRoutesRequest{ + SetRoutes: true, + AdvertiseRoutes: []string{"10.0.0.0/8"}, + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantEditPrefs: true, + wantExitNodeID: existingExitNodeID, + wantRoutes: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, + }, + { + name: "SetRoutes-not-allowed", + data: postRoutesRequest{ + SetRoutes: true, + AdvertiseRoutes: []string{"10.0.0.0/8"}, + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true}, + wantErr: true, + }, + { + name: "SetExitNode-and-SetRoutes", + data: postRoutesRequest{ + SetExitNode: true, + SetRoutes: true, + UseExitNode: "new-exit-node", + AdvertiseRoutes: []string{"10.0.0.0/8"}, + }, + peerCaps: peerCapabilities{capFeatureExitNodes: true, capFeatureSubnets: true}, + wantEditPrefs: true, + wantExitNodeID: "new-exit-node", + wantRoutes: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotPrefs *ipn.MaskedPrefs + + lal := memnet.Listen("local-tailscaled.sock:80") + defer lal.Close() + + localapi := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/localapi/v0/prefs" { + t.Errorf("unexpected localapi call to %q", r.URL.Path) + http.Error(w, "unexpected localapi call", http.StatusInternalServerError) + return + } + switch r.Method { + case httpm.GET: + writeJSON(w, existingPrefs) + case httpm.PATCH: + var mp ipn.MaskedPrefs + if err := json.NewDecoder(r.Body).Decode(&mp); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + gotPrefs = &mp + writeJSON(w, gotPrefs.Prefs) + default: + t.Errorf("unexpected method %q on /prefs", r.Method) + http.Error(w, "unexpected method", http.StatusMethodNotAllowed) + } + })} + defer localapi.Close() + go localapi.Serve(lal) + + s := &Server{ + mode: ManageServerMode, + lc: &local.Client{Dial: lal.Dial}, + } + + ctx := contextKeyPeer.WithValue(t.Context(), tt.peerCaps) + err := s.servePostRoutes(ctx, tt.data) + + if tt.wantErr { + if err == nil { + t.Error("wanted error, got nil") + } + if gotPrefs != nil { + t.Error("EditPrefs should not have been called on error") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if gotPrefs == nil { + t.Fatal("expected EditPrefs to be called") + } + if diff := cmp.Diff(tt.wantExitNodeID, gotPrefs.ExitNodeID); diff != "" { + t.Errorf("ExitNodeID mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tt.wantRoutes, gotPrefs.AdvertiseRoutes, cmp.Comparer(func(a, b netip.Prefix) bool { return a.Compare(b) == 0 })); diff != "" { + t.Errorf("AdvertiseRoutes mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/client/web/yarn.lock b/client/web/yarn.lock index a9b2ae8767b99..106a104b98d26 100644 --- a/client/web/yarn.lock +++ b/client/web/yarn.lock @@ -1087,11 +1087,9 @@ integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA== "@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.16.3", "@babel/runtime@^7.23.2", "@babel/runtime@^7.8.4": - version "7.23.4" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.4.tgz#36fa1d2b36db873d25ec631dcc4923fdc1cf2e2e" - integrity sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg== - dependencies: - regenerator-runtime "^0.14.0" + version "7.28.2" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.28.2.tgz#2ae5a9d51cc583bd1f5673b3bb70d6d819682473" + integrity sha512-KHp2IflsnGywDjBWDkR9iEqiWSpc8GIi0lgTT3mOElT0PP1tG26P4tmFI2YvAdzgq9RGyoHZQEIEdZy6Ec5xCA== "@babel/template@^7.22.15": version "7.22.15" @@ -1132,120 +1130,120 @@ resolved "https://registry.yarnpkg.com/@cush/relative/-/relative-1.0.0.tgz#8cd1769bf9bde3bb27dac356b1bc94af40f6cc16" integrity sha512-RpfLEtTlyIxeNPGKcokS+p3BZII/Q3bYxryFRglh5H3A3T8q9fsLYm72VYAMEOOIBLEa8o93kFLiBDUWKrwXZA== -"@esbuild/aix-ppc64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.19.12.tgz#d1bc06aedb6936b3b6d313bf809a5a40387d2b7f" - integrity sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA== - -"@esbuild/android-arm64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.19.12.tgz#7ad65a36cfdb7e0d429c353e00f680d737c2aed4" - integrity sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA== - -"@esbuild/android-arm@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.19.12.tgz#b0c26536f37776162ca8bde25e42040c203f2824" - integrity sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w== - -"@esbuild/android-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.19.12.tgz#cb13e2211282012194d89bf3bfe7721273473b3d" - integrity sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew== - -"@esbuild/darwin-arm64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.19.12.tgz#cbee41e988020d4b516e9d9e44dd29200996275e" - integrity sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g== - -"@esbuild/darwin-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.19.12.tgz#e37d9633246d52aecf491ee916ece709f9d5f4cd" - integrity sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A== - -"@esbuild/freebsd-arm64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.12.tgz#1ee4d8b682ed363b08af74d1ea2b2b4dbba76487" - integrity sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA== - -"@esbuild/freebsd-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.19.12.tgz#37a693553d42ff77cd7126764b535fb6cc28a11c" - integrity sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg== - -"@esbuild/linux-arm64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.19.12.tgz#be9b145985ec6c57470e0e051d887b09dddb2d4b" - integrity sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA== - -"@esbuild/linux-arm@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.19.12.tgz#207ecd982a8db95f7b5279207d0ff2331acf5eef" - integrity sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w== - -"@esbuild/linux-ia32@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.19.12.tgz#d0d86b5ca1562523dc284a6723293a52d5860601" - integrity sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA== - -"@esbuild/linux-loong64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.19.12.tgz#9a37f87fec4b8408e682b528391fa22afd952299" - integrity sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA== - -"@esbuild/linux-mips64el@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.19.12.tgz#4ddebd4e6eeba20b509d8e74c8e30d8ace0b89ec" - integrity sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w== - -"@esbuild/linux-ppc64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.19.12.tgz#adb67dadb73656849f63cd522f5ecb351dd8dee8" - integrity sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg== - -"@esbuild/linux-riscv64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.19.12.tgz#11bc0698bf0a2abf8727f1c7ace2112612c15adf" - integrity sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg== - -"@esbuild/linux-s390x@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.19.12.tgz#e86fb8ffba7c5c92ba91fc3b27ed5a70196c3cc8" - integrity sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg== - -"@esbuild/linux-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.19.12.tgz#5f37cfdc705aea687dfe5dfbec086a05acfe9c78" - integrity sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg== - -"@esbuild/netbsd-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.19.12.tgz#29da566a75324e0d0dd7e47519ba2f7ef168657b" - integrity sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA== - -"@esbuild/openbsd-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.19.12.tgz#306c0acbdb5a99c95be98bdd1d47c916e7dc3ff0" - integrity sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw== - -"@esbuild/sunos-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.19.12.tgz#0933eaab9af8b9b2c930236f62aae3fc593faf30" - integrity sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA== - -"@esbuild/win32-arm64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.19.12.tgz#773bdbaa1971b36db2f6560088639ccd1e6773ae" - integrity sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A== - -"@esbuild/win32-ia32@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.19.12.tgz#000516cad06354cc84a73f0943a4aa690ef6fd67" - integrity sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ== - -"@esbuild/win32-x64@0.19.12": - version "0.19.12" - resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.19.12.tgz#c57c8afbb4054a3ab8317591a0b7320360b444ae" - integrity sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA== +"@esbuild/aix-ppc64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz#c7184a326533fcdf1b8ee0733e21c713b975575f" + integrity sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ== + +"@esbuild/android-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz#09d9b4357780da9ea3a7dfb833a1f1ff439b4052" + integrity sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A== + +"@esbuild/android-arm@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.21.5.tgz#9b04384fb771926dfa6d7ad04324ecb2ab9b2e28" + integrity sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg== + +"@esbuild/android-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.21.5.tgz#29918ec2db754cedcb6c1b04de8cd6547af6461e" + integrity sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA== + +"@esbuild/darwin-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz#e495b539660e51690f3928af50a76fb0a6ccff2a" + integrity sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ== + +"@esbuild/darwin-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz#c13838fa57372839abdddc91d71542ceea2e1e22" + integrity sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw== + +"@esbuild/freebsd-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz#646b989aa20bf89fd071dd5dbfad69a3542e550e" + integrity sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g== + +"@esbuild/freebsd-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz#aa615cfc80af954d3458906e38ca22c18cf5c261" + integrity sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ== + +"@esbuild/linux-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz#70ac6fa14f5cb7e1f7f887bcffb680ad09922b5b" + integrity sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q== + +"@esbuild/linux-arm@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz#fc6fd11a8aca56c1f6f3894f2bea0479f8f626b9" + integrity sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA== + +"@esbuild/linux-ia32@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz#3271f53b3f93e3d093d518d1649d6d68d346ede2" + integrity sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg== + +"@esbuild/linux-loong64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz#ed62e04238c57026aea831c5a130b73c0f9f26df" + integrity sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg== + +"@esbuild/linux-mips64el@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz#e79b8eb48bf3b106fadec1ac8240fb97b4e64cbe" + integrity sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg== + +"@esbuild/linux-ppc64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz#5f2203860a143b9919d383ef7573521fb154c3e4" + integrity sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w== + +"@esbuild/linux-riscv64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz#07bcafd99322d5af62f618cb9e6a9b7f4bb825dc" + integrity sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA== + +"@esbuild/linux-s390x@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz#b7ccf686751d6a3e44b8627ababc8be3ef62d8de" + integrity sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A== + +"@esbuild/linux-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz#6d8f0c768e070e64309af8004bb94e68ab2bb3b0" + integrity sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ== + +"@esbuild/netbsd-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz#bbe430f60d378ecb88decb219c602667387a6047" + integrity sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg== + +"@esbuild/openbsd-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz#99d1cf2937279560d2104821f5ccce220cb2af70" + integrity sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow== + +"@esbuild/sunos-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz#08741512c10d529566baba837b4fe052c8f3487b" + integrity sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg== + +"@esbuild/win32-arm64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz#675b7385398411240735016144ab2e99a60fc75d" + integrity sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A== + +"@esbuild/win32-ia32@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz#1bfc3ce98aa6ca9a0969e4d2af72144c59c1193b" + integrity sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA== + +"@esbuild/win32-x64@0.21.5": + version "0.21.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz#acad351d582d157bb145535db2a6ff53dd514b5c" + integrity sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw== "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" @@ -1628,70 +1626,115 @@ estree-walker "^2.0.2" picomatch "^2.3.1" -"@rollup/rollup-android-arm-eabi@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.12.0.tgz#38c3abd1955a3c21d492af6b1a1dca4bb1d894d6" - integrity sha512-+ac02NL/2TCKRrJu2wffk1kZ+RyqxVUlbjSagNgPm94frxtr+XDL12E5Ll1enWskLrtrZ2r8L3wED1orIibV/w== - -"@rollup/rollup-android-arm64@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.12.0.tgz#3822e929f415627609e53b11cec9a4be806de0e2" - integrity sha512-OBqcX2BMe6nvjQ0Nyp7cC90cnumt8PXmO7Dp3gfAju/6YwG0Tj74z1vKrfRz7qAv23nBcYM8BCbhrsWqO7PzQQ== - -"@rollup/rollup-darwin-arm64@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.12.0.tgz#6c082de71f481f57df6cfa3701ab2a7afde96f69" - integrity sha512-X64tZd8dRE/QTrBIEs63kaOBG0b5GVEd3ccoLtyf6IdXtHdh8h+I56C2yC3PtC9Ucnv0CpNFJLqKFVgCYe0lOQ== - -"@rollup/rollup-darwin-x64@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.12.0.tgz#c34ca0d31f3c46a22c9afa0e944403eea0edcfd8" - integrity sha512-cc71KUZoVbUJmGP2cOuiZ9HSOP14AzBAThn3OU+9LcA1+IUqswJyR1cAJj3Mg55HbjZP6OLAIscbQsQLrpgTOg== - -"@rollup/rollup-linux-arm-gnueabihf@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.12.0.tgz#48e899c1e438629c072889b824a98787a7c2362d" - integrity sha512-a6w/Y3hyyO6GlpKL2xJ4IOh/7d+APaqLYdMf86xnczU3nurFTaVN9s9jOXQg97BE4nYm/7Ga51rjec5nfRdrvA== - -"@rollup/rollup-linux-arm64-gnu@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.12.0.tgz#788c2698a119dc229062d40da6ada8a090a73a68" - integrity sha512-0fZBq27b+D7Ar5CQMofVN8sggOVhEtzFUwOwPppQt0k+VR+7UHMZZY4y+64WJ06XOhBTKXtQB/Sv0NwQMXyNAA== - -"@rollup/rollup-linux-arm64-musl@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.12.0.tgz#3882a4e3a564af9e55804beeb67076857b035ab7" - integrity sha512-eTvzUS3hhhlgeAv6bfigekzWZjaEX9xP9HhxB0Dvrdbkk5w/b+1Sxct2ZuDxNJKzsRStSq1EaEkVSEe7A7ipgQ== - -"@rollup/rollup-linux-riscv64-gnu@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.12.0.tgz#0c6ad792e1195c12bfae634425a3d2aa0fe93ab7" - integrity sha512-ix+qAB9qmrCRiaO71VFfY8rkiAZJL8zQRXveS27HS+pKdjwUfEhqo2+YF2oI+H/22Xsiski+qqwIBxVewLK7sw== - -"@rollup/rollup-linux-x64-gnu@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.12.0.tgz#9d62485ea0f18d8674033b57aa14fb758f6ec6e3" - integrity sha512-TenQhZVOtw/3qKOPa7d+QgkeM6xY0LtwzR8OplmyL5LrgTWIXpTQg2Q2ycBf8jm+SFW2Wt/DTn1gf7nFp3ssVA== - -"@rollup/rollup-linux-x64-musl@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.12.0.tgz#50e8167e28b33c977c1f813def2b2074d1435e05" - integrity sha512-LfFdRhNnW0zdMvdCb5FNuWlls2WbbSridJvxOvYWgSBOYZtgBfW9UGNJG//rwMqTX1xQE9BAodvMH9tAusKDUw== - -"@rollup/rollup-win32-arm64-msvc@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.12.0.tgz#68d233272a2004429124494121a42c4aebdc5b8e" - integrity sha512-JPDxovheWNp6d7AHCgsUlkuCKvtu3RB55iNEkaQcf0ttsDU/JZF+iQnYcQJSk/7PtT4mjjVG8N1kpwnI9SLYaw== - -"@rollup/rollup-win32-ia32-msvc@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.12.0.tgz#366ca62221d1689e3b55a03f4ae12ae9ba595d40" - integrity sha512-fjtuvMWRGJn1oZacG8IPnzIV6GF2/XG+h71FKn76OYFqySXInJtseAqdprVTDTyqPxQOG9Exak5/E9Z3+EJ8ZA== - -"@rollup/rollup-win32-x64-msvc@4.12.0": - version "4.12.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.12.0.tgz#9ffdf9ed133a7464f4ae187eb9e1294413fab235" - integrity sha512-ZYmr5mS2wd4Dew/JjT0Fqi2NPB/ZhZ2VvPp7SmvPZb4Y1CG/LRcS6tcRo2cYU7zLK5A7cdbhWnnWmUjoI4qapg== +"@rollup/rollup-android-arm-eabi@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.52.5.tgz#0f44a2f8668ed87b040b6fe659358ac9239da4db" + integrity sha512-8c1vW4ocv3UOMp9K+gToY5zL2XiiVw3k7f1ksf4yO1FlDFQ1C2u72iACFnSOceJFsWskc2WZNqeRhFRPzv+wtQ== + +"@rollup/rollup-android-arm64@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.52.5.tgz#25b9a01deef6518a948431564c987bcb205274f5" + integrity sha512-mQGfsIEFcu21mvqkEKKu2dYmtuSZOBMmAl5CFlPGLY94Vlcm+zWApK7F/eocsNzp8tKmbeBP8yXyAbx0XHsFNA== + +"@rollup/rollup-darwin-arm64@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.52.5.tgz#8a102869c88f3780c7d5e6776afd3f19084ecd7f" + integrity sha512-takF3CR71mCAGA+v794QUZ0b6ZSrgJkArC+gUiG6LB6TQty9T0Mqh3m2ImRBOxS2IeYBo4lKWIieSvnEk2OQWA== + +"@rollup/rollup-darwin-x64@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.52.5.tgz#8e526417cd6f54daf1d0c04cf361160216581956" + integrity sha512-W901Pla8Ya95WpxDn//VF9K9u2JbocwV/v75TE0YIHNTbhqUTv9w4VuQ9MaWlNOkkEfFwkdNhXgcLqPSmHy0fA== + +"@rollup/rollup-freebsd-arm64@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.52.5.tgz#0e7027054493f3409b1f219a3eac5efd128ef899" + integrity sha512-QofO7i7JycsYOWxe0GFqhLmF6l1TqBswJMvICnRUjqCx8b47MTo46W8AoeQwiokAx3zVryVnxtBMcGcnX12LvA== + +"@rollup/rollup-freebsd-x64@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.52.5.tgz#72b204a920139e9ec3d331bd9cfd9a0c248ccb10" + integrity sha512-jr21b/99ew8ujZubPo9skbrItHEIE50WdV86cdSoRkKtmWa+DDr6fu2c/xyRT0F/WazZpam6kk7IHBerSL7LDQ== + +"@rollup/rollup-linux-arm-gnueabihf@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.52.5.tgz#ab1b522ebe5b7e06c99504cc38f6cd8b808ba41c" + integrity sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ== + +"@rollup/rollup-linux-arm-musleabihf@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.52.5.tgz#f8cc30b638f1ee7e3d18eac24af47ea29d9beb00" + integrity sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ== + +"@rollup/rollup-linux-arm64-gnu@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.52.5.tgz#7af37a9e85f25db59dc8214172907b7e146c12cc" + integrity sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg== + +"@rollup/rollup-linux-arm64-musl@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.52.5.tgz#a623eb0d3617c03b7a73716eb85c6e37b776f7e0" + integrity sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q== + +"@rollup/rollup-linux-loong64-gnu@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.52.5.tgz#76ea038b549c5c6c5f0d062942627c4066642ee2" + integrity sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA== + +"@rollup/rollup-linux-ppc64-gnu@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.52.5.tgz#d9a4c3f0a3492bc78f6fdfe8131ac61c7359ccd5" + integrity sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw== + +"@rollup/rollup-linux-riscv64-gnu@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.52.5.tgz#87ab033eebd1a9a1dd7b60509f6333ec1f82d994" + integrity sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw== + +"@rollup/rollup-linux-riscv64-musl@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.52.5.tgz#bda3eb67e1c993c1ba12bc9c2f694e7703958d9f" + integrity sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg== + +"@rollup/rollup-linux-s390x-gnu@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.52.5.tgz#f7bc10fbe096ab44694233dc42a2291ed5453d4b" + integrity sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ== + +"@rollup/rollup-linux-x64-gnu@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.52.5.tgz#a151cb1234cc9b2cf5e8cfc02aa91436b8f9e278" + integrity sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q== + +"@rollup/rollup-linux-x64-musl@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.52.5.tgz#7859e196501cc3b3062d45d2776cfb4d2f3a9350" + integrity sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg== + +"@rollup/rollup-openharmony-arm64@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.52.5.tgz#85d0df7233734df31e547c1e647d2a5300b3bf30" + integrity sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw== + +"@rollup/rollup-win32-arm64-msvc@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.52.5.tgz#e62357d00458db17277b88adbf690bb855cac937" + integrity sha512-w0cDWVR6MlTstla1cIfOGyl8+qb93FlAVutcor14Gf5Md5ap5ySfQ7R9S/NjNaMLSFdUnKGEasmVnu3lCMqB7w== + +"@rollup/rollup-win32-ia32-msvc@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.52.5.tgz#fc7cd40f44834a703c1f1c3fe8bcc27ce476cd50" + integrity sha512-Aufdpzp7DpOTULJCuvzqcItSGDH73pF3ko/f+ckJhxQyHtp67rHw3HMNxoIdDMUITJESNE6a8uh4Lo4SLouOUg== + +"@rollup/rollup-win32-x64-gnu@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.52.5.tgz#1a22acfc93c64a64a48c42672e857ee51774d0d3" + integrity sha512-UGBUGPFp1vkj6p8wCRraqNhqwX/4kNQPS57BCFc8wYh0g94iVIW33wJtQAx3G7vrjjNtRaxiMUylM0ktp/TRSQ== + +"@rollup/rollup-win32-x64-msvc@4.52.5": + version "4.52.5" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.52.5.tgz#1657f56326bbe0ac80eedc9f9c18fc1ddd24e107" + integrity sha512-TAcgQh2sSkykPRWLrdyy2AiceMckNf5loITqXxFI5VuQjS5tSuw3WlwdN8qv8vzjLAUTvYaH/mVjSFpbkFbpTg== "@rushstack/eslint-patch@^1.1.0": version "1.6.0" @@ -1865,7 +1908,12 @@ resolved "https://registry.yarnpkg.com/@swc/types/-/types-0.1.5.tgz#043b731d4f56a79b4897a3de1af35e75d56bc63a" integrity sha512-myfUej5naTBWnqOCc/MdVOLVjXUXtIA+NpDrDBKJtLLg2shUjBu3cZmB/85RyitKc55+lUUyl7oRfLOvkr2hsw== -"@types/estree@1.0.5", "@types/estree@^1.0.0": +"@types/estree@1.0.8": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.8.tgz#958b91c991b1867ced318bedea0e215ee050726e" + integrity sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w== + +"@types/estree@^1.0.0": version "1.0.5" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== @@ -2076,44 +2124,44 @@ dependencies: "@swc/core" "^1.3.107" -"@vitest/expect@1.3.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-1.3.1.tgz#d4c14b89c43a25fd400a6b941f51ba27fe0cb918" - integrity sha512-xofQFwIzfdmLLlHa6ag0dPV8YsnKOCP1KdAeVVh34vSjN2dcUiXYCD9htu/9eM7t8Xln4v03U9HLxLpPlsXdZw== +"@vitest/expect@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-1.6.1.tgz#b90c213f587514a99ac0bf84f88cff9042b0f14d" + integrity sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog== dependencies: - "@vitest/spy" "1.3.1" - "@vitest/utils" "1.3.1" + "@vitest/spy" "1.6.1" + "@vitest/utils" "1.6.1" chai "^4.3.10" -"@vitest/runner@1.3.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@vitest/runner/-/runner-1.3.1.tgz#e7f96cdf74842934782bfd310eef4b8695bbfa30" - integrity sha512-5FzF9c3jG/z5bgCnjr8j9LNq/9OxV2uEBAITOXfoe3rdZJTdO7jzThth7FXv/6b+kdY65tpRQB7WaKhNZwX+Kg== +"@vitest/runner@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@vitest/runner/-/runner-1.6.1.tgz#10f5857c3e376218d58c2bfacfea1161e27e117f" + integrity sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA== dependencies: - "@vitest/utils" "1.3.1" + "@vitest/utils" "1.6.1" p-limit "^5.0.0" pathe "^1.1.1" -"@vitest/snapshot@1.3.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@vitest/snapshot/-/snapshot-1.3.1.tgz#193a5d7febf6ec5d22b3f8c5a093f9e4322e7a88" - integrity sha512-EF++BZbt6RZmOlE3SuTPu/NfwBF6q4ABS37HHXzs2LUVPBLx2QoY/K0fKpRChSo8eLiuxcbCVfqKgx/dplCDuQ== +"@vitest/snapshot@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@vitest/snapshot/-/snapshot-1.6.1.tgz#90414451a634bb36cd539ccb29ae0d048a8c0479" + integrity sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ== dependencies: magic-string "^0.30.5" pathe "^1.1.1" pretty-format "^29.7.0" -"@vitest/spy@1.3.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@vitest/spy/-/spy-1.3.1.tgz#814245d46d011b99edd1c7528f5725c64e85a88b" - integrity sha512-xAcW+S099ylC9VLU7eZfdT9myV67Nor9w9zhf0mGCYJSO+zM2839tOeROTdikOi/8Qeusffvxb/MyBSOja1Uig== +"@vitest/spy@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@vitest/spy/-/spy-1.6.1.tgz#33376be38a5ed1ecd829eb986edaecc3e798c95d" + integrity sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw== dependencies: tinyspy "^2.2.0" -"@vitest/utils@1.3.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-1.3.1.tgz#7b05838654557544f694a372de767fcc9594d61a" - integrity sha512-d3Waie/299qqRyHTm2DjADeTaNdNSVsnwHPWrs20JMpjh6eiVq7ggggweO8rc4arhf6rRkWuHKwvxGvejUXZZQ== +"@vitest/utils@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-1.6.1.tgz#6d2f36cb6d866f2bbf59da854a324d6bf8040f17" + integrity sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g== dependencies: diff-sequences "^29.6.3" estree-walker "^3.0.3" @@ -2429,11 +2477,11 @@ brace-expansion@^2.0.1: balanced-match "^1.0.0" braces@^3.0.2, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" browserslist@^4.21.10, browserslist@^4.21.9, browserslist@^4.22.1: version "4.22.1" @@ -2450,6 +2498,14 @@ cac@^6.7.14: resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959" integrity sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ== +call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6" + integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ== + dependencies: + es-errors "^1.3.0" + function-bind "^1.1.2" + call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513" @@ -2621,9 +2677,9 @@ cosmiconfig@^8.1.3: path-type "^4.0.0" cross-spawn@^7.0.2, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" @@ -2767,6 +2823,15 @@ dot-case@^3.0.4: no-case "^3.0.4" tslib "^2.0.3" +dunder-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" + integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A== + dependencies: + call-bind-apply-helpers "^1.0.1" + es-errors "^1.3.0" + gopd "^1.2.0" + electron-to-chromium@^1.4.535: version "1.4.596" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.596.tgz#6752d1aa795d942d49dfc5d3764d6ea283fab1d7" @@ -2834,6 +2899,16 @@ es-abstract@^1.22.1: unbox-primitive "^1.0.2" which-typed-array "^1.1.13" +es-define-property@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" + integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== + +es-errors@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" + integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== + es-iterator-helpers@^1.0.12, es-iterator-helpers@^1.0.15: version "1.0.15" resolved "https://registry.yarnpkg.com/es-iterator-helpers/-/es-iterator-helpers-1.0.15.tgz#bd81d275ac766431d19305923707c3efd9f1ae40" @@ -2854,6 +2929,13 @@ es-iterator-helpers@^1.0.12, es-iterator-helpers@^1.0.15: iterator.prototype "^1.1.2" safe-array-concat "^1.0.1" +es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1" + integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA== + dependencies: + es-errors "^1.3.0" + es-set-tostringtag@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.2.tgz#11f7cc9f63376930a5f20be4915834f4bc74f9c9" @@ -2863,6 +2945,16 @@ es-set-tostringtag@^2.0.1: has-tostringtag "^1.0.0" hasown "^2.0.0" +es-set-tostringtag@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz#f31dbbe0c183b00a6d26eb6325c810c0fd18bd4d" + integrity sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA== + dependencies: + es-errors "^1.3.0" + get-intrinsic "^1.2.6" + has-tostringtag "^1.0.2" + hasown "^2.0.2" + es-shim-unscopables@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz#1f6942e71ecc7835ed1c8a83006d8771a63a3763" @@ -2879,34 +2971,34 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" -esbuild@^0.19.3: - version "0.19.12" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.19.12.tgz#dc82ee5dc79e82f5a5c3b4323a2a641827db3e04" - integrity sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg== +esbuild@^0.21.3: + version "0.21.5" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.21.5.tgz#9ca301b120922959b766360d8ac830da0d02997d" + integrity sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw== optionalDependencies: - "@esbuild/aix-ppc64" "0.19.12" - "@esbuild/android-arm" "0.19.12" - "@esbuild/android-arm64" "0.19.12" - "@esbuild/android-x64" "0.19.12" - "@esbuild/darwin-arm64" "0.19.12" - "@esbuild/darwin-x64" "0.19.12" - "@esbuild/freebsd-arm64" "0.19.12" - "@esbuild/freebsd-x64" "0.19.12" - "@esbuild/linux-arm" "0.19.12" - "@esbuild/linux-arm64" "0.19.12" - "@esbuild/linux-ia32" "0.19.12" - "@esbuild/linux-loong64" "0.19.12" - "@esbuild/linux-mips64el" "0.19.12" - "@esbuild/linux-ppc64" "0.19.12" - "@esbuild/linux-riscv64" "0.19.12" - "@esbuild/linux-s390x" "0.19.12" - "@esbuild/linux-x64" "0.19.12" - "@esbuild/netbsd-x64" "0.19.12" - "@esbuild/openbsd-x64" "0.19.12" - "@esbuild/sunos-x64" "0.19.12" - "@esbuild/win32-arm64" "0.19.12" - "@esbuild/win32-ia32" "0.19.12" - "@esbuild/win32-x64" "0.19.12" + "@esbuild/aix-ppc64" "0.21.5" + "@esbuild/android-arm" "0.21.5" + "@esbuild/android-arm64" "0.21.5" + "@esbuild/android-x64" "0.21.5" + "@esbuild/darwin-arm64" "0.21.5" + "@esbuild/darwin-x64" "0.21.5" + "@esbuild/freebsd-arm64" "0.21.5" + "@esbuild/freebsd-x64" "0.21.5" + "@esbuild/linux-arm" "0.21.5" + "@esbuild/linux-arm64" "0.21.5" + "@esbuild/linux-ia32" "0.21.5" + "@esbuild/linux-loong64" "0.21.5" + "@esbuild/linux-mips64el" "0.21.5" + "@esbuild/linux-ppc64" "0.21.5" + "@esbuild/linux-riscv64" "0.21.5" + "@esbuild/linux-s390x" "0.21.5" + "@esbuild/linux-x64" "0.21.5" + "@esbuild/netbsd-x64" "0.21.5" + "@esbuild/openbsd-x64" "0.21.5" + "@esbuild/sunos-x64" "0.21.5" + "@esbuild/win32-arm64" "0.21.5" + "@esbuild/win32-ia32" "0.21.5" + "@esbuild/win32-x64" "0.21.5" escalade@^3.1.1: version "3.1.1" @@ -3233,10 +3325,10 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" @@ -3270,12 +3362,14 @@ for-each@^0.3.3: is-callable "^1.1.3" form-data@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" - integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + version "4.0.4" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.4.tgz#784cdcce0669a9d68e94d11ac4eea98088edd2c4" + integrity sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" + es-set-tostringtag "^2.1.0" + hasown "^2.0.2" mime-types "^2.1.12" fraction.js@^4.2.0: @@ -3333,11 +3427,35 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@ has-symbols "^1.0.3" hasown "^2.0.0" +get-intrinsic@^1.2.6: + version "1.3.0" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" + integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== + dependencies: + call-bind-apply-helpers "^1.0.2" + es-define-property "^1.0.1" + es-errors "^1.3.0" + es-object-atoms "^1.1.1" + function-bind "^1.1.2" + get-proto "^1.0.1" + gopd "^1.2.0" + has-symbols "^1.1.0" + hasown "^2.0.2" + math-intrinsics "^1.1.0" + get-nonce@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/get-nonce/-/get-nonce-1.0.1.tgz#fdf3f0278073820d2ce9426c18f07481b1e0cdf3" integrity sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q== +get-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1" + integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g== + dependencies: + dunder-proto "^1.0.1" + es-object-atoms "^1.0.0" + get-stream@^8.0.1: version "8.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-8.0.1.tgz#def9dfd71742cd7754a7761ed43749a27d02eca2" @@ -3437,6 +3555,11 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" +gopd@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" + integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== + graphemer@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" @@ -3474,6 +3597,11 @@ has-symbols@^1.0.2, has-symbols@^1.0.3: resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== +has-symbols@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" + integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ== + has-tostringtag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" @@ -3481,6 +3609,13 @@ has-tostringtag@^1.0.0: dependencies: has-symbols "^1.0.2" +has-tostringtag@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" + integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== + dependencies: + has-symbols "^1.0.3" + hasown@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.0.tgz#f4c513d454a57b7c7e1650778de226b11700546c" @@ -3488,6 +3623,13 @@ hasown@^2.0.0: dependencies: function-bind "^1.1.2" +hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + html-encoding-sniffer@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz#696df529a7cfd82446369dc5193e590a3735b448" @@ -3793,9 +3935,9 @@ js-tokens@^8.0.2: integrity sha512-UfJMcSJc+SEXEl9lH/VLHSZbThQyLpw1vLO1Lb+j4RWDvG3N2f7yj3PVQA3cmkTBNldJ9eFnM+xEXxHIXrYiJw== js-yaml@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" - integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + version "4.1.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.1.tgz#854c292467705b699476e1a2decc0c8a3458806b" + integrity sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA== dependencies: argparse "^2.0.1" @@ -3946,9 +4088,9 @@ lodash.merge@^4.6.2: integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== lodash@^4.17.21: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + version "4.17.23" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.23.tgz#f113b0378386103be4f6893388c73d0bde7f2c5a" + integrity sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w== loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" @@ -3992,6 +4134,11 @@ magic-string@^0.30.5: dependencies: "@jridgewell/sourcemap-codec" "^1.4.15" +math-intrinsics@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" + integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g== + merge-stream@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" @@ -4075,10 +4222,10 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" -nanoid@^3.3.7: - version "3.3.7" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" - integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== +nanoid@^3.3.11: + version "3.3.11" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.11.tgz#4f4f112cefbe303202f2199838128936266d185b" + integrity sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w== natural-compare@^1.4.0: version "1.4.0" @@ -4306,10 +4453,10 @@ pathval@^1.1.1: resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.1.tgz#8534e77a77ce7ac5a2512ea21e0fdb8fcf6c3d8d" integrity sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ== -picocolors@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" - integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== +picocolors@^1.0.0, picocolors@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: version "2.3.1" @@ -4379,14 +4526,14 @@ postcss-value-parser@^4.0.0, postcss-value-parser@^4.2.0: resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514" integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== -postcss@^8.4.23, postcss@^8.4.31, postcss@^8.4.35: - version "8.4.35" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.35.tgz#60997775689ce09011edf083a549cea44aabe2f7" - integrity sha512-u5U8qYpBCpN13BsiEB0CbR1Hhh4Gc0zLFuedrHJKMctHCHAGrMdG0PRM/KErzAL3CU6/eckEtmHNB3x6e3c0vA== +postcss@^8.4.23, postcss@^8.4.31, postcss@^8.4.43: + version "8.5.6" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.6.tgz#2825006615a619b4f62a9e7426cc120b349a8f3c" + integrity sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg== dependencies: - nanoid "^3.3.7" - picocolors "^1.0.0" - source-map-js "^1.0.2" + nanoid "^3.3.11" + picocolors "^1.1.1" + source-map-js "^1.2.1" prelude-ls@^1.2.1: version "1.2.1" @@ -4543,11 +4690,6 @@ regenerate@^1.4.2: resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a" integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A== -regenerator-runtime@^0.14.0: - version "0.14.0" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz#5e19d68eb12d486f797e15a3c6a918f7cec5eb45" - integrity sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA== - regenerator-transform@^0.15.2: version "0.15.2" resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.15.2.tgz#5bbae58b522098ebdf09bca2f83838929001c7a4" @@ -4623,26 +4765,35 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" -rollup@^4.2.0: - version "4.12.0" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.12.0.tgz#0b6d1e5f3d46bbcf244deec41a7421dc54cc45b5" - integrity sha512-wz66wn4t1OHIJw3+XU7mJJQV/2NAfw5OAk6G6Hoo3zcvz/XOfQ52Vgi+AN4Uxoxi0KBBwk2g8zPrTDA4btSB/Q== +rollup@^4.20.0: + version "4.52.5" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.52.5.tgz#96982cdcaedcdd51b12359981f240f94304ec235" + integrity sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw== dependencies: - "@types/estree" "1.0.5" + "@types/estree" "1.0.8" optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.12.0" - "@rollup/rollup-android-arm64" "4.12.0" - "@rollup/rollup-darwin-arm64" "4.12.0" - "@rollup/rollup-darwin-x64" "4.12.0" - "@rollup/rollup-linux-arm-gnueabihf" "4.12.0" - "@rollup/rollup-linux-arm64-gnu" "4.12.0" - "@rollup/rollup-linux-arm64-musl" "4.12.0" - "@rollup/rollup-linux-riscv64-gnu" "4.12.0" - "@rollup/rollup-linux-x64-gnu" "4.12.0" - "@rollup/rollup-linux-x64-musl" "4.12.0" - "@rollup/rollup-win32-arm64-msvc" "4.12.0" - "@rollup/rollup-win32-ia32-msvc" "4.12.0" - "@rollup/rollup-win32-x64-msvc" "4.12.0" + "@rollup/rollup-android-arm-eabi" "4.52.5" + "@rollup/rollup-android-arm64" "4.52.5" + "@rollup/rollup-darwin-arm64" "4.52.5" + "@rollup/rollup-darwin-x64" "4.52.5" + "@rollup/rollup-freebsd-arm64" "4.52.5" + "@rollup/rollup-freebsd-x64" "4.52.5" + "@rollup/rollup-linux-arm-gnueabihf" "4.52.5" + "@rollup/rollup-linux-arm-musleabihf" "4.52.5" + "@rollup/rollup-linux-arm64-gnu" "4.52.5" + "@rollup/rollup-linux-arm64-musl" "4.52.5" + "@rollup/rollup-linux-loong64-gnu" "4.52.5" + "@rollup/rollup-linux-ppc64-gnu" "4.52.5" + "@rollup/rollup-linux-riscv64-gnu" "4.52.5" + "@rollup/rollup-linux-riscv64-musl" "4.52.5" + "@rollup/rollup-linux-s390x-gnu" "4.52.5" + "@rollup/rollup-linux-x64-gnu" "4.52.5" + "@rollup/rollup-linux-x64-musl" "4.52.5" + "@rollup/rollup-openharmony-arm64" "4.52.5" + "@rollup/rollup-win32-arm64-msvc" "4.52.5" + "@rollup/rollup-win32-ia32-msvc" "4.52.5" + "@rollup/rollup-win32-x64-gnu" "4.52.5" + "@rollup/rollup-win32-x64-msvc" "4.52.5" fsevents "~2.3.2" rrweb-cssom@^0.6.0: @@ -4770,10 +4921,10 @@ snake-case@^3.0.4: dot-case "^3.0.4" tslib "^2.0.3" -source-map-js@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" - integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== +source-map-js@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.1.tgz#1ce5650fddd87abc099eda37dcff024c2667ae46" + integrity sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA== stackback@0.0.2: version "0.0.2" @@ -4963,10 +5114,10 @@ tinybench@^2.5.1: resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.6.0.tgz#1423284ee22de07c91b3752c048d2764714b341b" integrity sha512-N8hW3PG/3aOoZAN5V/NSAEDz0ZixDSSt5b/a05iqtpgfLWMSVuCo7w0k2vVvEjdrIoeGqZzweX2WlyioNIHchA== -tinypool@^0.8.2: - version "0.8.2" - resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-0.8.2.tgz#84013b03dc69dacb322563a475d4c0a9be00f82a" - integrity sha512-SUszKYe5wgsxnNOVlBYO6IC+8VGWdVGZWAqUxp3UErNBtptZvWbwyUOyzNL59zigz2rCA92QiL3wvG+JDSdJdQ== +tinypool@^0.8.3: + version "0.8.4" + resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-0.8.4.tgz#e217fe1270d941b39e98c625dcecebb1408c9aa8" + integrity sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ== tinyspy@^2.2.0: version "2.2.1" @@ -5205,10 +5356,10 @@ util-deprecate@^1.0.2: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== -vite-node@1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-1.3.1.tgz#a93f7372212f5d5df38e945046b945ac3f4855d2" - integrity sha512-azbRrqRxlWTJEVbzInZCTchx0X69M/XPTCz4H+TLvlTcR/xH/3hkRqhOakT41fMJCMzXTu4UvegkZiEoJAWvng== +vite-node@1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-1.6.1.tgz#fff3ef309296ea03ceaa6ca4bb660922f5416c57" + integrity sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA== dependencies: cac "^6.7.14" debug "^4.3.4" @@ -5235,27 +5386,27 @@ vite-tsconfig-paths@^3.5.0: recrawl-sync "^2.0.3" tsconfig-paths "^4.0.0" -vite@^5.0.0, vite@^5.1.7: - version "5.1.7" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.1.7.tgz#9f685a2c4c70707fef6d37341b0e809c366da619" - integrity sha512-sgnEEFTZYMui/sTlH1/XEnVNHMujOahPLGMxn1+5sIT45Xjng1Ec1K78jRP15dSmVgg5WBin9yO81j3o9OxofA== +vite@^5.0.0, vite@^5.4.21: + version "5.4.21" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.4.21.tgz#84a4f7c5d860b071676d39ba513c0d598fdc7027" + integrity sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw== dependencies: - esbuild "^0.19.3" - postcss "^8.4.35" - rollup "^4.2.0" + esbuild "^0.21.3" + postcss "^8.4.43" + rollup "^4.20.0" optionalDependencies: fsevents "~2.3.3" -vitest@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/vitest/-/vitest-1.3.1.tgz#2d7e9861f030d88a4669392a4aecb40569d90937" - integrity sha512-/1QJqXs8YbCrfv/GPQ05wAZf2eakUPLPa18vkJAKE7RXOKfVHqMZZ1WlTjiwl6Gcn65M5vpNUB6EFLnEdRdEXQ== +vitest@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/vitest/-/vitest-1.6.1.tgz#b4a3097adf8f79ac18bc2e2e0024c534a7a78d2f" + integrity sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag== dependencies: - "@vitest/expect" "1.3.1" - "@vitest/runner" "1.3.1" - "@vitest/snapshot" "1.3.1" - "@vitest/spy" "1.3.1" - "@vitest/utils" "1.3.1" + "@vitest/expect" "1.6.1" + "@vitest/runner" "1.6.1" + "@vitest/snapshot" "1.6.1" + "@vitest/spy" "1.6.1" + "@vitest/utils" "1.6.1" acorn-walk "^8.3.2" chai "^4.3.10" debug "^4.3.4" @@ -5267,9 +5418,9 @@ vitest@^1.3.1: std-env "^3.5.0" strip-literal "^2.0.0" tinybench "^2.5.1" - tinypool "^0.8.2" + tinypool "^0.8.3" vite "^5.0.0" - vite-node "1.3.1" + vite-node "1.6.1" why-is-node-running "^2.2.2" w3c-xmlserializer@^5.0.0: diff --git a/clientupdate/clientupdate.go b/clientupdate/clientupdate.go index ffd3fb03bb80d..020aab1e9b408 100644 --- a/clientupdate/clientupdate.go +++ b/clientupdate/clientupdate.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package clientupdate implements tailscale client update for all supported @@ -16,6 +16,7 @@ import ( "errors" "fmt" "io" + "io/fs" "maps" "net/http" "os" @@ -27,6 +28,8 @@ import ( "strconv" "strings" + "tailscale.com/envknob" + "tailscale.com/feature" "tailscale.com/hostinfo" "tailscale.com/types/lazy" "tailscale.com/types/logger" @@ -35,9 +38,28 @@ import ( "tailscale.com/version/distro" ) +// GokrazyUpdateArgs contains arguments for updating a Gokrazy appliance from a +// GAF fetched from a URL. +type GokrazyUpdateArgs struct { + // URL is the GAF download URL. + URL string + + // AllowUnsigned permits installing a GAF without signature verification. + // This is intended for tests until signed GAF verification is implemented. + AllowUnsigned bool + + // Logf is optional; nil discards log messages. + Logf logger.Logf +} + +// GokrazyUpdateFromURL updates a Gokrazy appliance from a GAF fetched from a +// URL, if Gokrazy update support is linked into the binary. +var GokrazyUpdateFromURL feature.Hook[func(context.Context, GokrazyUpdateArgs) error] + const ( - StableTrack = "stable" - UnstableTrack = "unstable" + StableTrack = "stable" + UnstableTrack = "unstable" + ReleaseCandidateTrack = "release-candidate" ) var CurrentTrack = func() string { @@ -78,6 +100,8 @@ type Arguments struct { // running binary // - StableTrack and UnstableTrack will use the latest versions of the // corresponding tracks + // - ReleaseCandidateTrack will use the newest version from StableTrack + // and ReleaseCandidateTrack. // // Leaving this empty will use Version or fall back to CurrentTrack if both // Track and Version are empty. @@ -112,7 +136,7 @@ func (args Arguments) validate() error { return fmt.Errorf("only one of Version(%q) or Track(%q) can be set", args.Version, args.Track) } switch args.Track { - case StableTrack, UnstableTrack, "": + case StableTrack, UnstableTrack, ReleaseCandidateTrack, "": // All valid values. default: return fmt.Errorf("unsupported track %q", args.Track) @@ -252,9 +276,13 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) { var canAutoUpdateCache lazy.SyncValue[bool] -// CanAutoUpdate reports whether auto-updating via the clientupdate package +func init() { + feature.HookCanAutoUpdate.Set(canAutoUpdate) +} + +// canAutoUpdate reports whether auto-updating via the clientupdate package // is supported for the current os/distro. -func CanAutoUpdate() bool { return canAutoUpdateCache.Get(canAutoUpdateUncached) } +func canAutoUpdate() bool { return canAutoUpdateCache.Get(canAutoUpdateUncached) } func canAutoUpdateUncached() bool { if version.IsMacSysExt() { @@ -283,6 +311,10 @@ func Update(args Arguments) error { } func (up *Updater) confirm(ver string) bool { + if envknob.Bool("TS_UPDATE_SKIP_VERSION_CHECK") { + up.Logf("current version: %v, latest version %v; forcing an update due to TS_UPDATE_SKIP_VERSION_CHECK", up.currentVersion, ver) + return true + } // Only check version when we're not switching tracks. if up.Track == "" || up.Track == CurrentTrack { switch c := cmpver.Compare(up.currentVersion, ver); { @@ -413,13 +445,13 @@ func parseSynoinfo(path string) (string, error) { // Extract the CPU in the middle (88f6282 in the above example). s := bufio.NewScanner(f) for s.Scan() { - l := s.Text() - if !strings.HasPrefix(l, "unique=") { + line := s.Text() + if !strings.HasPrefix(line, "unique=") { continue } - parts := strings.SplitN(l, "_", 3) + parts := strings.SplitN(line, "_", 3) if len(parts) != 3 { - return "", fmt.Errorf(`malformed %q: found %q, expected format like 'unique="synology_$cpu_$model'`, path, l) + return "", fmt.Errorf(`malformed %q: found %q, expected format like 'unique="synology_$cpu_$model'`, path, line) } return parts[1], nil } @@ -486,10 +518,10 @@ func (up *Updater) updateDebLike() error { const aptSourcesFile = "/etc/apt/sources.list.d/tailscale.list" // updateDebianAptSourcesList updates the /etc/apt/sources.list.d/tailscale.list -// file to make sure it has the provided track (stable or unstable) in it. +// file to make sure it has the provided track (stable, unstable, or release-candidate) in it. // -// If it already has the right track (including containing both stable and -// unstable), it does nothing. +// If it already has the right track (including containing both stable, +// unstable, and release-candidate), it does nothing. func updateDebianAptSourcesList(dstTrack string) (rewrote bool, err error) { was, err := os.ReadFile(aptSourcesFile) if err != nil { @@ -512,7 +544,7 @@ func updateDebianAptSourcesListBytes(was []byte, dstTrack string) (newContent [] bs := bufio.NewScanner(bytes.NewReader(was)) hadCorrect := false commentLine := regexp.MustCompile(`^\s*\#`) - pkgsURL := regexp.MustCompile(`\bhttps://pkgs\.tailscale\.com/((un)?stable)/`) + pkgsURL := regexp.MustCompile(`\bhttps://pkgs\.tailscale\.com/(stable|unstable|release-candidate)/`) for bs.Scan() { line := bs.Bytes() if !commentLine.Match(line) { @@ -606,15 +638,15 @@ func (up *Updater) updateFedoraLike(packageManager string) func() error { } // updateYUMRepoTrack updates the repoFile file to make sure it has the -// provided track (stable or unstable) in it. +// provided track (stable, unstable, or release-candidate) in it. func updateYUMRepoTrack(repoFile, dstTrack string) (rewrote bool, err error) { was, err := os.ReadFile(repoFile) if err != nil { return false, err } - urlRe := regexp.MustCompile(`^(baseurl|gpgkey)=https://pkgs\.tailscale\.com/(un)?stable/`) - urlReplacement := fmt.Sprintf("$1=https://pkgs.tailscale.com/%s/", dstTrack) + urlRe := regexp.MustCompile(`^(baseurl|gpgkey)=https://pkgs\.tailscale\.com/(stable|unstable|release-candidate)`) + urlReplacement := fmt.Sprintf("$1=https://pkgs.tailscale.com/%s", dstTrack) s := bufio.NewScanner(bytes.NewReader(was)) newContent := bytes.NewBuffer(make([]byte, 0, len(was))) @@ -648,7 +680,7 @@ func updateYUMRepoTrack(repoFile, dstTrack string) (rewrote bool, err error) { func (up *Updater) updateAlpineLike() (err error) { if up.Version != "" { - return errors.New("installing a specific version on Alpine-based distros is not supported") + return errors.New("installing a specific version on apk-based distros is not supported") } if err := requireRoot(); err != nil { return err @@ -678,7 +710,7 @@ func (up *Updater) updateAlpineLike() (err error) { return fmt.Errorf(`failed to parse latest version from "apk info tailscale": %w`, err) } if !up.confirm(ver) { - if err := checkOutdatedAlpineRepo(up.Logf, ver, up.Track); err != nil { + if err := checkOutdatedAlpineRepo(up.Logf, apkDirPaths, ver, up.Track); err != nil { up.Logf("failed to check whether Alpine release is outdated: %v", err) } return nil @@ -718,9 +750,12 @@ func parseAlpinePackageVersion(out []byte) (string, error) { return "", errors.New("tailscale version not found in output") } -var apkRepoVersionRE = regexp.MustCompile(`v[0-9]+\.[0-9]+`) +var ( + apkRepoVersionRE = regexp.MustCompile(`v[0-9]+\.[0-9]+`) + apkDirPaths = []string{"/etc/apk/repositories", "/etc/apk/repositories.d/distfeeds.list"} +) -func checkOutdatedAlpineRepo(logf logger.Logf, apkVer, track string) error { +func checkOutdatedAlpineRepo(logf logger.Logf, filePaths []string, apkVer, track string) error { latest, err := LatestTailscaleVersion(track) if err != nil { return err @@ -729,22 +764,34 @@ func checkOutdatedAlpineRepo(logf logger.Logf, apkVer, track string) error { // Actually on latest release. return nil } - f, err := os.Open("/etc/apk/repositories") - if err != nil { - return err - } - defer f.Close() - // Read the first repo line. Typically, there are multiple repos that all - // contain the same version in the path, like: - // https://dl-cdn.alpinelinux.org/alpine/v3.20/main - // https://dl-cdn.alpinelinux.org/alpine/v3.20/community - s := bufio.NewScanner(f) - if !s.Scan() { - return s.Err() - } - alpineVer := apkRepoVersionRE.FindString(s.Text()) - if alpineVer != "" { - logf("The latest Tailscale release for Linux is %q, but your apk repository only provides %q.\nYour Alpine version is %q, you may need to upgrade the system to get the latest Tailscale version: https://wiki.alpinelinux.org/wiki/Upgrading_Alpine", latest, apkVer, alpineVer) + + // OpenWrt uses a different repo file in repositories.d, check for that as well. + for _, repoFile := range filePaths { + f, err := os.Open(repoFile) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } else { + return err + } + } + defer f.Close() + // Read the first repo line. Typically, there are multiple repos that all + // contain the same version in the path, like: + // https://dl-cdn.alpinelinux.org/alpine/v3.20/main + // https://dl-cdn.alpinelinux.org/alpine/v3.20/community + s := bufio.NewScanner(f) + if !s.Scan() { + if s.Err() != nil { + return s.Err() + } + logf("The latest Tailscale release for Linux is %q, but your apk repository only provides %q.\nYou may need to upgrade your Alpine system to get the latest Tailscale version: https://wiki.alpinelinux.org/wiki/Upgrading_Alpine", latest, apkVer) + } + alpineVer := apkRepoVersionRE.FindString(s.Text()) + if alpineVer != "" { + logf("The latest Tailscale release for Linux is %q, but your apk repository only provides %q.\nYour Alpine version is %q, you may need to upgrade the system to get the latest Tailscale version: https://wiki.alpinelinux.org/wiki/Upgrading_Alpine", latest, apkVer, alpineVer) + } + return nil } return nil } @@ -860,12 +907,17 @@ func (up *Updater) updateLinuxBinary() error { if err := os.Remove(dlPath); err != nil { up.Logf("failed to clean up %q: %v", dlPath, err) } - if err := restartSystemdUnit(context.Background()); err != nil { + + err = restartSystemdUnit(up.Logf) + if errors.Is(err, errors.ErrUnsupported) { + err = restartInitD() if errors.Is(err, errors.ErrUnsupported) { - up.Logf("Tailscale binaries updated successfully.\nPlease restart tailscaled to finish the update.") - } else { - up.Logf("Tailscale binaries updated successfully, but failed to restart tailscaled: %s.\nPlease restart tailscaled to finish the update.", err) + err = errors.New("tailscaled is not running under systemd or init.d") } + } + if err != nil { + up.Logf("Tailscale binaries updated successfully, but failed to restart tailscaled: %s.\nPlease restart tailscaled to finish the update.", err) + } else { up.Logf("Success") } @@ -873,13 +925,13 @@ func (up *Updater) updateLinuxBinary() error { return nil } -func restartSystemdUnit(ctx context.Context) error { +func restartSystemdUnit(logf logger.Logf) error { if _, err := exec.LookPath("systemctl"); err != nil { // Likely not a systemd-managed distro. return errors.ErrUnsupported } if out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput(); err != nil { - return fmt.Errorf("systemctl daemon-reload failed: %w\noutput: %s", err, out) + logf("systemctl daemon-reload failed: %w\noutput: %s", err, out) } if out, err := exec.Command("systemctl", "restart", "tailscaled.service").CombinedOutput(); err != nil { return fmt.Errorf("systemctl restart failed: %w\noutput: %s", err, out) @@ -887,6 +939,40 @@ func restartSystemdUnit(ctx context.Context) error { return nil } +// restartInitD attempts best-effort restart of tailscale on init.d systems +// (for example, GL.iNet KVM devices running busybox). It returns +// errors.ErrUnsupported if the expected service script is not found. +// +// There's probably a million variations of init.d configs out there, and this +// function does not intend to support all of them. +func restartInitD() error { + const initDir = "/etc/init.d/" + files, err := os.ReadDir(initDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return errors.ErrUnsupported + } + return err + } + for _, f := range files { + // Skip anything other than regular files. + if !f.Type().IsRegular() { + continue + } + // The script will be called something like /etc/init.d/tailscale or + // /etc/init.d/S99tailscale. + if n := f.Name(); strings.HasSuffix(n, "tailscale") { + path := filepath.Join(initDir, n) + if out, err := exec.Command(path, "restart").CombinedOutput(); err != nil { + return fmt.Errorf("%q failed: %w\noutput: %s", path+" restart", err, out) + } + return nil + } + } + // Init script for tailscale not found. + return errors.ErrUnsupported +} + func (up *Updater) downloadLinuxTarball(ver string) (string, error) { dlDir, err := os.UserCacheDir() if err != nil { @@ -1194,8 +1280,10 @@ type trackPackages struct { SPKsVersion string } +var tailscaleHTTPEndpoint = "https://pkgs.tailscale.com" + func latestPackages(track string) (*trackPackages, error) { - url := fmt.Sprintf("https://pkgs.tailscale.com/%s/?mode=json&os=%s", track, runtime.GOOS) + url := fmt.Sprintf("%s/%s/?mode=json&os=%s", tailscaleHTTPEndpoint, track, runtime.GOOS) res, err := http.Get(url) if err != nil { return nil, fmt.Errorf("fetching latest tailscale version: %w", err) @@ -1223,6 +1311,6 @@ func requireRoot() error { } func isExitError(err error) bool { - var exitErr *exec.ExitError - return errors.As(err, &exitErr) + _, ok := errors.AsType[*exec.ExitError](err) + return ok } diff --git a/clientupdate/clientupdate_downloads.go b/clientupdate/clientupdate_downloads.go index 18d3176b42afe..9458f88fe8a18 100644 --- a/clientupdate/clientupdate_downloads.go +++ b/clientupdate/clientupdate_downloads.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build (linux && !android) || windows diff --git a/clientupdate/clientupdate_gokrazy.go b/clientupdate/clientupdate_gokrazy.go new file mode 100644 index 0000000000000..d2f2dabd43723 --- /dev/null +++ b/clientupdate/clientupdate_gokrazy.go @@ -0,0 +1,177 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package clientupdate + +import ( + "archive/zip" + "context" + "fmt" + "hash/crc32" + "io" + "net" + "net/http" + "os" + "strings" + + "tailscale.com/types/logger" +) + +const ( + gokrazyUpdateSocket = "/run/gokrazy-http.sock" + gokrazyUpdateBaseURL = "http://gokrazy-local-unixsock" +) + +// GokrazyUpdateFromURL downloads a Gokrazy archive format file from args.URL, +// installs its partitions using the local gokrazy init update API, switches to +// the new root partition, and asks gokrazy to reboot. +// +// The local gokrazy API is reached over gokrazyUpdateSocket. The +// gokrazyUpdateBaseURL host is only a net/http URL sentinel; it is not resolved +// with DNS. +func init() { + GokrazyUpdateFromURL.Set(gokrazyUpdateFromURL) +} + +func gokrazyUpdateFromURL(ctx context.Context, args GokrazyUpdateArgs) error { + logf := args.Logf + if logf == nil { + logf = logger.Discard + } + if !args.AllowUnsigned { + return fmt.Errorf("signed GAF verification is not implemented yet; see https://github.com/tailscale/tailscale/issues/20002") + } + + tmp, err := os.CreateTemp("", "tailscale-gokrazy-*.gaf") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + req, err := http.NewRequestWithContext(ctx, "GET", args.URL, nil) + if err != nil { + tmp.Close() + return err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + tmp.Close() + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + tmp.Close() + return fmt.Errorf("download GAF: %s", res.Status) + } + if _, err := io.Copy(tmp, res.Body); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + + zr, err := zip.OpenReader(tmpName) + if err != nil { + return err + } + defer zr.Close() + + gokClient := gokrazyHTTPClient() + for _, part := range []struct { + name string + path string + }{ + {"root.img", "/update/root"}, + {"boot.img", "/update/boot"}, + {"mbr.img", "/update/mbr"}, + } { + if err := putGokrazyGAFMember(ctx, gokClient, zr.File, part.name, part.path); err != nil { + return err + } + logf("wrote %s", part.name) + } + if err := postGokrazy(ctx, gokClient, "/update/switch"); err != nil { + return err + } + logf("switched boot target") + if err := postGokrazy(ctx, gokClient, "/reboot?async=true&kexec_merge_cmdline=true"); err != nil { + return err + } + logf("reboot requested") + return nil +} + +func gokrazyHTTPClient() *http.Client { + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", gokrazyUpdateSocket) + } + return &http.Client{ + Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +func putGokrazyGAFMember(ctx context.Context, hc *http.Client, files []*zip.File, name, path string) error { + var zf *zip.File + for _, f := range files { + if f.Name == name { + zf = f + break + } + } + if zf == nil { + return fmt.Errorf("GAF is missing %s", name) + } + rc, err := zf.Open() + if err != nil { + return err + } + defer rc.Close() + + h := crc32.NewIEEE() + body := io.TeeReader(rc, h) + req, err := http.NewRequestWithContext(ctx, "PUT", gokrazyUpdateBaseURL+path, body) + if err != nil { + return err + } + req.ContentLength = int64(zf.UncompressedSize64) + req.Header.Set("X-Gokrazy-Update-Hash", "crc32") + res, err := hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + resBody, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("PUT %s: %s: %s", path, res.Status, strings.TrimSpace(string(resBody))) + } + if got, want := strings.TrimSpace(string(resBody)), fmt.Sprintf("%08x", h.Sum32()); got != want { + return fmt.Errorf("PUT %s: gokrazy checksum = %q; want %q", path, got, want) + } + return nil +} + +func postGokrazy(ctx context.Context, hc *http.Client, path string) error { + req, err := http.NewRequestWithContext(ctx, "POST", gokrazyUpdateBaseURL+path, nil) + if err != nil { + return err + } + res, err := hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + return fmt.Errorf("POST %s: %s: %s", path, res.Status, strings.TrimSpace(string(body))) + } + return nil +} diff --git a/clientupdate/clientupdate_not_downloads.go b/clientupdate/clientupdate_not_downloads.go index 057b4f2cd7574..aaffb76f05b3e 100644 --- a/clientupdate/clientupdate_not_downloads.go +++ b/clientupdate/clientupdate_not_downloads.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !((linux && !android) || windows) diff --git a/clientupdate/clientupdate_notwindows.go b/clientupdate/clientupdate_notwindows.go index edadc210c8a15..12035ff73495a 100644 --- a/clientupdate/clientupdate_notwindows.go +++ b/clientupdate/clientupdate_notwindows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !windows diff --git a/clientupdate/clientupdate_test.go b/clientupdate/clientupdate_test.go index b265d56411bdc..8095151c8169c 100644 --- a/clientupdate/clientupdate_test.go +++ b/clientupdate/clientupdate_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package clientupdate @@ -6,9 +6,12 @@ package clientupdate import ( "archive/tar" "compress/gzip" + "encoding/json" "fmt" "io/fs" "maps" + "net/http" + "net/http/httptest" "os" "path/filepath" "slices" @@ -86,18 +89,8 @@ func TestUpdateDebianAptSourcesListBytes(t *testing.T) { } } -func TestUpdateYUMRepoTrack(t *testing.T) { - tests := []struct { - desc string - before string - track string - after string - rewrote bool - wantErr bool - }{ - { - desc: "same track", - before: ` +var YUMRepos = map[string]string{ + StableTrack: ` [tailscale-stable] name=Tailscale stable baseurl=https://pkgs.tailscale.com/stable/fedora/$basearch @@ -107,46 +100,30 @@ repo_gpgcheck=1 gpgcheck=0 gpgkey=https://pkgs.tailscale.com/stable/fedora/repo.gpg `, - track: StableTrack, - after: ` -[tailscale-stable] -name=Tailscale stable -baseurl=https://pkgs.tailscale.com/stable/fedora/$basearch -enabled=1 -type=rpm -repo_gpgcheck=1 -gpgcheck=0 -gpgkey=https://pkgs.tailscale.com/stable/fedora/repo.gpg -`, - }, - { - desc: "change track", - before: ` -[tailscale-stable] -name=Tailscale stable -baseurl=https://pkgs.tailscale.com/stable/fedora/$basearch + + UnstableTrack: ` +[tailscale-unstable] +name=Tailscale unstable +baseurl=https://pkgs.tailscale.com/unstable/fedora/$basearch enabled=1 type=rpm repo_gpgcheck=1 gpgcheck=0 -gpgkey=https://pkgs.tailscale.com/stable/fedora/repo.gpg +gpgkey=https://pkgs.tailscale.com/unstable/fedora/repo.gpg `, - track: UnstableTrack, - after: ` -[tailscale-unstable] -name=Tailscale unstable -baseurl=https://pkgs.tailscale.com/unstable/fedora/$basearch + + ReleaseCandidateTrack: ` +[tailscale-release-candidate] +name=Tailscale release-candidate +baseurl=https://pkgs.tailscale.com/release-candidate/fedora/$basearch enabled=1 type=rpm repo_gpgcheck=1 gpgcheck=0 -gpgkey=https://pkgs.tailscale.com/unstable/fedora/repo.gpg +gpgkey=https://pkgs.tailscale.com/release-candidate/fedora/repo.gpg `, - rewrote: true, - }, - { - desc: "non-tailscale repo file", - before: ` + + "FakeRepo": ` [fedora] name=Fedora $releasever - $basearch #baseurl=http://download.example/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/ @@ -158,8 +135,41 @@ repo_gpgcheck=0 type=rpm gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch -skip_if_unavailable=False -`, +skip_if_unavailable=False`, +} + +func TestUpdateYUMRepoTrack(t *testing.T) { + tests := []struct { + desc string + before string + track string + after string + rewrote bool + wantErr bool + }{ + { + desc: "same-track", + before: YUMRepos[StableTrack], + track: StableTrack, + after: YUMRepos[StableTrack], + }, + { + desc: "change-track", + before: YUMRepos[StableTrack], + track: UnstableTrack, + after: YUMRepos[UnstableTrack], + rewrote: true, + }, + { + desc: "change-track-RC", + before: YUMRepos[StableTrack], + track: ReleaseCandidateTrack, + after: YUMRepos[ReleaseCandidateTrack], + rewrote: true, + }, + { + desc: "non-tailscale-repo-file", + before: YUMRepos["FakeRepo"], track: StableTrack, wantErr: true, }, @@ -205,7 +215,7 @@ func TestParseAlpinePackageVersion(t *testing.T) { wantErr bool }{ { - desc: "valid version", + desc: "valid-version", out: ` tailscale-1.44.2-r0 description: The easiest, most secure way to use WireGuard and 2FA @@ -219,7 +229,7 @@ tailscale-1.44.2-r0 installed size: want: "1.44.2", }, { - desc: "wrong package output", + desc: "wrong-package-output", out: ` busybox-1.36.1-r0 description: Size optimized toolbox of many common UNIX utilities @@ -233,7 +243,7 @@ busybox-1.36.1-r0 installed size: wantErr: true, }, { - desc: "missing version", + desc: "missing-version", out: ` tailscale description: The easiest, most secure way to use WireGuard and 2FA @@ -247,12 +257,12 @@ tailscale installed size: wantErr: true, }, { - desc: "empty output", + desc: "empty-output", out: "", wantErr: true, }, { - desc: "multiple versions", + desc: "multiple-versions", out: ` tailscale-1.54.1-r0 description: The easiest, most secure way to use WireGuard and 2FA @@ -292,6 +302,127 @@ tailscale-1.58.2-r0 installed size: } } +func TestCheckOutdatedAlpineRepo(t *testing.T) { + anyToString := func(a any) string { + str, ok := a.(string) + if !ok { + panic("failed to parse param as string") + } + return str + } + + tests := []struct { + name string + fileContent string + latestHTTPVersion string + latestApkVersion string + wantHTTPVersion string + wantApkVersion string + wantAlpineVersion string + track string + }{ + { + name: "up-to-date", + fileContent: "https://dl-cdn.alpinelinux.org/alpine/v3.20/main", + latestHTTPVersion: "1.95.3", + latestApkVersion: "1.95.3", + track: "unstable", + }, + { + name: "behind-unstable", + fileContent: "https://dl-cdn.alpinelinux.org/alpine/v3.20/main", + latestHTTPVersion: "1.95.4", + latestApkVersion: "1.95.3", + wantHTTPVersion: "1.95.4", + wantApkVersion: "1.95.3", + wantAlpineVersion: "v3.20", + track: "unstable", + }, + { + name: "behind-stable", + fileContent: "https://dl-cdn.alpinelinux.org/alpine/v2.40/main", + latestHTTPVersion: "1.94.3", + latestApkVersion: "1.92.1", + wantHTTPVersion: "1.94.3", + wantApkVersion: "1.92.1", + wantAlpineVersion: "v2.40", + track: "stable", + }, + { + name: "nothing-in-dist-file", + fileContent: "", + latestHTTPVersion: "1.94.3", + latestApkVersion: "1.92.1", + wantHTTPVersion: "1.94.3", + wantApkVersion: "1.92.1", + track: "stable", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := os.MkdirTemp("", "example") + if err != nil { + t.Fatalf("error creating temp dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) // clean up + + file := filepath.Join(dir, "distfile") + if err := os.WriteFile(file, []byte(tt.fileContent), 0o666); err != nil { + t.Fatalf("error creating dist file: %v", err) + } + + testServ := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + version := trackPackages{ + MSIsVersion: tt.latestHTTPVersion, + MacZipsVersion: tt.latestHTTPVersion, + TarballsVersion: tt.latestHTTPVersion, + SPKsVersion: tt.latestHTTPVersion, + } + jsonData, err := json.Marshal(version) + if err != nil { + t.Errorf("failed to marshal version string: %v", err) + } + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(jsonData); err != nil { + t.Errorf("failed to write json blob: %v", err) + } + }, + )) + defer testServ.Close() + + oldEndpoint := tailscaleHTTPEndpoint + tailscaleHTTPEndpoint = testServ.URL + defer func() { tailscaleHTTPEndpoint = oldEndpoint }() + + var paramLatest string + var paramApkVer string + var paramAlpineVer string + logf := func(_ string, params ...any) { + paramLatest = anyToString(params[0]) + paramApkVer = anyToString(params[1]) + if len(params) > 2 { + paramAlpineVer = anyToString(params[2]) + } + } + + err = checkOutdatedAlpineRepo(logf, []string{file}, tt.latestApkVersion, tt.track) + if err != nil { + t.Errorf("did not expect error, got: %v", err) + } + if paramLatest != tt.wantHTTPVersion { + t.Errorf("expected HTTP version '%s', got '%s'", tt.wantHTTPVersion, paramLatest) + } + if paramApkVer != tt.wantApkVersion { + t.Errorf("expected APK version '%s', got '%s'", tt.wantApkVersion, paramApkVer) + } + if paramAlpineVer != tt.wantAlpineVersion { + t.Errorf("expected alpine version '%s', got '%s'", tt.wantAlpineVersion, paramAlpineVer) + } + }) + } +} + func TestSynoArch(t *testing.T) { tests := []struct { goarch string @@ -320,7 +451,7 @@ func TestSynoArch(t *testing.T) { synoinfoConfPath := filepath.Join(t.TempDir(), "synoinfo.conf") if err := os.WriteFile( synoinfoConfPath, - []byte(fmt.Sprintf("unique=%q\n", tt.synoinfoUnique)), + fmt.Appendf(nil, "unique=%q\n", tt.synoinfoUnique), 0600, ); err != nil { t.Fatal(err) @@ -374,14 +505,14 @@ unique=synology_88f6281_213air want: "88f6281", }, { - desc: "missing unique", + desc: "missing-unique", content: ` company_title="Synology" `, wantErr: true, }, { - desc: "empty unique", + desc: "empty-unique", content: ` company_title="Synology" unique= @@ -389,7 +520,7 @@ unique= wantErr: true, }, { - desc: "empty unique double-quoted", + desc: "empty-unique-double-quoted", content: ` company_title="Synology" unique="" @@ -397,7 +528,7 @@ unique="" wantErr: true, }, { - desc: "empty unique single-quoted", + desc: "empty-unique-single-quoted", content: ` company_title="Synology" unique='' @@ -405,7 +536,7 @@ unique='' wantErr: true, }, { - desc: "malformed unique", + desc: "malformed-unique", content: ` company_title="Synology" unique="synology_88f6281" @@ -413,12 +544,12 @@ unique="synology_88f6281" wantErr: true, }, { - desc: "empty file", + desc: "empty-file", content: ``, wantErr: true, }, { - desc: "empty lines and comments", + desc: "empty-lines-and-comments", content: ` # In a file named synoinfo? Shocking! @@ -482,7 +613,7 @@ func TestUnpackLinuxTarball(t *testing.T) { }, }, { - desc: "don't touch unrelated files", + desc: "skip-unrelated-files", // don't touch unrelated files before: map[string]string{ "tailscale": "v1", "tailscaled": "v1", @@ -514,7 +645,7 @@ func TestUnpackLinuxTarball(t *testing.T) { }, }, { - desc: "ignore extra tarball files", + desc: "ignore-extra-tarball-files", before: map[string]string{ "tailscale": "v1", "tailscaled": "v1", @@ -530,7 +661,7 @@ func TestUnpackLinuxTarball(t *testing.T) { }, }, { - desc: "tarball missing tailscaled", + desc: "tarball-missing-tailscaled", before: map[string]string{ "tailscale": "v1", "tailscaled": "v1", @@ -546,7 +677,7 @@ func TestUnpackLinuxTarball(t *testing.T) { wantErr: true, }, { - desc: "duplicate tailscale binary", + desc: "duplicate-tailscale-binary", before: map[string]string{ "tailscale": "v1", "tailscaled": "v1", @@ -565,7 +696,7 @@ func TestUnpackLinuxTarball(t *testing.T) { wantErr: true, }, { - desc: "empty archive", + desc: "empty-archive", before: map[string]string{ "tailscale": "v1", "tailscaled": "v1", @@ -821,17 +952,18 @@ func TestCleanupOldDownloads(t *testing.T) { func TestParseUnraidPluginVersion(t *testing.T) { tests := []struct { + name string plgPath string wantVer string wantErr string }{ - {plgPath: "testdata/tailscale-1.52.0.plg", wantVer: "1.52.0"}, - {plgPath: "testdata/tailscale-1.54.0.plg", wantVer: "1.54.0"}, - {plgPath: "testdata/tailscale-nover.plg", wantErr: "version not found in plg file"}, - {plgPath: "testdata/tailscale-nover-path-mentioned.plg", wantErr: "version not found in plg file"}, + {name: "v1_52_0", plgPath: "testdata/tailscale-1.52.0.plg", wantVer: "1.52.0"}, + {name: "v1_54_0", plgPath: "testdata/tailscale-1.54.0.plg", wantVer: "1.54.0"}, + {name: "nover", plgPath: "testdata/tailscale-nover.plg", wantErr: "version not found in plg file"}, + {name: "nover-path-mentioned", plgPath: "testdata/tailscale-nover-path-mentioned.plg", wantErr: "version not found in plg file"}, } for _, tt := range tests { - t.Run(tt.plgPath, func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { got, err := parseUnraidPluginVersion(tt.plgPath) if got != tt.wantVer { t.Errorf("got version: %q, want %q", got, tt.wantVer) @@ -861,7 +993,7 @@ func TestConfirm(t *testing.T) { want bool }{ { - desc: "on latest stable", + desc: "on-latest-stable", fromTrack: StableTrack, toTrack: StableTrack, fromVer: "1.66.0", @@ -869,7 +1001,7 @@ func TestConfirm(t *testing.T) { want: false, }, { - desc: "stable upgrade", + desc: "stable-upgrade", fromTrack: StableTrack, toTrack: StableTrack, fromVer: "1.66.0", @@ -877,7 +1009,7 @@ func TestConfirm(t *testing.T) { want: true, }, { - desc: "unstable upgrade", + desc: "unstable-upgrade", fromTrack: UnstableTrack, toTrack: UnstableTrack, fromVer: "1.67.1", @@ -885,7 +1017,7 @@ func TestConfirm(t *testing.T) { want: true, }, { - desc: "from stable to unstable", + desc: "from-stable-to-unstable", fromTrack: StableTrack, toTrack: UnstableTrack, fromVer: "1.66.0", @@ -893,7 +1025,7 @@ func TestConfirm(t *testing.T) { want: true, }, { - desc: "from unstable to stable", + desc: "from-unstable-to-stable", fromTrack: UnstableTrack, toTrack: StableTrack, fromVer: "1.67.1", @@ -901,7 +1033,7 @@ func TestConfirm(t *testing.T) { want: true, }, { - desc: "confirm callback rejects", + desc: "confirm-callback-rejects", fromTrack: StableTrack, toTrack: StableTrack, fromVer: "1.66.0", @@ -912,7 +1044,7 @@ func TestConfirm(t *testing.T) { want: false, }, { - desc: "confirm callback allows", + desc: "confirm-callback-allows", fromTrack: StableTrack, toTrack: StableTrack, fromVer: "1.66.0", diff --git a/clientupdate/clientupdate_windows.go b/clientupdate/clientupdate_windows.go index b79d447ad4d30..50b77c38b4e5a 100644 --- a/clientupdate/clientupdate_windows.go +++ b/clientupdate/clientupdate_windows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Windows-specific stuff that can't go in clientupdate.go because it needs @@ -30,11 +30,6 @@ const ( // tailscale.exe process from running before the msiexec process runs and // tries to overwrite ourselves. winMSIEnv = "TS_UPDATE_WIN_MSI" - // winExePathEnv is the environment variable that is set along with - // winMSIEnv and carries the full path of the calling tailscale.exe binary. - // It is used to re-launch the GUI process (tailscale-ipn.exe) after - // install is complete. - winExePathEnv = "TS_UPDATE_WIN_EXE_PATH" // winVersionEnv is the environment variable that is set along with // winMSIEnv and carries the version of tailscale that is being installed. // It is used for logging purposes. @@ -43,12 +38,12 @@ const ( updaterPrefix = "tailscale-updater" ) -func makeSelfCopy() (origPathExe, tmpPathExe string, err error) { - selfExe, err := os.Executable() +func makeCmdTailscaleCopy() (origPathExe, tmpPathExe string, err error) { + srcExe, err := findCmdTailscale() if err != nil { return "", "", err } - f, err := os.Open(selfExe) + f, err := os.Open(srcExe) if err != nil { return "", "", err } @@ -64,7 +59,25 @@ func makeSelfCopy() (origPathExe, tmpPathExe string, err error) { f2.Close() return "", "", err } - return selfExe, f2.Name(), f2.Close() + return srcExe, f2.Name(), f2.Close() +} + +// findCmdTailscale returns the path to the binary that should be copied for the update +// re-execution. The copy is re-executed with "update" as a subcommand, so it must be +// a binary that handles "update" (ie tailscale.exe, not tailscaled.exe) +func findCmdTailscale() (string, error) { + selfExe, err := os.Executable() + if err != nil { + return "", err + } + if strings.EqualFold(filepath.Base(selfExe), "tailscale.exe") { + return selfExe, nil + } + ts := filepath.Join(filepath.Dir(selfExe), "tailscale.exe") + if _, err := os.Stat(ts); err != nil { + return "", fmt.Errorf("cannot find tailscale.exe alongside %s: %w", selfExe, err) + } + return ts, nil } func markTempFileWindows(name string) error { @@ -78,6 +91,17 @@ func verifyAuthenticode(path string) error { return authenticode.Verify(path, certSubjectTailscale) } +func isTSGUIPresent() bool { + us, err := os.Executable() + if err != nil { + return false + } + + tsgui := filepath.Join(filepath.Dir(us), "tsgui.dll") + _, err = os.Stat(tsgui) + return err == nil +} + func (up *Updater) updateWindows() error { if msi := os.Getenv(winMSIEnv); msi != "" { // stdout/stderr from this part of the install could be lost since the @@ -131,7 +155,15 @@ you can run the command prompt as Administrator one of these ways: return err } up.cleanupOldDownloads(filepath.Join(msiDir, "*.msi")) - pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s-%s.msi", up.Track, ver, arch) + + qualifiers := []string{ver, arch} + // TODO(aaron): Temporary hack so autoupdate still works on winui builds; + // remove when we enable winui by default on the unstable track. + if isTSGUIPresent() { + qualifiers = append(qualifiers, "winui") + } + + pkgsPath := fmt.Sprintf("%s/tailscale-setup-%s.msi", up.Track, strings.Join(qualifiers, "-")) msiTarget := filepath.Join(msiDir, path.Base(pkgsPath)) if err := up.downloadURLToFile(pkgsPath, msiTarget); err != nil { return err @@ -145,15 +177,15 @@ you can run the command prompt as Administrator one of these ways: up.Logf("making tailscale.exe copy to switch to...") up.cleanupOldDownloads(filepath.Join(os.TempDir(), updaterPrefix+"-*.exe")) - selfOrig, selfCopy, err := makeSelfCopy() + _, cmdTailscaleCopy, err := makeCmdTailscaleCopy() if err != nil { return err } - defer os.Remove(selfCopy) + defer os.Remove(cmdTailscaleCopy) up.Logf("running tailscale.exe copy for final install...") - cmd := exec.Command(selfCopy, "update") - cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winExePathEnv+"="+selfOrig, winVersionEnv+"="+ver) + cmd := exec.Command(cmdTailscaleCopy, "update") + cmd.Env = append(os.Environ(), winMSIEnv+"="+msiTarget, winVersionEnv+"="+ver) cmd.Stdout = up.Stderr cmd.Stderr = up.Stderr cmd.Stdin = os.Stdin @@ -189,7 +221,7 @@ func (up *Updater) installMSI(msi string) error { case windows.ERROR_SUCCESS_REBOOT_REQUIRED: // In most cases, updating Tailscale should not require a reboot. // If it does, it might be because we failed to close the GUI - // and the installer couldn't replace tailscale-ipn.exe. + // and the installer couldn't replace its executable. // The old GUI will continue to run until the next reboot. // Not ideal, but also not a retryable error. up.Logf("[unexpected] reboot required") diff --git a/clientupdate/distsign/distsign.go b/clientupdate/distsign/distsign.go index eba4b9267b119..c13ed84971462 100644 --- a/clientupdate/distsign/distsign.go +++ b/clientupdate/distsign/distsign.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package distsign implements signature and validation of arbitrary @@ -55,7 +55,8 @@ import ( "github.com/hdevalence/ed25519consensus" "golang.org/x/crypto/blake2s" - "tailscale.com/net/tshttpproxy" + "tailscale.com/feature" + "tailscale.com/net/netutil" "tailscale.com/types/logger" "tailscale.com/util/httpm" "tailscale.com/util/must" @@ -329,10 +330,16 @@ func fetch(url string, limit int64) ([]byte, error) { // download writes the response body of url into a local file at dst, up to // limit bytes. On success, the returned value is a BLAKE2s hash of the file. func (c *Client) download(ctx context.Context, url, dst string, limit int64) ([]byte, int64, error) { - tr := http.DefaultTransport.(*http.Transport).Clone() - tr.Proxy = tshttpproxy.ProxyFromEnvironment + tr := netutil.NewDefaultTransport() + tr.Proxy = feature.HookProxyFromEnvironment.GetOrNil() defer tr.CloseIdleConnections() - hc := &http.Client{Transport: tr} + hc := &http.Client{ + Transport: tr, + CheckRedirect: func(r *http.Request, via []*http.Request) error { + c.logf("Download redirected to %q", r.URL) + return nil + }, + } quickCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() diff --git a/clientupdate/distsign/distsign_test.go b/clientupdate/distsign/distsign_test.go index 09a701f499198..1380078859f3a 100644 --- a/clientupdate/distsign/distsign_test.go +++ b/clientupdate/distsign/distsign_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package distsign @@ -30,7 +30,7 @@ func TestDownload(t *testing.T) { wantErr bool }{ { - desc: "missing file", + desc: "missing-file", before: func(*testing.T) {}, src: "hello", wantErr: true, @@ -44,7 +44,7 @@ func TestDownload(t *testing.T) { want: []byte("world"), }, { - desc: "no signature", + desc: "no-signature", before: func(*testing.T) { srv.add("hello", []byte("world")) }, @@ -52,7 +52,7 @@ func TestDownload(t *testing.T) { wantErr: true, }, { - desc: "bad signature", + desc: "bad-signature", before: func(*testing.T) { srv.add("hello", []byte("world")) srv.add("hello.sig", []byte("potato")) @@ -61,7 +61,7 @@ func TestDownload(t *testing.T) { wantErr: true, }, { - desc: "signed with untrusted key", + desc: "signed-untrusted-key", before: func(t *testing.T) { srv.add("hello", []byte("world")) srv.add("hello.sig", newSigningKeyPair(t).sign([]byte("world"))) @@ -70,7 +70,7 @@ func TestDownload(t *testing.T) { wantErr: true, }, { - desc: "signed with root key", + desc: "signed-with-root-key", before: func(t *testing.T) { srv.add("hello", []byte("world")) srv.add("hello.sig", ed25519.Sign(srv.roots[0].k, []byte("world"))) @@ -79,7 +79,7 @@ func TestDownload(t *testing.T) { wantErr: true, }, { - desc: "bad signing key signature", + desc: "bad-signing-key-signature", before: func(t *testing.T) { srv.add("distsign.pub.sig", []byte("potato")) srv.addSigned("hello", []byte("world")) @@ -130,7 +130,7 @@ func TestValidateLocalBinary(t *testing.T) { wantErr bool }{ { - desc: "missing file", + desc: "missing-file", before: func(*testing.T) {}, src: "hello", wantErr: true, @@ -143,7 +143,7 @@ func TestValidateLocalBinary(t *testing.T) { src: "hello", }, { - desc: "contents changed", + desc: "contents-changed", before: func(*testing.T) { srv.addSigned("hello", []byte("new world")) }, @@ -151,7 +151,7 @@ func TestValidateLocalBinary(t *testing.T) { wantErr: true, }, { - desc: "no signature", + desc: "no-signature", before: func(*testing.T) { srv.add("hello", []byte("world")) }, @@ -159,7 +159,7 @@ func TestValidateLocalBinary(t *testing.T) { wantErr: true, }, { - desc: "bad signature", + desc: "bad-signature", before: func(*testing.T) { srv.add("hello", []byte("world")) srv.add("hello.sig", []byte("potato")) @@ -168,7 +168,7 @@ func TestValidateLocalBinary(t *testing.T) { wantErr: true, }, { - desc: "signed with untrusted key", + desc: "signed-untrusted-key", before: func(t *testing.T) { srv.add("hello", []byte("world")) srv.add("hello.sig", newSigningKeyPair(t).sign([]byte("world"))) @@ -177,7 +177,7 @@ func TestValidateLocalBinary(t *testing.T) { wantErr: true, }, { - desc: "signed with root key", + desc: "signed-with-root-key", before: func(t *testing.T) { srv.add("hello", []byte("world")) srv.add("hello.sig", ed25519.Sign(srv.roots[0].k, []byte("world"))) @@ -186,7 +186,7 @@ func TestValidateLocalBinary(t *testing.T) { wantErr: true, }, { - desc: "bad signing key signature", + desc: "bad-signing-key-signature", before: func(t *testing.T) { srv.add("distsign.pub.sig", []byte("potato")) srv.addSigned("hello", []byte("world")) @@ -341,7 +341,7 @@ func TestParseRootKey(t *testing.T) { wantErr: true, }, { - desc: "invalid PEM tag", + desc: "invalid-PEM-tag", generate: func() ([]byte, []byte, error) { priv, pub, err := GenerateRootKey() priv = bytes.Replace(priv, []byte("ROOT "), nil, -1) @@ -350,7 +350,7 @@ func TestParseRootKey(t *testing.T) { wantErr: true, }, { - desc: "not PEM", + desc: "not-PEM", generate: func() ([]byte, []byte, error) { return []byte("s3cr3t"), nil, nil }, wantErr: true, }, @@ -399,7 +399,7 @@ func TestParseSigningKey(t *testing.T) { wantErr: true, }, { - desc: "invalid PEM tag", + desc: "invalid-PEM-tag", generate: func() ([]byte, []byte, error) { priv, pub, err := GenerateSigningKey() priv = bytes.Replace(priv, []byte("SIGNING "), nil, -1) @@ -408,7 +408,7 @@ func TestParseSigningKey(t *testing.T) { wantErr: true, }, { - desc: "not PEM", + desc: "not-PEM", generate: func() ([]byte, []byte, error) { return []byte("s3cr3t"), nil, nil }, wantErr: true, }, diff --git a/clientupdate/distsign/roots.go b/clientupdate/distsign/roots.go index d5b47b7b62e92..2fab3aab90373 100644 --- a/clientupdate/distsign/roots.go +++ b/clientupdate/distsign/roots.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package distsign diff --git a/clientupdate/distsign/roots_test.go b/clientupdate/distsign/roots_test.go index 7a94529538ef1..562b06c1c29c1 100644 --- a/clientupdate/distsign/roots_test.go +++ b/clientupdate/distsign/roots_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package distsign diff --git a/cmd/addlicense/main.go b/cmd/addlicense/main.go index 1cd1b0f19354a..35d97b72f70b4 100644 --- a/cmd/addlicense/main.go +++ b/cmd/addlicense/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Program addlicense adds a license header to a file. @@ -67,7 +67,7 @@ func check(err error) { } var license = ` -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause `[1:] diff --git a/cmd/build-webclient/build-webclient.go b/cmd/build-webclient/build-webclient.go index f92c0858fae25..949d9ef349ef1 100644 --- a/cmd/build-webclient/build-webclient.go +++ b/cmd/build-webclient/build-webclient.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The build-webclient tool generates the static resources needed for the diff --git a/cmd/checkmetrics/checkmetrics.go b/cmd/checkmetrics/checkmetrics.go index fb9e8ab4c61ec..5612ffbf512f9 100644 --- a/cmd/checkmetrics/checkmetrics.go +++ b/cmd/checkmetrics/checkmetrics.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // checkmetrics validates that all metrics in the tailscale client-metrics diff --git a/cmd/cigocacher/cigocacher.go b/cmd/cigocacher/cigocacher.go new file mode 100644 index 0000000000000..74ed083679743 --- /dev/null +++ b/cmd/cigocacher/cigocacher.go @@ -0,0 +1,340 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// cigocacher is an opinionated-to-Tailscale client for gocached. It connects +// at a URL like "https://ci-gocached-azure-1.corp.ts.net:31364", but that is +// stored in a GitHub actions variable so that its hostname can be updated for +// all branches at the same time in sync with the actual infrastructure. +// +// It authenticates using GitHub OIDC tokens, and all HTTP errors are ignored +// so that its failure mode is just that builds get slower and fall back to +// disk-only cache. +package main + +import ( + "context" + jsonv1 "encoding/json" + "flag" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime/debug" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/bradfitz/go-tool-cache/cacheproc" + "github.com/bradfitz/go-tool-cache/cachers" +) + +func main() { + var ( + version = flag.Bool("version", false, "print version and exit") + auth = flag.Bool("auth", false, "auth with cigocached and exit, printing the access token as output") + stats = flag.Bool("stats", false, "fetch and print cigocached stats and exit") + token = flag.String("token", "", "the cigocached access token to use, as created using --auth") + srvURL = flag.String("cigocached-url", "", "optional cigocached URL (scheme, host, and port). Empty means to not use one.") + srvHostDial = flag.String("cigocached-host", "", "optional cigocached host to dial instead of the host in the provided --cigocached-url. Useful for public TLS certs on private addresses.") + dir = flag.String("cache-dir", "", "cache directory; empty means automatic") + verbose = flag.Bool("verbose", false, "enable verbose logging") + ) + flag.Parse() + + if *version { + info, ok := debug.ReadBuildInfo() + if !ok { + log.Fatal("no build info") + } + var ( + rev string + dirty bool + ) + for _, s := range info.Settings { + switch s.Key { + case "vcs.revision": + rev = s.Value + case "vcs.modified": + dirty, _ = strconv.ParseBool(s.Value) + } + } + if dirty { + rev += "-dirty" + } + fmt.Println(rev) + return + } + + var srvHost string + if *srvHostDial != "" && *srvURL != "" { + u, err := url.Parse(*srvURL) + if err != nil { + log.Fatal(err) + } + srvHost = u.Hostname() + } + + if *auth { + if *srvURL == "" { + log.Print("--cigocached-url is empty, skipping auth") + return + } + tk, err := fetchAccessToken(httpClient(srvHost, *srvHostDial), os.Getenv("ACTIONS_ID_TOKEN_REQUEST_URL"), os.Getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN"), *srvURL) + if err != nil { + log.Printf("error fetching access token, skipping auth: %v", err) + return + } + fmt.Println(tk) + return + } + + if *stats { + if *srvURL == "" { + log.Fatal("--cigocached-url is empty; cannot fetch stats") + } + tk := *token + if tk == "" { + log.Fatal("--token is empty; cannot fetch stats") + } + stats, err := fetchStats(httpClient(srvHost, *srvHostDial), *srvURL, tk) + if err != nil { + // Errors that are not due to misconfiguration are non-fatal so we + // don't fail builds if e.g. cigocached is down. + // + // Print error as JSON so it can still be piped through jq. + statsErr := map[string]any{ + "error": fmt.Sprintf("fetching gocached stats: %v", err), + } + b, _ := jsonv1.Marshal(statsErr) + fmt.Println(string(b)) + } else { + fmt.Println(stats) + } + + return + } + + if *dir == "" { + d, err := os.UserCacheDir() + if err != nil { + log.Fatal(err) + } + *dir = filepath.Join(d, "go-cacher") + log.Printf("Defaulting to cache dir %v ...", *dir) + } + if err := os.MkdirAll(*dir, 0750); err != nil { + log.Fatal(err) + } + + c := &cigocacher{ + disk: &cachers.DiskCache{ + Dir: *dir, + Verbose: *verbose, + }, + verbose: *verbose, + } + if *srvURL != "" { + if *verbose { + log.Printf("Using cigocached at %s", *srvURL) + } + c.remote = &cachers.HTTPClient{ + BaseURL: *srvURL, + Disk: c.disk, + HTTPClient: httpClient(srvHost, *srvHostDial), + AccessToken: *token, + Verbose: *verbose, + BestEffortHTTP: true, + } + } + var p *cacheproc.Process + p = &cacheproc.Process{ + Close: func() error { + if c.verbose { + log.Printf("gocacheprog: closing; %d gets (%d hits, %d misses, %d errors); %d puts (%d errors)", + p.Gets.Load(), p.GetHits.Load(), p.GetMisses.Load(), p.GetErrors.Load(), p.Puts.Load(), p.PutErrors.Load()) + } + return c.close() + }, + Get: c.get, + Put: c.put, + } + + if err := p.Run(); err != nil { + log.Fatal(err) + } +} + +func httpClient(srvHost, srvHostDial string) *http.Client { + if srvHost == "" || srvHostDial == "" { + return http.DefaultClient + } + return &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + if host, port, err := net.SplitHostPort(addr); err == nil && host == srvHost { + // This allows us to serve a publicly trusted TLS cert + // while also minimising latency by explicitly using a + // private network address. + addr = net.JoinHostPort(srvHostDial, port) + } + var d net.Dialer + return d.DialContext(ctx, network, addr) + }, + }, + } +} + +type cigocacher struct { + disk *cachers.DiskCache + remote *cachers.HTTPClient // nil if no remote server + verbose bool + + getNanos atomic.Int64 // total nanoseconds spent in gets + putNanos atomic.Int64 // total nanoseconds spent in puts + getHTTP atomic.Int64 // HTTP get requests made + getHTTPBytes atomic.Int64 // HTTP get bytes transferred + getHTTPHits atomic.Int64 // HTTP get hits + getHTTPMisses atomic.Int64 // HTTP get misses + getHTTPErrors atomic.Int64 // HTTP get errors ignored on best-effort basis + getHTTPNanos atomic.Int64 // total nanoseconds spent in HTTP gets + putHTTP atomic.Int64 // HTTP put requests made + putHTTPBytes atomic.Int64 // HTTP put bytes transferred + putHTTPErrors atomic.Int64 // HTTP put errors ignored on best-effort basis + putHTTPNanos atomic.Int64 // total nanoseconds spent in HTTP puts +} + +func (c *cigocacher) get(ctx context.Context, actionID string) (outputID, diskPath string, err error) { + t0 := time.Now() + defer func() { + c.getNanos.Add(time.Since(t0).Nanoseconds()) + }() + + outputID, diskPath, err = c.disk.Get(ctx, actionID) + if c.remote == nil || (err == nil && outputID != "") { + return outputID, diskPath, err + } + + // Disk miss; try remote. HTTPClient.Get handles the HTTP fetch + // (including lz4 decompression) and writes to disk for us. + c.getHTTP.Add(1) + t0HTTP := time.Now() + defer func() { + c.getHTTPNanos.Add(time.Since(t0HTTP).Nanoseconds()) + }() + outputID, diskPath, err = c.remote.Get(ctx, actionID) + if err != nil { + c.getHTTPErrors.Add(1) + return "", "", nil + } + if outputID == "" { + c.getHTTPMisses.Add(1) + return "", "", nil + } + + c.getHTTPHits.Add(1) + if fi, err := os.Stat(diskPath); err == nil { + c.getHTTPBytes.Add(fi.Size()) + } + return outputID, diskPath, nil +} + +func (c *cigocacher) put(ctx context.Context, actionID, outputID string, size int64, r io.Reader) (diskPath string, err error) { + t0 := time.Now() + defer func() { + c.putNanos.Add(time.Since(t0).Nanoseconds()) + }() + + if c.remote == nil { + return c.disk.Put(ctx, actionID, outputID, size, r) + } + + c.putHTTP.Add(1) + diskPath, err = c.remote.Put(ctx, actionID, outputID, size, r) + c.putHTTPNanos.Add(time.Since(t0).Nanoseconds()) + if err != nil { + c.putHTTPErrors.Add(1) + } else { + c.putHTTPBytes.Add(size) + } + + return diskPath, err +} + +func (c *cigocacher) close() error { + if !c.verbose || c.remote == nil { + return nil + } + + log.Printf("cigocacher HTTP stats: %d gets (%.1fMiB, %.2fs, %d hits, %d misses, %d errors ignored); %d puts (%.1fMiB, %.2fs, %d errors ignored)", + c.getHTTP.Load(), float64(c.getHTTPBytes.Load())/float64(1<<20), float64(c.getHTTPNanos.Load())/float64(time.Second), c.getHTTPHits.Load(), c.getHTTPMisses.Load(), c.getHTTPErrors.Load(), + c.putHTTP.Load(), float64(c.putHTTPBytes.Load())/float64(1<<20), float64(c.putHTTPNanos.Load())/float64(time.Second), c.putHTTPErrors.Load()) + + stats, err := fetchStats(c.remote.HTTPClient, c.remote.BaseURL, c.remote.AccessToken) + if err != nil { + log.Printf("error fetching gocached stats: %v", err) + } else { + log.Printf("gocached session stats: %s", stats) + } + + return nil +} + +func fetchAccessToken(cl *http.Client, idTokenURL, idTokenRequestToken, gocachedURL string) (string, error) { + req, err := http.NewRequest("GET", idTokenURL+"&audience=gocached", nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Bearer "+idTokenRequestToken) + resp, err := cl.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + type idTokenResp struct { + Value string `json:"value"` + } + var idToken idTokenResp + if err := jsonv1.NewDecoder(resp.Body).Decode(&idToken); err != nil { + return "", err + } + + req, _ = http.NewRequest("POST", gocachedURL+"/auth/exchange-token", strings.NewReader(`{"jwt":"`+idToken.Value+`"}`)) + req.Header.Set("Content-Type", "application/json") + resp, err = cl.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + type accessTokenResp struct { + AccessToken string `json:"access_token"` + } + var accessToken accessTokenResp + if err := jsonv1.NewDecoder(resp.Body).Decode(&accessToken); err != nil { + return "", err + } + + return accessToken.AccessToken, nil +} + +func fetchStats(cl *http.Client, baseURL, accessToken string) (string, error) { + req, _ := http.NewRequest("GET", baseURL+"/session/stats", nil) + req.Header.Set("Authorization", "Bearer "+accessToken) + resp, err := cl.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("fetching stats: %s", resp.Status) + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/cmd/cloner/cloner.go b/cmd/cloner/cloner.go index a1ffc30feafb2..8b4cacf7a8849 100644 --- a/cmd/cloner/cloner.go +++ b/cmd/cloner/cloner.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Cloner is a tool to automate the creation of a Clone method. @@ -121,7 +121,18 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { continue } if !hasBasicUnderlying(ft) { - writef("dst.%s = *src.%s.Clone()", fname, fname) + // don't dereference if the underlying type is an interface + if _, isInterface := ft.Underlying().(*types.Interface); isInterface { + writef("if src.%s != nil { dst.%s = src.%s.Clone() }", fname, fname, fname) + } else { + writef("dst.%s = *src.%s.Clone()", fname, fname) + } + continue + } + // Named types with basic underlying types (map/slice) that + // have their own Clone method should use it directly. + if methodResultType(ft, "Clone") != nil { + writef("dst.%s = src.%s.Clone()", fname, fname) continue } } @@ -132,27 +143,9 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { writef("if src.%s != nil {", fname) writef("dst.%s = make([]%s, len(src.%s))", fname, n, fname) writef("for i := range dst.%s {", fname) - if ptr, isPtr := ft.Elem().(*types.Pointer); isPtr { - writef("if src.%s[i] == nil { dst.%s[i] = nil } else {", fname, fname) - if codegen.ContainsPointers(ptr.Elem()) { - if _, isIface := ptr.Elem().Underlying().(*types.Interface); isIface { - it.Import("tailscale.com/types/ptr") - writef("\tdst.%s[i] = ptr.To((*src.%s[i]).Clone())", fname, fname) - } else { - writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname) - } - } else { - it.Import("tailscale.com/types/ptr") - writef("\tdst.%s[i] = ptr.To(*src.%s[i])", fname, fname) - } - writef("}") - } else if ft.Elem().String() == "encoding/json.RawMessage" { - writef("\tdst.%s[i] = append(src.%s[i][:0:0], src.%s[i]...)", fname, fname, fname) - } else if _, isIface := ft.Elem().Underlying().(*types.Interface); isIface { - writef("\tdst.%s[i] = src.%s[i].Clone()", fname, fname) - } else { - writef("\tdst.%s[i] = *src.%s[i].Clone()", fname, fname) - } + writeSliceElemClone(writef, ft.Elem(), + fmt.Sprintf("src.%s[i]", fname), + fmt.Sprintf("dst.%s[i]", fname)) writef("}") writef("}") } else { @@ -165,12 +158,11 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { writef("dst.%s = src.%s.Clone()", fname, fname) continue } - it.Import("tailscale.com/types/ptr") writef("if dst.%s != nil {", fname) if _, isIface := base.Underlying().(*types.Interface); isIface && hasPtrs { - writef("\tdst.%s = ptr.To((*src.%s).Clone())", fname, fname) + writef("\tdst.%s = new((*src.%s).Clone())", fname, fname) } else if !hasPtrs { - writef("\tdst.%s = ptr.To(*src.%s)", fname, fname) + writef("\tdst.%s = new(*src.%s)", fname, fname) } else { writef("\t" + `panic("TODO pointers in pointers")`) } @@ -181,51 +173,57 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { n := it.QualifiedName(sliceType.Elem()) writef("if dst.%s != nil {", fname) writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem)) - writef("\tfor k := range src.%s {", fname) - // use zero-length slice instead of nil to ensure - // the key is always copied. - writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname) - writef("\t}") + if codegen.ContainsPointers(sliceType.Elem()) { + writef("\tfor k, sv := range src.%s {", fname) + writef("\t\tif sv == nil {") + writef("\t\t\tdst.%s[k] = nil", fname) + writef("\t\t\tcontinue") + writef("\t\t}") + writef("\t\tdst.%s[k] = make([]%s, len(sv))", fname, n) + writef("\t\tfor i := range sv {") + innerWritef := func(format string, args ...any) { + writef("\t\t"+format, args...) + } + writeSliceElemClone(innerWritef, sliceType.Elem(), + "sv[i]", fmt.Sprintf("dst.%s[k][i]", fname)) + writef("\t\t}") + writef("\t}") + } else { + writef("\tfor k := range src.%s {", fname) + // use zero-length slice instead of nil to ensure + // the key is always copied. + writef("\t\tdst.%s[k] = append([]%s{}, src.%s[k]...)", fname, n, fname) + writef("\t}") + } writef("}") - } else if codegen.ContainsPointers(elem) { + } else if codegen.IsViewType(elem) || !codegen.ContainsPointers(elem) { + // If the map values are view types (which are + // immutable and don't need cloning) or don't + // themselves contain pointers, we can just + // clone the map itself. + it.Import("", "maps") + writef("\tdst.%s = maps.Clone(src.%s)", fname, fname) + } else { + // Otherwise we need to clone each element of + // the map using our recursive helper. writef("if dst.%s != nil {", fname) writef("\tdst.%s = map[%s]%s{}", fname, it.QualifiedName(ft.Key()), it.QualifiedName(elem)) writef("\tfor k, v := range src.%s {", fname) - switch elem := elem.Underlying().(type) { - case *types.Pointer: - writef("\t\tif v == nil { dst.%s[k] = nil } else {", fname) - if base := elem.Elem().Underlying(); codegen.ContainsPointers(base) { - if _, isIface := base.(*types.Interface); isIface { - it.Import("tailscale.com/types/ptr") - writef("\t\t\tdst.%s[k] = ptr.To((*v).Clone())", fname) - } else { - writef("\t\t\tdst.%s[k] = v.Clone()", fname) - } - } else { - it.Import("tailscale.com/types/ptr") - writef("\t\t\tdst.%s[k] = ptr.To(*v)", fname) - } - writef("}") - case *types.Interface: - if cloneResultType := methodResultType(elem, "Clone"); cloneResultType != nil { - if _, isPtr := cloneResultType.(*types.Pointer); isPtr { - writef("\t\tdst.%s[k] = *(v.Clone())", fname) - } else { - writef("\t\tdst.%s[k] = v.Clone()", fname) - } - } else { - writef(`panic("%s (%v) does not have a Clone method")`, fname, elem) - } - default: - writef("\t\tdst.%s[k] = *(v.Clone())", fname) - } - + // Use a recursive helper here; this handles + // arbitrarily nested maps in addition to + // simpler types. + writeMapValueClone(mapValueCloneParams{ + Buf: buf, + It: it, + Elem: elem, + SrcExpr: "v", + DstExpr: fmt.Sprintf("dst.%s[k]", fname), + BaseIndent: "\t", + Depth: 1, + }) writef("\t}") writef("}") - } else { - it.Import("maps") - writef("\tdst.%s = maps.Clone(src.%s)", fname, fname) } case *types.Interface: // If ft is an interface with a "Clone() ft" method, it can be used to clone the field. @@ -245,6 +243,31 @@ func gen(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named) { buf.Write(codegen.AssertStructUnchanged(t, name, typeParams, "Clone", it)) } +// writeSliceElemClone generates code to deep-clone a single slice element +// from srcExpr to dstExpr. It handles pointer, json.RawMessage, interface, +// and named struct element types. +func writeSliceElemClone(writef func(string, ...any), elemType types.Type, srcExpr, dstExpr string) { + if ptr, isPtr := elemType.(*types.Pointer); isPtr { + writef("if %s == nil { %s = nil } else {", srcExpr, dstExpr) + if codegen.ContainsPointers(ptr.Elem()) { + if _, isIface := ptr.Elem().Underlying().(*types.Interface); isIface { + writef("\t%s = new((*%s).Clone())", dstExpr, srcExpr) + } else { + writef("\t%s = %s.Clone()", dstExpr, srcExpr) + } + } else { + writef("\t%s = new(*%s)", dstExpr, srcExpr) + } + writef("}") + } else if elemType.String() == "encoding/json.RawMessage" { + writef("%s = append(%s[:0:0], %s...)", dstExpr, srcExpr, srcExpr) + } else if _, isIface := elemType.Underlying().(*types.Interface); isIface { + writef("%s = %s.Clone()", dstExpr, srcExpr) + } else { + writef("%s = *%s.Clone()", dstExpr, srcExpr) + } +} + // hasBasicUnderlying reports true when typ.Underlying() is a slice or a map. func hasBasicUnderlying(typ types.Type) bool { switch typ.Underlying().(type) { @@ -266,3 +289,97 @@ func methodResultType(typ types.Type, method string) types.Type { } return sig.Results().At(0).Type() } + +type mapValueCloneParams struct { + // Buf is the buffer to write generated code to + Buf *bytes.Buffer + // It is the import tracker for managing imports. + It *codegen.ImportTracker + // Elem is the type of the map value to clone + Elem types.Type + // SrcExpr is the expression for the source value (e.g., "v", "v2", "v3") + SrcExpr string + // DstExpr is the expression for the destination (e.g., "dst.Field[k]", "dst.Field[k][k2]") + DstExpr string + // BaseIndent is the "base" indentation string for the generated code + // (i.e. 1 or more tabs). Additional indentation will be added based on + // the Depth parameter. + BaseIndent string + // Depth is the current nesting depth (1 for first level, 2 for second, etc.) + Depth int +} + +// writeMapValueClone generates code to clone a map value recursively. +// It handles arbitrary nesting of maps, pointers, and interfaces. +func writeMapValueClone(params mapValueCloneParams) { + indent := params.BaseIndent + strings.Repeat("\t", params.Depth) + writef := func(format string, args ...any) { + fmt.Fprintf(params.Buf, indent+format+"\n", args...) + } + + switch elem := params.Elem.Underlying().(type) { + case *types.Pointer: + writef("if %s == nil { %s = nil } else {", params.SrcExpr, params.DstExpr) + if base := elem.Elem().Underlying(); codegen.ContainsPointers(base) { + if _, isIface := base.(*types.Interface); isIface { + writef("\t%s = new((*%s).Clone())", params.DstExpr, params.SrcExpr) + } else { + writef("\t%s = %s.Clone()", params.DstExpr, params.SrcExpr) + } + } else { + writef("\t%s = new(*%s)", params.DstExpr, params.SrcExpr) + } + writef("}") + + case *types.Map: + // Recursively handle nested maps + innerElem := elem.Elem() + if codegen.IsViewType(innerElem) || !codegen.ContainsPointers(innerElem) { + // Inner map values don't need deep cloning + params.It.Import("", "maps") + writef("%s = maps.Clone(%s)", params.DstExpr, params.SrcExpr) + } else { + // Inner map values need cloning + keyType := params.It.QualifiedName(elem.Key()) + valueType := params.It.QualifiedName(innerElem) + // Generate unique variable names for nested loops based on depth + keyVar := fmt.Sprintf("k%d", params.Depth+1) + valVar := fmt.Sprintf("v%d", params.Depth+1) + + writef("if %s == nil {", params.SrcExpr) + writef("\t%s = nil", params.DstExpr) + writef("\tcontinue") + writef("}") + writef("%s = map[%s]%s{}", params.DstExpr, keyType, valueType) + writef("for %s, %s := range %s {", keyVar, valVar, params.SrcExpr) + + // Recursively generate cloning code for the nested map value + nestedDstExpr := fmt.Sprintf("%s[%s]", params.DstExpr, keyVar) + writeMapValueClone(mapValueCloneParams{ + Buf: params.Buf, + It: params.It, + Elem: innerElem, + SrcExpr: valVar, + DstExpr: nestedDstExpr, + BaseIndent: params.BaseIndent, + Depth: params.Depth + 1, + }) + + writef("}") + } + + case *types.Interface: + if cloneResultType := methodResultType(elem, "Clone"); cloneResultType != nil { + if _, isPtr := cloneResultType.(*types.Pointer); isPtr { + writef("%s = *(%s.Clone())", params.DstExpr, params.SrcExpr) + } else { + writef("%s = %s.Clone()", params.DstExpr, params.SrcExpr) + } + } else { + writef(`panic("map value (%%v) does not have a Clone method")`, elem) + } + + default: + writef("%s = *(%s.Clone())", params.DstExpr, params.SrcExpr) + } +} diff --git a/cmd/cloner/cloner_test.go b/cmd/cloner/cloner_test.go index cf1063714afda..f8beb4a88b952 100644 --- a/cmd/cloner/cloner_test.go +++ b/cmd/cloner/cloner_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -7,6 +7,7 @@ import ( "reflect" "testing" + "github.com/google/go-cmp/cmp" "tailscale.com/cmd/cloner/clonerex" ) @@ -59,3 +60,226 @@ func TestSliceContainer(t *testing.T) { }) } } + +func TestInterfaceContainer(t *testing.T) { + examples := []struct { + name string + in *clonerex.InterfaceContainer + }{ + { + name: "nil", + in: nil, + }, + { + name: "zero", + in: &clonerex.InterfaceContainer{}, + }, + { + name: "with_interface", + in: &clonerex.InterfaceContainer{ + Interface: &clonerex.CloneableImpl{Value: 42}, + }, + }, + { + name: "with_nil_interface", + in: &clonerex.InterfaceContainer{ + Interface: nil, + }, + }, + } + + for _, ex := range examples { + t.Run(ex.name, func(t *testing.T) { + out := ex.in.Clone() + if !reflect.DeepEqual(ex.in, out) { + t.Errorf("Clone() = %v, want %v", out, ex.in) + } + + // Verify no aliasing: modifying the clone should not affect the original + if ex.in != nil && ex.in.Interface != nil { + if impl, ok := out.Interface.(*clonerex.CloneableImpl); ok { + impl.Value = 999 + if origImpl, ok := ex.in.Interface.(*clonerex.CloneableImpl); ok { + if origImpl.Value == 999 { + t.Errorf("Clone() aliased memory with original") + } + } + } + } + }) + } +} + +func TestMapWithPointers(t *testing.T) { + num1, num2 := 42, 100 + orig := &clonerex.MapWithPointers{ + Nested: map[string]*int{ + "foo": &num1, + "bar": &num2, + }, + WithCloneMethod: map[string]*clonerex.SliceContainer{ + "container1": {Slice: []*int{&num1, &num2}}, + "container2": {Slice: []*int{&num1}}, + }, + CloneInterface: map[string]clonerex.Cloneable{ + "impl1": &clonerex.CloneableImpl{Value: 123}, + "impl2": &clonerex.CloneableImpl{Value: 456}, + }, + } + + cloned := orig.Clone() + if !reflect.DeepEqual(orig, cloned) { + t.Errorf("Clone() = %v, want %v", cloned, orig) + } + + // Mutate cloned.Nested pointer values + *cloned.Nested["foo"] = 999 + if *orig.Nested["foo"] == 999 { + t.Errorf("Clone() aliased memory in Nested: original was modified") + } + + // Mutate cloned.WithCloneMethod slice values + *cloned.WithCloneMethod["container1"].Slice[0] = 888 + if *orig.WithCloneMethod["container1"].Slice[0] == 888 { + t.Errorf("Clone() aliased memory in WithCloneMethod: original was modified") + } + + // Mutate cloned.CloneInterface values + if impl, ok := cloned.CloneInterface["impl1"].(*clonerex.CloneableImpl); ok { + impl.Value = 777 + if origImpl, ok := orig.CloneInterface["impl1"].(*clonerex.CloneableImpl); ok { + if origImpl.Value == 777 { + t.Errorf("Clone() aliased memory in CloneInterface: original was modified") + } + } + } +} + +func TestNamedMapContainer(t *testing.T) { + orig := &clonerex.NamedMapContainer{ + Attrs: clonerex.NamedMap{ + "str": "hello", + "num": int64(42), + "bool": true, + }, + } + + cloned := orig.Clone() + if !reflect.DeepEqual(orig, cloned) { + t.Errorf("Clone() = %v, want %v", cloned, orig) + } + + // Mutate the cloned map to verify no aliasing. + cloned.Attrs["str"] = "modified" + if orig.Attrs["str"] == "modified" { + t.Errorf("Clone() aliased memory in Attrs: original was modified") + } + + // Verify nil handling. + nilContainer := &clonerex.NamedMapContainer{} + nilClone := nilContainer.Clone() + if !reflect.DeepEqual(nilContainer, nilClone) { + t.Errorf("Clone() of nil Attrs = %v, want %v", nilClone, nilContainer) + } +} + +func TestMapSlicePointerContainer(t *testing.T) { + num := 42 + orig := &clonerex.MapSlicePointerContainer{ + Routes: map[string][]*clonerex.SliceContainer{ + "route1": { + {Slice: []*int{&num}}, + {Slice: []*int{&num, &num}}, + }, + "route2": { + {Slice: []*int{&num}}, + }, + }, + } + + cloned := orig.Clone() + if !reflect.DeepEqual(orig, cloned) { + t.Errorf("Clone() = %v, want %v", cloned, orig) + } + + // Mutate cloned.Routes pointer values + *cloned.Routes["route1"][0].Slice[0] = 999 + if *orig.Routes["route1"][0].Slice[0] == 999 { + t.Errorf("Clone() aliased memory in Routes: original was modified") + } +} + +func TestMapSlicePointerContainerNilValue(t *testing.T) { + num := 7 + orig := &clonerex.MapSlicePointerContainer{ + Routes: map[string][]*clonerex.SliceContainer{ + "nil-value": nil, + "non-nil": {{Slice: []*int{&num}}}, + }, + } + cloned := orig.Clone() + if diff := cmp.Diff(orig.Routes, cloned.Routes); diff != "" { + t.Errorf("Clone() Routes mismatch (-orig +cloned):\n%s", diff) + } +} + +func TestDeeplyNestedMap(t *testing.T) { + num := 123 + orig := &clonerex.DeeplyNestedMap{ + ThreeLevels: map[string]map[string]map[string]int{ + "a": { + "b": {"c": 1, "d": 2}, + "e": {"f": 3}, + }, + "g": { + "h": {"i": 4}, + }, + }, + FourLevels: map[string]map[string]map[string]map[string]*clonerex.SliceContainer{ + "l1a": { + "l2a": { + "l3a": { + "l4a": {Slice: []*int{&num}}, + "l4b": {Slice: []*int{&num, &num}}, + }, + }, + }, + }, + } + + cloned := orig.Clone() + if !reflect.DeepEqual(orig, cloned) { + t.Errorf("Clone() = %v, want %v", cloned, orig) + } + + // Mutate the clone's ThreeLevels map + cloned.ThreeLevels["a"]["b"]["c"] = 777 + if orig.ThreeLevels["a"]["b"]["c"] == 777 { + t.Errorf("Clone() aliased memory in ThreeLevels: original was modified") + } + + // Mutate the clone's FourLevels map at the deepest pointer level + *cloned.FourLevels["l1a"]["l2a"]["l3a"]["l4a"].Slice[0] = 666 + if *orig.FourLevels["l1a"]["l2a"]["l3a"]["l4a"].Slice[0] == 666 { + t.Errorf("Clone() aliased memory in FourLevels: original was modified") + } + + // Add a new top-level key to the clone's FourLevels map + newNum := 999 + cloned.FourLevels["l1b"] = map[string]map[string]map[string]*clonerex.SliceContainer{ + "l2b": { + "l3b": { + "l4c": {Slice: []*int{&newNum}}, + }, + }, + } + if _, exists := orig.FourLevels["l1b"]; exists { + t.Errorf("Clone() aliased FourLevels map: new top-level key appeared in original") + } + + // Add a new nested key to the clone's FourLevels map + cloned.FourLevels["l1a"]["l2a"]["l3a"]["l4c"] = &clonerex.SliceContainer{Slice: []*int{&newNum}} + if _, exists := orig.FourLevels["l1a"]["l2a"]["l3a"]["l4c"]; exists { + t.Errorf("Clone() aliased FourLevels map: new nested key appeared in original") + } +} diff --git a/cmd/cloner/clonerex/clonerex.go b/cmd/cloner/clonerex/clonerex.go index 96bf8a0bd6e9d..41626d3ae8b45 100644 --- a/cmd/cloner/clonerex/clonerex.go +++ b/cmd/cloner/clonerex/clonerex.go @@ -1,7 +1,7 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -//go:generate go run tailscale.com/cmd/cloner -clonefunc=true -type SliceContainer +//go:generate go run tailscale.com/cmd/cloner -clonefunc=true -type SliceContainer,InterfaceContainer,MapWithPointers,DeeplyNestedMap,NamedMapContainer,MapSlicePointerContainer // Package clonerex is an example package for the cloner tool. package clonerex @@ -9,3 +9,66 @@ package clonerex type SliceContainer struct { Slice []*int } + +// Cloneable is an interface with a Clone method. +type Cloneable interface { + Clone() Cloneable +} + +// CloneableImpl is a concrete type that implements Cloneable. +type CloneableImpl struct { + Value int +} + +func (c *CloneableImpl) Clone() Cloneable { + if c == nil { + return nil + } + return &CloneableImpl{Value: c.Value} +} + +// InterfaceContainer has a pointer to an interface field, which tests +// the special handling for interface types in the cloner. +type InterfaceContainer struct { + Interface Cloneable +} + +type MapWithPointers struct { + Nested map[string]*int + WithCloneMethod map[string]*SliceContainer + CloneInterface map[string]Cloneable +} + +// NamedMap is a named map type with its own Clone method. +// This tests that the cloner uses the type's Clone method +// rather than trying to descend into the map's value type. +type NamedMap map[string]any + +func (m NamedMap) Clone() NamedMap { + if m == nil { + return nil + } + m2 := make(NamedMap, len(m)) + for k, v := range m { + m2[k] = v + } + return m2 +} + +// NamedMapContainer has a field whose type is a named map with a Clone method. +type NamedMapContainer struct { + Attrs NamedMap +} + +// MapSlicePointerContainer has a map whose values are slices of pointers. +// This tests that the cloner deep-clones the pointer elements in the slice, +// not just the slice itself (which would leave aliased pointers). +type MapSlicePointerContainer struct { + Routes map[string][]*SliceContainer +} + +// DeeplyNestedMap tests arbitrary depth of map nesting (3+ levels) +type DeeplyNestedMap struct { + ThreeLevels map[string]map[string]map[string]int + FourLevels map[string]map[string]map[string]map[string]*SliceContainer +} diff --git a/cmd/cloner/clonerex/clonerex_clone.go b/cmd/cloner/clonerex/clonerex_clone.go index e334a4e3a1bf4..9a4413177bb47 100644 --- a/cmd/cloner/clonerex/clonerex_clone.go +++ b/cmd/cloner/clonerex/clonerex_clone.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Code generated by tailscale.com/cmd/cloner; DO NOT EDIT. @@ -6,7 +6,7 @@ package clonerex import ( - "tailscale.com/types/ptr" + "maps" ) // Clone makes a deep copy of SliceContainer. @@ -23,7 +23,7 @@ func (src *SliceContainer) Clone() *SliceContainer { if src.Slice[i] == nil { dst.Slice[i] = nil } else { - dst.Slice[i] = ptr.To(*src.Slice[i]) + dst.Slice[i] = new(*src.Slice[i]) } } } @@ -35,9 +35,183 @@ var _SliceContainerCloneNeedsRegeneration = SliceContainer(struct { Slice []*int }{}) +// Clone makes a deep copy of InterfaceContainer. +// The result aliases no memory with the original. +func (src *InterfaceContainer) Clone() *InterfaceContainer { + if src == nil { + return nil + } + dst := new(InterfaceContainer) + *dst = *src + if src.Interface != nil { + dst.Interface = src.Interface.Clone() + } + return dst +} + +// A compilation failure here means this code must be regenerated, with the command at the top of this file. +var _InterfaceContainerCloneNeedsRegeneration = InterfaceContainer(struct { + Interface Cloneable +}{}) + +// Clone makes a deep copy of MapWithPointers. +// The result aliases no memory with the original. +func (src *MapWithPointers) Clone() *MapWithPointers { + if src == nil { + return nil + } + dst := new(MapWithPointers) + *dst = *src + if dst.Nested != nil { + dst.Nested = map[string]*int{} + for k, v := range src.Nested { + if v == nil { + dst.Nested[k] = nil + } else { + dst.Nested[k] = new(*v) + } + } + } + if dst.WithCloneMethod != nil { + dst.WithCloneMethod = map[string]*SliceContainer{} + for k, v := range src.WithCloneMethod { + if v == nil { + dst.WithCloneMethod[k] = nil + } else { + dst.WithCloneMethod[k] = v.Clone() + } + } + } + if dst.CloneInterface != nil { + dst.CloneInterface = map[string]Cloneable{} + for k, v := range src.CloneInterface { + dst.CloneInterface[k] = v.Clone() + } + } + return dst +} + +// A compilation failure here means this code must be regenerated, with the command at the top of this file. +var _MapWithPointersCloneNeedsRegeneration = MapWithPointers(struct { + Nested map[string]*int + WithCloneMethod map[string]*SliceContainer + CloneInterface map[string]Cloneable +}{}) + +// Clone makes a deep copy of DeeplyNestedMap. +// The result aliases no memory with the original. +func (src *DeeplyNestedMap) Clone() *DeeplyNestedMap { + if src == nil { + return nil + } + dst := new(DeeplyNestedMap) + *dst = *src + if dst.ThreeLevels != nil { + dst.ThreeLevels = map[string]map[string]map[string]int{} + for k, v := range src.ThreeLevels { + if v == nil { + dst.ThreeLevels[k] = nil + continue + } + dst.ThreeLevels[k] = map[string]map[string]int{} + for k2, v2 := range v { + dst.ThreeLevels[k][k2] = maps.Clone(v2) + } + } + } + if dst.FourLevels != nil { + dst.FourLevels = map[string]map[string]map[string]map[string]*SliceContainer{} + for k, v := range src.FourLevels { + if v == nil { + dst.FourLevels[k] = nil + continue + } + dst.FourLevels[k] = map[string]map[string]map[string]*SliceContainer{} + for k2, v2 := range v { + if v2 == nil { + dst.FourLevels[k][k2] = nil + continue + } + dst.FourLevels[k][k2] = map[string]map[string]*SliceContainer{} + for k3, v3 := range v2 { + if v3 == nil { + dst.FourLevels[k][k2][k3] = nil + continue + } + dst.FourLevels[k][k2][k3] = map[string]*SliceContainer{} + for k4, v4 := range v3 { + if v4 == nil { + dst.FourLevels[k][k2][k3][k4] = nil + } else { + dst.FourLevels[k][k2][k3][k4] = v4.Clone() + } + } + } + } + } + } + return dst +} + +// A compilation failure here means this code must be regenerated, with the command at the top of this file. +var _DeeplyNestedMapCloneNeedsRegeneration = DeeplyNestedMap(struct { + ThreeLevels map[string]map[string]map[string]int + FourLevels map[string]map[string]map[string]map[string]*SliceContainer +}{}) + +// Clone makes a deep copy of NamedMapContainer. +// The result aliases no memory with the original. +func (src *NamedMapContainer) Clone() *NamedMapContainer { + if src == nil { + return nil + } + dst := new(NamedMapContainer) + *dst = *src + dst.Attrs = src.Attrs.Clone() + return dst +} + +// A compilation failure here means this code must be regenerated, with the command at the top of this file. +var _NamedMapContainerCloneNeedsRegeneration = NamedMapContainer(struct { + Attrs NamedMap +}{}) + +// Clone makes a deep copy of MapSlicePointerContainer. +// The result aliases no memory with the original. +func (src *MapSlicePointerContainer) Clone() *MapSlicePointerContainer { + if src == nil { + return nil + } + dst := new(MapSlicePointerContainer) + *dst = *src + if dst.Routes != nil { + dst.Routes = map[string][]*SliceContainer{} + for k, sv := range src.Routes { + if sv == nil { + dst.Routes[k] = nil + continue + } + dst.Routes[k] = make([]*SliceContainer, len(sv)) + for i := range sv { + if sv[i] == nil { + dst.Routes[k][i] = nil + } else { + dst.Routes[k][i] = sv[i].Clone() + } + } + } + } + return dst +} + +// A compilation failure here means this code must be regenerated, with the command at the top of this file. +var _MapSlicePointerContainerCloneNeedsRegeneration = MapSlicePointerContainer(struct { + Routes map[string][]*SliceContainer +}{}) + // Clone duplicates src into dst and reports whether it succeeded. // To succeed, must be of types <*T, *T> or <*T, **T>, -// where T is one of SliceContainer. +// where T is one of SliceContainer,InterfaceContainer,MapWithPointers,DeeplyNestedMap,NamedMapContainer,MapSlicePointerContainer. func Clone(dst, src any) bool { switch src := src.(type) { case *SliceContainer: @@ -49,6 +223,51 @@ func Clone(dst, src any) bool { *dst = src.Clone() return true } + case *InterfaceContainer: + switch dst := dst.(type) { + case *InterfaceContainer: + *dst = *src.Clone() + return true + case **InterfaceContainer: + *dst = src.Clone() + return true + } + case *MapWithPointers: + switch dst := dst.(type) { + case *MapWithPointers: + *dst = *src.Clone() + return true + case **MapWithPointers: + *dst = src.Clone() + return true + } + case *DeeplyNestedMap: + switch dst := dst.(type) { + case *DeeplyNestedMap: + *dst = *src.Clone() + return true + case **DeeplyNestedMap: + *dst = src.Clone() + return true + } + case *NamedMapContainer: + switch dst := dst.(type) { + case *NamedMapContainer: + *dst = *src.Clone() + return true + case **NamedMapContainer: + *dst = src.Clone() + return true + } + case *MapSlicePointerContainer: + switch dst := dst.(type) { + case *MapSlicePointerContainer: + *dst = *src.Clone() + return true + case **MapSlicePointerContainer: + *dst = src.Clone() + return true + } } return false } diff --git a/cmd/connector-gen/advertise-routes.go b/cmd/connector-gen/advertise-routes.go index 446f4906a4d65..57c101e27af7c 100644 --- a/cmd/connector-gen/advertise-routes.go +++ b/cmd/connector-gen/advertise-routes.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/connector-gen/aws.go b/cmd/connector-gen/aws.go index bd2632ae27960..b0d6566b915f6 100644 --- a/cmd/connector-gen/aws.go +++ b/cmd/connector-gen/aws.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/connector-gen/connector-gen.go b/cmd/connector-gen/connector-gen.go index 6947f6410a96f..8693a1bf0490f 100644 --- a/cmd/connector-gen/connector-gen.go +++ b/cmd/connector-gen/connector-gen.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // connector-gen is a tool to generate app connector configuration and flags from service provider address data. diff --git a/cmd/connector-gen/github.go b/cmd/connector-gen/github.go index def40872d52c1..a0162aa06cae3 100644 --- a/cmd/connector-gen/github.go +++ b/cmd/connector-gen/github.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/containerboot/egressservices.go b/cmd/containerboot/egressservices.go index 71141f17a9bb6..c8b8a770af572 100644 --- a/cmd/containerboot/egressservices.go +++ b/cmd/containerboot/egressservices.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -18,16 +18,16 @@ import ( "reflect" "strconv" "strings" + "sync" "time" "github.com/fsnotify/fsnotify" + "tailscale.com/client/local" - "tailscale.com/ipn" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" - "tailscale.com/syncs" - "tailscale.com/tailcfg" + "tailscale.com/types/views" "tailscale.com/util/httpm" "tailscale.com/util/linuxfw" "tailscale.com/util/mak" @@ -55,7 +55,7 @@ type egressProxy struct { tsClient *local.Client // never nil - netmapChan chan ipn.Notify // chan to receive netmap updates on + netmapChan chan netmapState // chan to receive netmap state updates on podIPv4 string // never empty string, currently only IPv4 is supported @@ -87,7 +87,7 @@ type httpClient interface { // - the mounted egress config has changed // - the proxy's tailnet IP addresses have changed // - tailnet IPs have changed for any backend targets specified by tailnet FQDN -func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRunOpts) error { +func (ep *egressProxy) run(ctx context.Context, nm netmapState, opts egressProxyRunOpts) error { ep.configure(opts) var tickChan <-chan time.Time var eventChan <-chan fsnotify.Event @@ -106,7 +106,7 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRu eventChan = w.Events } - if err := ep.sync(ctx, n); err != nil { + if err := ep.sync(ctx, nm); err != nil { return err } for { @@ -117,14 +117,14 @@ func (ep *egressProxy) run(ctx context.Context, n ipn.Notify, opts egressProxyRu log.Printf("periodic sync, ensuring firewall config is up to date...") case <-eventChan: log.Printf("config file change detected, ensuring firewall config is up to date...") - case n = <-ep.netmapChan: - shouldResync := ep.shouldResync(n) + case nm = <-ep.netmapChan: + shouldResync := ep.shouldResync(nm) if !shouldResync { continue } log.Printf("netmap change detected, ensuring firewall config is up to date...") } - if err := ep.sync(ctx, n); err != nil { + if err := ep.sync(ctx, nm); err != nil { return fmt.Errorf("error syncing egress service config: %w", err) } } @@ -136,7 +136,7 @@ type egressProxyRunOpts struct { kc kubeclient.Client tsClient *local.Client stateSecret string - netmapChan chan ipn.Notify + netmapChan chan netmapState podIPv4 string tailnetAddrs []netip.Prefix } @@ -165,7 +165,7 @@ func (ep *egressProxy) configure(opts egressProxyRunOpts) { // any firewall rules need to be updated. Currently using status in state Secret as a reference for what is the current // firewall configuration is good enough because - the status is keyed by the Pod IP - we crash the Pod on errors such // as failed firewall update -func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error { +func (ep *egressProxy) sync(ctx context.Context, nm netmapState) error { cfgs, err := ep.getConfigs() if err != nil { return fmt.Errorf("error retrieving egress service configs: %w", err) @@ -174,28 +174,27 @@ func (ep *egressProxy) sync(ctx context.Context, n ipn.Notify) error { if err != nil { return fmt.Errorf("error retrieving current egress proxy status: %w", err) } - newStatus, err := ep.syncEgressConfigs(cfgs, status, n) + newStatus, err := ep.syncEgressConfigs(cfgs, status, nm) if err != nil { return fmt.Errorf("error syncing egress service configs: %w", err) } if !servicesStatusIsEqual(newStatus, status) { - if err := ep.setStatus(ctx, newStatus, n); err != nil { + if err := ep.setStatus(ctx, newStatus, nm); err != nil { return fmt.Errorf("error setting egress proxy status: %w", err) } } return nil } -// addrsHaveChanged returns true if the provided netmap update contains tailnet address change for this proxy node. -// Netmap must not be nil. -func (ep *egressProxy) addrsHaveChanged(n ipn.Notify) bool { - return !reflect.DeepEqual(ep.tailnetAddrs, n.NetMap.SelfNode.Addresses()) +// addrsHaveChanged returns true if the provided netmap state contains tailnet address change for this proxy node. +func (ep *egressProxy) addrsHaveChanged(nm netmapState) bool { + return !views.SliceEqual(views.SliceOf(ep.tailnetAddrs), nm.self.Addresses()) } // syncEgressConfigs adds and deletes firewall rules to match the desired // configuration. It uses the provided status to determine what is currently // applied and updates the status after a successful sync. -func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *egressservices.Status, n ipn.Notify) (*egressservices.Status, error) { +func (ep *egressProxy) syncEgressConfigs(cfgs egressservices.Configs, status *egressservices.Status, nm netmapState) (*egressservices.Status, error) { if !(wantsServicesConfigured(cfgs) || hasServicesConfigured(status)) { return nil, nil } @@ -213,8 +212,8 @@ func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *e // Add new services, update rules for any that have changed. rulesPerSvcToAdd := make(map[string][]rule, 0) rulesPerSvcToDelete := make(map[string][]rule, 0) - for svcName, cfg := range *cfgs { - tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, n) + for svcName, cfg := range cfgs { + tailnetTargetIPs, err := ep.tailnetTargetIPsForSvc(cfg, nm) if err != nil { return nil, fmt.Errorf("error determining tailnet target IPs: %w", err) } @@ -229,12 +228,12 @@ func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *e if len(rulesToDelete) != 0 { mak.Set(&rulesPerSvcToDelete, svcName, rulesToDelete) } - if len(rulesToAdd) != 0 || ep.addrsHaveChanged(n) { + if len(rulesToAdd) != 0 || ep.addrsHaveChanged(nm) { // For each tailnet target, set up SNAT from the local tailnet device address of the matching // family. for _, t := range tailnetTargetIPs { var local netip.Addr - for _, pfx := range n.NetMap.SelfNode.Addresses().All() { + for _, pfx := range nm.self.Addresses().All() { if !pfx.IsSingleIP() { continue } @@ -250,6 +249,9 @@ func (ep *egressProxy) syncEgressConfigs(cfgs *egressservices.Configs, status *e if err := ep.nfr.EnsureSNATForDst(local, t); err != nil { return nil, fmt.Errorf("error setting up SNAT rule: %w", err) } + if err := ep.nfr.ClampMSSToPMTU(tailscaleTunInterface, t); err != nil { + return nil, fmt.Errorf("error clamping MSS to PMTU: %w", err) + } } } // Update the status. Status will be written back to the state Secret by the caller. @@ -353,7 +355,7 @@ func updatesForCfg(svcName string, cfg egressservices.Config, status *egressserv // deleteUnneccessaryServices ensure that any services found on status, but not // present in config are deleted. -func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, status *egressservices.Status) error { +func (ep *egressProxy) deleteUnnecessaryServices(cfgs egressservices.Configs, status *egressservices.Status) error { if !hasServicesConfigured(status) { return nil } @@ -368,7 +370,7 @@ func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, s } for svcName, svc := range status.Services { - if _, ok := (*cfgs)[svcName]; !ok { + if _, ok := cfgs[svcName]; !ok { log.Printf("service %s is no longer required, deleting", svcName) if err := ensureServiceDeleted(svcName, svc, ep.nfr); err != nil { return fmt.Errorf("error deleting service %s: %w", svcName, err) @@ -380,7 +382,7 @@ func (ep *egressProxy) deleteUnnecessaryServices(cfgs *egressservices.Configs, s } // getConfigs gets the mounted egress service configuration. -func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) { +func (ep *egressProxy) getConfigs() (egressservices.Configs, error) { svcsCfg := filepath.Join(ep.cfgPath, egressservices.KeyEgressServices) j, err := os.ReadFile(svcsCfg) if os.IsNotExist(err) { @@ -392,7 +394,7 @@ func (ep *egressProxy) getConfigs() (*egressservices.Configs, error) { if len(j) == 0 || string(j) == "" { return nil, nil } - cfg := &egressservices.Configs{} + cfg := egressservices.Configs{} if err := json.Unmarshal(j, &cfg); err != nil { return nil, err } @@ -424,7 +426,7 @@ func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, e // setStatus writes egress proxy's currently configured firewall to the state // Secret and updates proxy's tailnet addresses. -func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, n ipn.Notify) error { +func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Status, nm netmapState) error { // Pod IP is used to determine if a stored status applies to THIS proxy Pod. if status == nil { status = &egressservices.Status{} @@ -447,7 +449,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta if err := ep.kc.JSONPatchResource(ctx, ep.stateSecret, kubeclient.TypeSecrets, []kubeclient.JSONPatch{patch}); err != nil { return fmt.Errorf("error patching state Secret: %w", err) } - ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice() + ep.tailnetAddrs = nm.self.Addresses().AsSlice() return nil } @@ -457,7 +459,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta // FQDN, resolve the FQDN and return the resolved IPs. It checks if the // netfilter runner supports IPv6 NAT and skips any IPv6 addresses if it // doesn't. -func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.Notify) (addrs []netip.Addr, err error) { +func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, nm netmapState) (addrs []netip.Addr, err error) { if svc.TailnetTarget.IP != "" { addr, err := netip.ParseAddr(svc.TailnetTarget.IP) if err != nil { @@ -473,57 +475,54 @@ func (ep *egressProxy) tailnetTargetIPsForSvc(svc egressservices.Config, n ipn.N if svc.TailnetTarget.FQDN == "" { return nil, errors.New("unexpected egress service config- neither tailnet target IP nor FQDN is set") } - if n.NetMap == nil { - log.Printf("netmap is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN) + if !nm.self.Valid() { + log.Printf("netmap state is not available, unable to determine backend addresses for %s", svc.TailnetTarget.FQDN) return addrs, nil } - var ( - node tailcfg.NodeView - nodeFound bool - ) - for _, nn := range n.NetMap.Peers { - if equalFQDNs(nn.Name(), svc.TailnetTarget.FQDN) { - node = nn - nodeFound = true - break - } + egressAddrs, err := resolveTailnetFQDN(nm, svc.TailnetTarget.FQDN) + if err != nil { + log.Printf("error fetching backend addresses for %q: %v", svc.TailnetTarget.FQDN, err) + return addrs, nil } - if nodeFound { - for _, addr := range node.Addresses().AsSlice() { - if addr.Addr().Is6() && !ep.nfr.HasIPV6NAT() { - log.Printf("tailnet target %v is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode, skipping.", addr.Addr().String()) - continue - } - addrs = append(addrs, addr.Addr()) + if len(egressAddrs) == 0 { + log.Printf("tailnet target %q does not have any backend addresses, skipping", svc.TailnetTarget.FQDN) + return addrs, nil + } + + for _, addr := range egressAddrs { + if addr.Addr().Is6() && !ep.nfr.HasIPV6NAT() { + log.Printf("tailnet target %v is an IPv6 address, but this host does not support IPv6 in the chosen firewall mode, skipping.", addr.Addr().String()) + continue } - // Egress target endpoints configured via FQDN are stored, so - // that we can determine if a netmap update should trigger a - // resync. - mak.Set(&ep.targetFQDNs, svc.TailnetTarget.FQDN, node.Addresses().AsSlice()) + addrs = append(addrs, addr.Addr()) } + // Egress target endpoints configured via FQDN are stored, so + // that we can determine if a netmap update should trigger a + // resync. + mak.Set(&ep.targetFQDNs, svc.TailnetTarget.FQDN, egressAddrs) return addrs, nil } -// shouldResync parses netmap update and returns true if the update contains +// shouldResync parses netmap state update and returns true if the update contains // changes for which the egress proxy's firewall should be reconfigured. -func (ep *egressProxy) shouldResync(n ipn.Notify) bool { - if n.NetMap == nil { +func (ep *egressProxy) shouldResync(nm netmapState) bool { + if !nm.self.Valid() { return false } // If proxy's tailnet addresses have changed, resync. - if !reflect.DeepEqual(n.NetMap.SelfNode.Addresses().AsSlice(), ep.tailnetAddrs) { + if !views.SliceEqual(nm.self.Addresses(), views.SliceOf(ep.tailnetAddrs)) { log.Printf("node addresses have changed, trigger egress config resync") - ep.tailnetAddrs = n.NetMap.SelfNode.Addresses().AsSlice() + ep.tailnetAddrs = nm.self.Addresses().AsSlice() return true } // If the IPs for any of the egress services configured via FQDN have // changed, resync. for fqdn, ips := range ep.targetFQDNs { - for _, nn := range n.NetMap.Peers { + for nn := range nm.peers() { if equalFQDNs(nn.Name(), fqdn) { - if !reflect.DeepEqual(ips, nn.Addresses().AsSlice()) { + if !views.SliceEqual(views.SliceOf(ips), nn.Addresses()) { log.Printf("backend addresses for egress target %q have changed old IPs %v, new IPs %v trigger egress config resync", nn.Name(), ips, nn.Addresses().AsSlice()) return true } @@ -570,7 +569,7 @@ func ensureRulesAdded(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner } // ensureRulesDeleted ensures that the given rules are deleted from the firewall -// configuration. For any rules that do not exist, calling this funcion is a +// configuration. For any rules that do not exist, calling this function is a // no-op. func ensureRulesDeleted(rulesPerSvc map[string][]rule, nfr linuxfw.NetfilterRunner) error { for svc, rules := range rulesPerSvc { @@ -606,8 +605,8 @@ type rule struct { protocol string } -func wantsServicesConfigured(cfgs *egressservices.Configs) bool { - return cfgs != nil && len(*cfgs) != 0 +func wantsServicesConfigured(cfgs egressservices.Configs) bool { + return cfgs != nil && len(cfgs) != 0 } func hasServicesConfigured(status *egressservices.Status) bool { @@ -661,14 +660,13 @@ func (ep *egressProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { // would normally be this Pod. When this Pod is being deleted, the operator should have removed it from the Service // backends and eventually kube proxy routing rules should be updated to no longer route traffic for the Service to this // Pod. -func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs *egressservices.Configs, hp int) { - if cfgs == nil || len(*cfgs) == 0 { // avoid sleeping if no services are configured +func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs egressservices.Configs, hp int) { + if cfgs == nil || len(cfgs) == 0 { // avoid sleeping if no services are configured return } log.Printf("Ensuring that cluster traffic for egress targets is no longer routed via this Pod...") - wg := syncs.WaitGroup{} - - for s, cfg := range *cfgs { + var wg sync.WaitGroup + for s, cfg := range cfgs { hep := cfg.HealthCheckEndpoint if hep == "" { log.Printf("Tailnet target %q does not have a cluster healthcheck specified, unable to verify if cluster traffic for the target is still routed via this Pod", s) diff --git a/cmd/containerboot/egressservices_test.go b/cmd/containerboot/egressservices_test.go index 724626b072c2b..b30765f19425a 100644 --- a/cmd/containerboot/egressservices_test.go +++ b/cmd/containerboot/egressservices_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -255,13 +255,13 @@ func TestWaitTillSafeToShutdown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfgs := &egressservices.Configs{} + cfgs := egressservices.Configs{} switches := make(map[string]int) for svc, callsToSwitch := range tt.services { endpoint := fmt.Sprintf("http://%s.local", svc) if tt.healthCheckSet { - (*cfgs)[svc] = egressservices.Config{ + cfgs[svc] = egressservices.Config{ HealthCheckEndpoint: endpoint, } } diff --git a/cmd/containerboot/forwarding.go b/cmd/containerboot/forwarding.go index 04d34836c92d8..6d90fbaaa9723 100644 --- a/cmd/containerboot/forwarding.go +++ b/cmd/containerboot/forwarding.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -51,7 +51,7 @@ func ensureIPForwarding(root, clusterProxyTargetIP, tailnetTargetIP, tailnetTarg v4Forwarding = true } if routes != nil && *routes != "" { - for _, route := range strings.Split(*routes, ",") { + for route := range strings.SplitSeq(*routes, ",") { cidr, err := netip.ParsePrefix(route) if err != nil { return fmt.Errorf("invalid subnet route: %v", err) diff --git a/cmd/containerboot/healthz.go b/cmd/containerboot/healthz.go deleted file mode 100644 index d6a64a37c4ac5..0000000000000 --- a/cmd/containerboot/healthz.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build linux - -package main - -import ( - "fmt" - "log" - "net/http" - "sync" - - "tailscale.com/kube/kubetypes" -) - -// healthz is a simple health check server, if enabled it returns 200 OK if -// this tailscale node currently has at least one tailnet IP address else -// returns 503. -type healthz struct { - sync.Mutex - hasAddrs bool - podIPv4 string -} - -func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h.Lock() - defer h.Unlock() - - if h.hasAddrs { - w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4) - if _, err := w.Write([]byte("ok")); err != nil { - http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError) - } - } else { - http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable) - } -} - -func (h *healthz) update(healthy bool) { - h.Lock() - defer h.Unlock() - - if h.hasAddrs != healthy { - log.Println("Setting healthy", healthy) - } - h.hasAddrs = healthy -} - -// registerHealthHandlers registers a simple health handler at /healthz. -// A containerized tailscale instance is considered healthy if -// it has at least one tailnet IP address. -func registerHealthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { - h := &healthz{podIPv4: podIPv4} - mux.Handle("GET /healthz", h) - return h -} diff --git a/cmd/containerboot/ingressservices.go b/cmd/containerboot/ingressservices.go index 1a2da95675f4e..d8ad017170379 100644 --- a/cmd/containerboot/ingressservices.go +++ b/cmd/containerboot/ingressservices.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -265,7 +265,13 @@ func ensureIngressRulesAdded(cfgs map[string]ingressservices.Config, nfr linuxfw func addDNATRuleForSvc(nfr linuxfw.NetfilterRunner, serviceName string, tsIP, clusterIP netip.Addr) error { log.Printf("adding DNAT rule for Tailscale Service %s with IP %s to Kubernetes Service IP %s", serviceName, tsIP, clusterIP) - return nfr.EnsureDNATRuleForSvc(serviceName, tsIP, clusterIP) + if err := nfr.EnsureDNATRuleForSvc(serviceName, tsIP, clusterIP); err != nil { + return err + } + if err := nfr.ClampMSSToPMTU(tailscaleTunInterface, clusterIP); err != nil { + return fmt.Errorf("error clamping MSS to PMTU: %w", err) + } + return nil } // ensureIngressRulesDeleted takes a map of Tailscale Services and rules and ensures that the firewall rules are deleted. diff --git a/cmd/containerboot/ingressservices_test.go b/cmd/containerboot/ingressservices_test.go index 228bbb159f463..1643bb11c069e 100644 --- a/cmd/containerboot/ingressservices_test.go +++ b/cmd/containerboot/ingressservices_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -7,6 +7,7 @@ package main import ( "net/netip" + "slices" "testing" "tailscale.com/kube/ingressservices" @@ -22,6 +23,7 @@ func TestSyncIngressConfigs(t *testing.T) { TailscaleServiceIP netip.Addr ClusterIP netip.Addr } + wantClampedAddrs []netip.Addr // cluster IPs that should have MSS clamping applied }{ { name: "add_new_rules_when_no_existing_config", @@ -35,6 +37,7 @@ func TestSyncIngressConfigs(t *testing.T) { }{ "svc:foo": makeWantService("100.64.0.1", "10.0.0.1"), }, + wantClampedAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, }, { name: "add_multiple_services", @@ -52,6 +55,11 @@ func TestSyncIngressConfigs(t *testing.T) { "svc:bar": makeWantService("100.64.0.2", "10.0.0.2"), "svc:baz": makeWantService("100.64.0.3", "10.0.0.3"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.1"), + netip.MustParseAddr("10.0.0.2"), + netip.MustParseAddr("10.0.0.3"), + }, }, { name: "add_both_ipv4_and_ipv6_rules", @@ -65,6 +73,10 @@ func TestSyncIngressConfigs(t *testing.T) { }{ "svc:foo": makeWantService("2001:db8::1", "2001:db8::2"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.1"), + netip.MustParseAddr("2001:db8::2"), + }, }, { name: "add_ipv6_only_rules", @@ -78,6 +90,7 @@ func TestSyncIngressConfigs(t *testing.T) { }{ "svc:ipv6": makeWantService("2001:db8::10", "2001:db8::20"), }, + wantClampedAddrs: []netip.Addr{netip.MustParseAddr("2001:db8::20")}, }, { name: "delete_all_rules_when_config_removed", @@ -94,6 +107,7 @@ func TestSyncIngressConfigs(t *testing.T) { TailscaleServiceIP netip.Addr ClusterIP netip.Addr }{}, + wantClampedAddrs: nil, // no rules added, no clamping }, { name: "add_remove_modify", @@ -117,6 +131,10 @@ func TestSyncIngressConfigs(t *testing.T) { "svc:foo": makeWantService("100.64.0.1", "10.0.0.2"), "svc:new": makeWantService("100.64.0.4", "10.0.0.4"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.2"), + netip.MustParseAddr("10.0.0.4"), + }, }, { name: "update_with_outdated_status", @@ -152,12 +170,17 @@ func TestSyncIngressConfigs(t *testing.T) { "svc:web-ipv6": makeWantService("2001:db8::10", "2001:db8::20"), "svc:api": makeWantService("100.64.0.20", "10.0.0.20"), }, + wantClampedAddrs: []netip.Addr{ + netip.MustParseAddr("10.0.0.10"), + netip.MustParseAddr("10.0.0.20"), + netip.MustParseAddr("2001:db8::20"), + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var nfr linuxfw.NetfilterRunner = linuxfw.NewFakeNetfilterRunner() + nfr := linuxfw.NewFakeNetfilterRunner() ep := &ingressProxy{ nfr: nfr, @@ -170,8 +193,7 @@ func TestSyncIngressConfigs(t *testing.T) { t.Fatalf("syncIngressConfigs failed: %v", err) } - fake := nfr.(*linuxfw.FakeNetfilterRunner) - gotServices := fake.GetServiceState() + gotServices := nfr.GetServiceState() if len(gotServices) != len(tt.wantServices) { t.Errorf("got %d services, want %d", len(gotServices), len(tt.wantServices)) } @@ -188,6 +210,20 @@ func TestSyncIngressConfigs(t *testing.T) { t.Errorf("service %s: got ClusterIP %v, want %v", svc, got.ClusterIP, want.ClusterIP) } } + + gotClamped := nfr.GetClampedAddrs() + slices.SortFunc(gotClamped, func(a, b netip.Addr) int { return a.Compare(b) }) + slices.SortFunc(tt.wantClampedAddrs, func(a, b netip.Addr) int { return a.Compare(b) }) + if len(gotClamped) != len(tt.wantClampedAddrs) { + t.Errorf("ClampMSSToPMTU: got %v, want %v", gotClamped, tt.wantClampedAddrs) + } else { + for i := range gotClamped { + if gotClamped[i] != tt.wantClampedAddrs[i] { + t.Errorf("ClampMSSToPMTU: got %v, want %v", gotClamped, tt.wantClampedAddrs) + break + } + } + } }) } } diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 0a2dfa1bf342f..3e97710da6c92 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -14,18 +14,26 @@ import ( "net/http" "net/netip" "os" + "path/filepath" "strings" "time" + "github.com/fsnotify/fsnotify" + "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/kube/authkey" + "tailscale.com/kube/egressservices" + "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" - "tailscale.com/logtail/backoff" "tailscale.com/tailcfg" "tailscale.com/types/logger" + "tailscale.com/util/backoff" ) +const fieldManager = "tailscale-container" + // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use // this rather than any of the upstream Kubernetes client libaries to avoid extra imports. type kubeClient struct { @@ -43,7 +51,7 @@ func newKubeClient(root string, stateSecret string) (*kubeClient, error) { var err error kc, err := kubeclient.New("tailscale-container") if err != nil { - return nil, fmt.Errorf("Error creating kube client: %w", err) + return nil, fmt.Errorf("error creating kube client: %w", err) } if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" { // Derive the API server address from the environment variables @@ -60,7 +68,7 @@ func (kc *kubeClient) storeDeviceID(ctx context.Context, deviceID tailcfg.Stable kubetypes.KeyDeviceID: []byte(deviceID), }, } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) } // storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields 'device_ips', 'device_fqdn' of client's @@ -81,7 +89,7 @@ func (kc *kubeClient) storeDeviceEndpoints(ctx context.Context, fqdn string, add kubetypes.KeyDeviceIPs: deviceIPs, }, } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) } // storeHTTPSEndpoint writes an HTTPS endpoint exposed by this device via 'tailscale serve' to the client's state @@ -93,7 +101,7 @@ func (kc *kubeClient) storeHTTPSEndpoint(ctx context.Context, ep string) error { kubetypes.KeyHTTPSEndpoint: []byte(ep), }, } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) } // deleteAuthKey deletes the 'authkey' field of the given kube @@ -117,21 +125,89 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error { return nil } -// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale -// state Secret. -// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container. -func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error { - capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion) - d := map[string][]byte{ - kubetypes.KeyCapVer: []byte(capVerS), +// resetContainerbootState resets state from previous runs of containerboot to +// ensure the operator doesn't use stale state when a Pod is first recreated. +// +// Device identity keys (device_id, device_fqdn, device_ips) are preserved so +// the operator can clean up the old device from the control plane. +func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string, tailscaledConfigAuthkey string) error { + existingSecret, err := kc.GetSecret(ctx, kc.stateSecret) + switch { + case kubeclient.IsNotFoundErr(err): + // In the case that the Secret doesn't exist, we don't have any state to reset and can return early. + return nil + case err != nil: + return fmt.Errorf("failed to read state Secret %q to reset state: %w", kc.stateSecret, err) + } + + s := &kubeapi.Secret{ + Data: map[string][]byte{ + kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, } if podUID != "" { - d[kubetypes.KeyPodUID] = []byte(podUID) + s.Data[kubetypes.KeyPodUID] = []byte(podUID) } - s := &kubeapi.Secret{ - Data: d, + + // Only clear reissue_authkey if the operator has actioned it. + brokenAuthkey, ok := existingSecret.Data[kubetypes.KeyReissueAuthkey] + if ok && tailscaledConfigAuthkey != "" && string(brokenAuthkey) != tailscaledConfigAuthkey { + s.Data[kubetypes.KeyReissueAuthkey] = nil + } + + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, fieldManager) +} + +func (kc *kubeClient) setAndWaitForAuthKeyReissue(ctx context.Context, client *local.Client, cfg *settings, tailscaledConfigAuthKey string) error { + err := client.DisconnectControl(ctx) + if err != nil { + return fmt.Errorf("error disconnecting from control: %w", err) } - return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") + + err = authkey.SetReissueAuthKey(ctx, kc.Client, kc.stateSecret, tailscaledConfigAuthKey, authkey.TailscaleContainerFieldManager) + if err != nil { + return fmt.Errorf("failed to set reissue_authkey in Kubernetes Secret: %w", err) + } + + clearFn := func(ctx context.Context) error { + return authkey.ClearReissueAuthKey(ctx, kc.Client, kc.stateSecret, authkey.TailscaleContainerFieldManager) + } + + getAuthKey := func() string { return authkey.AuthKeyFromConfig(cfg.TailscaledConfigFilePath) } + tailscaledCfgDir := filepath.Dir(cfg.TailscaledConfigFilePath) + var notify <-chan struct{} + if w, err := fsnotify.NewWatcher(); err != nil { + log.Printf("auth key reissue: fsnotify unavailable, using polling: %v", err) + } else if err := w.Add(tailscaledCfgDir); err != nil { + w.Close() + log.Printf("auth key reissue: fsnotify watch failed, using polling: %v", err) + } else { + defer w.Close() + ch := make(chan struct{}, 1) + toWatch := filepath.Join(tailscaledCfgDir, "..data") + go func() { + for ev := range w.Events { + if ev.Name == toWatch { + select { + case ch <- struct{}{}: + default: + } + } + } + }() + notify = ch + log.Printf("auth key reissue: watching for config changes via fsnotify") + } + + err = authkey.WaitForAuthKeyReissue(ctx, tailscaledConfigAuthKey, 10*time.Minute, getAuthKey, clearFn, notify) + if err != nil { + return fmt.Errorf("failed to receive new auth key: %w", err) + } + + return nil } // waitForConsistentState waits for tailscaled to finish writing state if it diff --git a/cmd/containerboot/kube_test.go b/cmd/containerboot/kube_test.go index 413971bc6df23..fec0b74f7d8f1 100644 --- a/cmd/containerboot/kube_test.go +++ b/cmd/containerboot/kube_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -8,13 +8,18 @@ package main import ( "context" "errors" + "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "tailscale.com/ipn" + "tailscale.com/kube/egressservices" + "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" ) func TestSetupKube(t *testing.T) { @@ -26,7 +31,7 @@ func TestSetupKube(t *testing.T) { kc *kubeClient }{ { - name: "TS_AUTHKEY set, state Secret exists", + name: "authkey-set-secret-exists", cfg: &settings{ AuthKey: "foo", KubeSecret: "foo", @@ -45,7 +50,7 @@ func TestSetupKube(t *testing.T) { }, }, { - name: "TS_AUTHKEY set, state Secret does not exist, we have permissions to create it", + name: "authkey-set-secret-missing-can-create", cfg: &settings{ AuthKey: "foo", KubeSecret: "foo", @@ -64,7 +69,7 @@ func TestSetupKube(t *testing.T) { }, }, { - name: "TS_AUTHKEY set, state Secret does not exist, we do not have permissions to create it", + name: "authkey-set-secret-missing-cannot-create", cfg: &settings{ AuthKey: "foo", KubeSecret: "foo", @@ -84,7 +89,7 @@ func TestSetupKube(t *testing.T) { wantErr: true, }, { - name: "TS_AUTHKEY set, we encounter a non-404 error when trying to retrieve the state Secret", + name: "authkey-set-get-secret-non-404-error", cfg: &settings{ AuthKey: "foo", KubeSecret: "foo", @@ -104,7 +109,7 @@ func TestSetupKube(t *testing.T) { wantErr: true, }, { - name: "TS_AUTHKEY set, we encounter a non-404 error when trying to check Secret permissions", + name: "authkey-set-check-perms-error", cfg: &settings{ AuthKey: "foo", KubeSecret: "foo", @@ -122,7 +127,7 @@ func TestSetupKube(t *testing.T) { }, { // Interactive login using URL in Pod logs - name: "TS_AUTHKEY not set, state Secret does not exist, we have permissions to create it", + name: "no-authkey-secret-missing-can-create", cfg: &settings{ KubeSecret: "foo", }, @@ -140,7 +145,7 @@ func TestSetupKube(t *testing.T) { }, { // Interactive login using URL in Pod logs - name: "TS_AUTHKEY not set, state Secret exists, but does not contain auth key", + name: "no-authkey-secret-exists-no-key", cfg: &settings{ KubeSecret: "foo", }, @@ -157,7 +162,7 @@ func TestSetupKube(t *testing.T) { }}, }, { - name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it", + name: "no-authkey-secret-has-key-cannot-patch", cfg: &settings{ KubeSecret: "foo", }, @@ -175,7 +180,7 @@ func TestSetupKube(t *testing.T) { wantErr: true, }, { - name: "TS_AUTHKEY not set, state Secret contains auth key, we have RBAC to patch it", + name: "no-authkey-secret-has-key-can-patch", cfg: &settings{ KubeSecret: "foo", }, @@ -238,3 +243,126 @@ func TestWaitForConsistentState(t *testing.T) { t.Fatalf("expected nil, got %v", err) } } + +func TestResetContainerbootState(t *testing.T) { + capver := fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + for name, tc := range map[string]struct { + podUID string + authkey string + initial map[string][]byte + expected map[string][]byte + }{ + "empty_initial": { + podUID: "1234", + authkey: "new-authkey", + initial: map[string][]byte{}, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: []byte("1234"), + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, + "empty_initial_no_pod_uid": { + initial: map[string][]byte{}, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, + "only_relevant_keys_updated": { + podUID: "1234", + authkey: "new-authkey", + initial: map[string][]byte{ + kubetypes.KeyCapVer: []byte("1"), + kubetypes.KeyPodUID: []byte("5678"), + kubetypes.KeyDeviceID: []byte("device-id"), + kubetypes.KeyDeviceFQDN: []byte("device-fqdn"), + kubetypes.KeyDeviceIPs: []byte(`["192.0.2.1"]`), + kubetypes.KeyHTTPSEndpoint: []byte("https://example.com"), + egressservices.KeyEgressServices: []byte("egress-services"), + ingressservices.IngressConfigKey: []byte("ingress-config"), + "_current-profile": []byte("current-profile"), + "_machinekey": []byte("machine-key"), + "_profiles": []byte("profiles"), + "_serve_e0ce": []byte("serve-e0ce"), + "profile-e0ce": []byte("profile-e0ce"), + }, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: []byte("1234"), + // Cleared keys. + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + // Tailscaled keys not included in patch. + }, + }, + "new_authkey_issued": { + initial: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-authkey"), + }, + authkey: "new-authkey", + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyReissueAuthkey: nil, + // Cleared keys. + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, + "authkey_not_yet_updated": { + initial: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-authkey"), + }, + authkey: "old-authkey", + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + // reissue_authkey not cleared. + // Cleared keys. + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, + "authkey_deleted_from_config": { + initial: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-authkey"), + }, + authkey: "", + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + // reissue_authkey not cleared. + // Cleared keys. + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + }, + }, + } { + t.Run(name, func(t *testing.T) { + var actual map[string][]byte + kc := &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{ + GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) { + return &kubeapi.Secret{ + Data: tc.initial, + }, nil + }, + StrategicMergePatchSecretImpl: func(ctx context.Context, name string, secret *kubeapi.Secret, _ string) error { + actual = secret.Data + return nil + }, + }} + if err := kc.resetContainerbootState(context.Background(), tc.podUID, tc.authkey); err != nil { + t.Fatalf("resetContainerbootState() error = %v", err) + } + if diff := cmp.Diff(tc.expected, actual); diff != "" { + t.Errorf("resetContainerbootState() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 9543308975b79..51bd842531226 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -11,7 +11,21 @@ // As with most container things, configuration is passed through environment // variables. All configuration is optional. // -// - TS_AUTHKEY: the authkey to use for login. +// - TS_AUTHKEY: the authkey to use for login. Also accepts TS_AUTH_KEY. +// If the value begins with "file:", it is treated as a path to a file containing the key. +// - TS_CLIENT_ID: the OAuth client ID. Can be used alone (ID token auto-generated +// in well-known environments), with TS_CLIENT_SECRET, or with TS_ID_TOKEN. +// - TS_CLIENT_SECRET: the OAuth client secret for generating authkeys. +// If the value begins with "file:", it is treated as a path to a file containing the secret. +// - TS_ID_TOKEN: the ID token from the identity provider for workload identity federation. +// Must be used together with TS_CLIENT_ID. If the value begins with "file:", it is +// treated as a path to a file containing the token. +// - TS_AUDIENCE: the audience to use when requesting an ID token from a well-known identity provider +// to exchange with the control server for workload identity federation. Must be used together +// with TS_CLIENT_ID. +// - Note: TS_AUTHKEY is mutually exclusive with TS_CLIENT_ID, TS_CLIENT_SECRET, TS_ID_TOKEN, +// and TS_AUDIENCE. +// TS_CLIENT_SECRET, TS_ID_TOKEN, and TS_AUDIENCE cannot be used together. // - TS_HOSTNAME: the hostname to request for the node. // - TS_ROUTES: subnet routes to advertise. Explicitly setting it to an empty // value will cause containerboot to stop acting as a subnet router for any @@ -67,8 +81,8 @@ // - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a // directory that containers tailscaled config in file. The config file needs to be // named cap-.hujson. If this is set, TS_HOSTNAME, -// TS_EXTRA_ARGS, TS_AUTHKEY, -// TS_ROUTES, TS_ACCEPT_DNS env vars must not be set. If this is set, +// TS_EXTRA_ARGS, TS_AUTHKEY, TS_CLIENT_ID, TS_CLIENT_SECRET, TS_ID_TOKEN, +// TS_ROUTES, TS_ACCEPT_DNS, TS_AUDIENCE env vars must not be set. If this is set, // containerboot only runs `tailscaled --config ` // and not `tailscale up` or `tailscale set`. // The config file contents are currently read once on container start. @@ -87,6 +101,10 @@ // cluster using the same hostname (in this case, the MagicDNS name of the ingress proxy) // as a non-cluster workload on tailnet. // This is only meant to be configured by the Kubernetes operator. +// - TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT: If set to true and if this +// containerboot instance is not running in Kubernetes, autoadvertise any services +// defined in the devices serve config, and unadvertise on shutdown. Defaults +// to `true`, but can be disabled to allow user specific advertisement configuration. // // When running on Kubernetes, containerboot defaults to storing state in the // "tailscale" kube secret. To store state on local disk instead, set @@ -102,6 +120,7 @@ import ( "errors" "fmt" "io/fs" + "iter" "log" "math" "net" @@ -117,15 +136,24 @@ import ( "syscall" "time" + "github.com/benbjohnson/immutable" "golang.org/x/sys/unix" - "tailscale.com/client/tailscale" + + "tailscale.com/health" "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" kubeutils "tailscale.com/k8s-operator" + "tailscale.com/kube/authkey" + healthz "tailscale.com/kube/health" "tailscale.com/kube/kubetypes" + klc "tailscale.com/kube/localclient" + "tailscale.com/kube/metrics" + "tailscale.com/kube/services" "tailscale.com/tailcfg" "tailscale.com/types/logger" - "tailscale.com/types/ptr" + "tailscale.com/types/views" "tailscale.com/util/deephash" + "tailscale.com/util/dnsname" "tailscale.com/util/linuxfw" ) @@ -136,6 +164,141 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) { return linuxfw.New(logf, "") } +func getAutoAdvertiseBool() bool { + return defaultBool("TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", true) +} + +const containerbootWatchMask = ipn.NotifyInitialStatus | + ipn.NotifyPeerChanges | + ipn.NotifyNoNetMap + +func notifyState(n ipn.Notify) (_ ipn.State, ok bool) { + if n.State != nil { + return *n.State, true + } + if n.InitialStatus != nil && n.InitialStatus.BackendState != "" { + if state, ok := ipn.StateFromString(n.InitialStatus.BackendState); ok { + return state, true + } + } + return ipn.NoState, false +} + +var netmapStatePeerIDHasher = immutable.NewHasher(tailcfg.NodeID(0)) + +type netmapState struct { + self tailcfg.NodeView + peersByID *immutable.Map[tailcfg.NodeID, tailcfg.NodeView] + peersByName *immutable.Map[string, tailcfg.NodeView] // keyed by tailcfg.Node.Name when NodeID is unavailable + certDomains views.Slice[string] + dnsExtraRecords views.Slice[tailcfg.DNSRecord] +} + +func (s netmapState) updateFromNotify(n ipn.Notify) netmapState { + if n.InitialStatus != nil { + s = s.updateFromStatus(n.InitialStatus) + } + if n.SelfChange != nil { + s.self = n.SelfChange.View() + } + for _, p := range n.PeersChanged { + s = s.upsertPeer(p.View()) + } + for _, id := range n.PeersRemoved { + if s.peersByID != nil { + s.peersByID = s.peersByID.Delete(id) + } + } + return s +} + +func (s netmapState) updateFromStatus(st *ipnstate.Status) netmapState { + s.certDomains = views.SliceOf(st.CertDomains) + s.dnsExtraRecords = views.SliceOf(st.ExtraRecords) + if st.Self != nil { + s.self = nodeFromPeerStatus(st.Self).View() + } + if len(st.Peer) != 0 { + s.peersByID = nil + s.peersByName = nil + for _, ps := range st.Peer { + s = s.upsertPeer(nodeFromPeerStatus(ps).View()) + } + } + return s +} + +func (s netmapState) upsertPeer(n tailcfg.NodeView) netmapState { + if !n.Valid() { + return s + } + if s.peersByID == nil { + s.peersByID = immutable.NewMap[tailcfg.NodeID, tailcfg.NodeView](netmapStatePeerIDHasher) + } + if s.peersByName == nil { + s.peersByName = immutable.NewMap[string, tailcfg.NodeView](nil) + } + if n.ID() != 0 { + s.peersByID = s.peersByID.Set(n.ID(), n) + if name := n.Name(); name != "" { + s.peersByName = s.peersByName.Delete(name) + } + return s + } + if n.Name() != "" { + s.peersByName = s.peersByName.Set(n.Name(), n) + } + return s +} + +func nodeFromPeerStatus(ps *ipnstate.PeerStatus) *tailcfg.Node { + if ps == nil { + return nil + } + n := &tailcfg.Node{ + ID: ps.NodeID, + StableID: ps.ID, + Name: ps.DNSName, + Key: ps.PublicKey, + } + for _, ip := range ps.TailscaleIPs { + n.Addresses = append(n.Addresses, netip.PrefixFrom(ip, ip.BitLen())) + } + if ps.AllowedIPs != nil { + n.AllowedIPs = ps.AllowedIPs.AsSlice() + } + return n +} + +func (s netmapState) peers() iter.Seq[tailcfg.NodeView] { + return func(yield func(tailcfg.NodeView) bool) { + if s.peersByID != nil { + it := s.peersByID.Iterator() + for { + _, p, ok := it.Next() + if !ok { + break + } + if !yield(p) { + return + } + } + } + if s.peersByName != nil { + it := s.peersByName.Iterator() + for { + _, p, ok := it.Next() + if !ok { + break + } + if !yield(p) { + return + } + } + } + } +} + func main() { if err := run(); err != nil && !errors.Is(err, context.Canceled) { log.Fatal(err) @@ -144,7 +307,6 @@ func main() { func run() error { log.SetPrefix("boot: ") - tailscale.I_Acknowledge_This_API_Is_Unstable = true cfg, err := configFromEnv() if err != nil { @@ -179,8 +341,13 @@ func run() error { bootCtx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() + var tailscaledConfigAuthkey string + if isOneStepConfig(cfg) { + tailscaledConfigAuthkey = authkey.AuthKeyFromConfig(cfg.TailscaledConfigFilePath) + } + var kc *kubeClient - if cfg.InKubernetes { + if cfg.KubeSecret != "" { kc, err = newKubeClient(cfg.Root, cfg.KubeSecret) if err != nil { return fmt.Errorf("error initializing kube client: %w", err) @@ -188,6 +355,14 @@ func run() error { if err := cfg.setupKube(bootCtx, kc); err != nil { return fmt.Errorf("error setting up for running on Kubernetes: %w", err) } + // Clear out any state from previous runs of containerboot. Check + // hasKubeStateStore because although we know we're in kube, that + // doesn't guarantee the state store is properly configured. + if hasKubeStateStore(cfg) { + if err := kc.resetContainerbootState(bootCtx, cfg.PodUID, tailscaledConfigAuthkey); err != nil { + return fmt.Errorf("error clearing previous state from Secret: %w", err) + } + } } client, daemonProcess, err := startTailscaled(bootCtx, cfg) @@ -202,7 +377,8 @@ func run() error { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) defer cancel() - if err := ensureServicesNotAdvertised(ctx, client); err != nil { + // we are shutting down, we always want to unadvertise here + if err := services.EnsureServicesNotAdvertised(ctx, client, log.Printf); err != nil { log.Printf("Error ensuring services are not advertised: %v", err) } @@ -223,13 +399,13 @@ func run() error { } defer killTailscaled() - var healthCheck *healthz + var healthCheck *healthz.Healthz ep := &egressProxy{} if cfg.HealthCheckAddrPort != "" { mux := http.NewServeMux() log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort) - healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) close := runHTTPServer(mux, cfg.HealthCheckAddrPort) defer close() @@ -240,12 +416,12 @@ func run() error { if cfg.localMetricsEnabled() { log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort) - registerMetricsHandlers(mux, client, cfg.DebugAddrPort) + metrics.RegisterMetricsHandlers(mux, client, cfg.DebugAddrPort) } if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) - healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) } if cfg.egressSvcsTerminateEPEnabled() { @@ -263,7 +439,7 @@ func run() error { } } - w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState) + w, err := client.WatchIPNBus(bootCtx, containerbootWatchMask|ipn.NotifyInitialPrefs|ipn.NotifyInitialHealthState) if err != nil { return fmt.Errorf("failed to watch tailscaled for updates: %w", err) } @@ -303,7 +479,7 @@ func run() error { if err := tailscaleUp(bootCtx, cfg); err != nil { return fmt.Errorf("failed to auth tailscale: %w", err) } - w, err = client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) + w, err = client.WatchIPNBus(bootCtx, containerbootWatchMask) if err != nil { return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) } @@ -323,14 +499,29 @@ authLoop: return fmt.Errorf("failed to read from tailscaled: %w", err) } - if n.State != nil { - switch *n.State { + if state, ok := notifyState(n); ok { + switch state { case ipn.NeedsLogin: if isOneStepConfig(cfg) { // This could happen if this is the first time tailscaled was run for this // device and the auth key was not passed via the configfile. - return fmt.Errorf("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file.") + if hasKubeStateStore(cfg) { + log.Printf("Auth key missing or invalid (NeedsLogin state), disconnecting from control and requesting new key from operator") + + err := kc.setAndWaitForAuthKeyReissue(ctx, client, cfg, tailscaledConfigAuthkey) + if err != nil { + return fmt.Errorf("failed to get a reissued authkey: %w", err) + } + + log.Printf("Successfully received new auth key, restarting to apply configuration") + + // we don't return an error here since we have handled the reissue gracefully. + return nil + } + + return errors.New("invalid state: tailscaled daemon started with a config file, but tailscale is not logged in: ensure you pass a valid auth key in the config file") } + if err := authTailscale(); err != nil { return fmt.Errorf("failed to auth tailscale: %w", err) } @@ -345,7 +536,28 @@ authLoop: // deadline to continue monitoring for changes. break authLoop default: - log.Printf("tailscaled in state %q, waiting", *n.State) + log.Printf("tailscaled in state %q, waiting", state) + } + } + + if n.Health != nil { + // This can happen if the config has an auth key but it's invalid, + // for example if it was single-use and already got used, but the + // device state was lost. + if _, ok := n.Health.Warnings[health.LoginStateWarnable.Code]; ok { + if isOneStepConfig(cfg) && hasKubeStateStore(cfg) { + log.Printf("Auth key failed to authenticate (may be expired or single-use), disconnecting from control and requesting new key from operator") + + err := kc.setAndWaitForAuthKeyReissue(ctx, client, cfg, tailscaledConfigAuthkey) + if err != nil { + return fmt.Errorf("failed to get a reissued authkey: %w", err) + } + + // we don't return an error here since we have handled the reissue gracefully. + log.Printf("Successfully received new auth key, restarting to apply configuration") + + return nil + } } } } @@ -367,38 +579,29 @@ authLoop: if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil { return fmt.Errorf("failed to unset serve config: %w", err) } - if hasKubeStateStore(cfg) { - if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil { - return fmt.Errorf("failed to update HTTPS endpoint in tailscale state: %w", err) - } - } } if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) { // We were told to only auth once, so any secret-bound // authkey is no longer needed. We don't strictly need to // wipe it, but it's good hygiene. - log.Printf("Deleting authkey from kube secret") + log.Printf("Deleting authkey from Kubernetes Secret") if err := kc.deleteAuthKey(ctx); err != nil { - return fmt.Errorf("deleting authkey from kube secret: %w", err) - } - } - - if hasKubeStateStore(cfg) { - if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil { - return fmt.Errorf("storing capability version and UID: %w", err) + return fmt.Errorf("deleting authkey from Kubernetes Secret: %w", err) } } - w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) + w, err = client.WatchIPNBus(ctx, containerbootWatchMask) if err != nil { return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) } // If tailscaled config was read from a mounted file, watch the file for updates and reload. cfgWatchErrChan := make(chan error) + cfgWatchCtx, cfgWatchCancel := context.WithCancel(ctx) + defer cfgWatchCancel() if cfg.TailscaledConfigFilePath != "" { - go watchTailscaledConfigChanges(ctx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan) + go watchTailscaledConfigChanges(cfgWatchCtx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan) } var ( @@ -440,8 +643,8 @@ authLoop: ) // egressSvcsErrorChan will get an error sent to it if this containerboot instance is configured to expose 1+ // egress services in HA mode and errored. - var egressSvcsErrorChan = make(chan error) - var ingressSvcsErrorChan = make(chan error) + egressSvcsErrorChan := make(chan error) + ingressSvcsErrorChan := make(chan error) defer t.Stop() // resetTimer resets timer for when to next attempt to resolve the DNS // name for the proxy configured with TS_EXPERIMENTAL_DEST_DNS_NAME. The @@ -467,7 +670,7 @@ authLoop: failedResolveAttempts++ } - var egressSvcsNotify chan ipn.Notify + var egressSvcsNotify chan netmapState notifyChan := make(chan ipn.Notify) errChan := make(chan error) go func() { @@ -481,10 +684,12 @@ authLoop: } } }() + var nmState netmapState var wg sync.WaitGroup runLoop: for { + var processNetmap bool select { case <-ctx.Done(): // Although killTailscaled() is deferred earlier, if we @@ -498,266 +703,281 @@ runLoop: case err := <-cfgWatchErrChan: return fmt.Errorf("failed to watch tailscaled config: %w", err) case n := <-notifyChan: - if n.State != nil && *n.State != ipn.Running { + nmState = nmState.updateFromNotify(n) + if state, ok := notifyState(n); ok && state != ipn.Running { // Something's gone wrong and we've left the authenticated state. // Our container image never recovered gracefully from this, and the // control flow required to make it work now is hard. So, just crash // the container and rely on the container runtime to restart us, // whereupon we'll go through initial auth again. - return fmt.Errorf("tailscaled left running state (now in state %q), exiting", *n.State) + return fmt.Errorf("tailscaled left running state (now in state %q), exiting", state) } - if n.NetMap != nil { - addrs = n.NetMap.SelfNode.Addresses().AsSlice() - newCurrentIPs := deephash.Hash(&addrs) - ipsHaveChanged := newCurrentIPs != currentIPs - - // Store device ID in a Kubernetes Secret before - // setting up any routing rules. This ensures - // that, for containerboot instances that are - // Kubernetes operator proxies, the operator is - // able to retrieve the device ID from the - // Kubernetes Secret to clean up tailnet nodes - // for proxies whose route setup continuously - // fails. - deviceID := n.NetMap.SelfNode.StableID() - if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceID, &deviceID) { - if err := kc.storeDeviceID(ctx, n.NetMap.SelfNode.StableID()); err != nil { - return fmt.Errorf("storing device ID in Kubernetes Secret: %w", err) - } + if n.InitialStatus != nil || n.SelfChange != nil || len(n.PeersChanged) != 0 || len(n.PeersRemoved) != 0 || len(n.PeerChangedPatch) != 0 { + processNetmap = true + } + case <-tc: + newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName) + if err != nil { + log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err) + resetTimer(true) + continue + } + backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool { + return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) }) + })) + if backendsHaveChanged && len(addrs) != 0 { + log.Printf("Backend address change detected, installing proxy rules for backends %v", newBackendAddrs) + if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { + return fmt.Errorf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err) } - if cfg.TailnetTargetFQDN != "" { - var ( - egressAddrs []netip.Prefix - newCurentEgressIPs deephash.Sum - egressIPsHaveChanged bool - node tailcfg.NodeView - nodeFound bool - ) - for _, n := range n.NetMap.Peers { - if strings.EqualFold(n.Name(), cfg.TailnetTargetFQDN) { - node = n - nodeFound = true - break - } - } - if !nodeFound { - log.Printf("Tailscale node %q not found; it either does not exist, or not reachable because of ACLs", cfg.TailnetTargetFQDN) - break - } - egressAddrs = node.Addresses().AsSlice() - newCurentEgressIPs = deephash.Hash(&egressAddrs) - egressIPsHaveChanged = newCurentEgressIPs != currentEgressIPs - // The firewall rules get (re-)installed: - // - on startup - // - when the tailnet IPs of the tailnet target have changed - // - when the tailnet IPs of this node have changed - if (egressIPsHaveChanged || ipsHaveChanged) && len(egressAddrs) != 0 { - var rulesInstalled bool - for _, egressAddr := range egressAddrs { - ea := egressAddr.Addr() - if ea.Is4() || (ea.Is6() && nfr.HasIPV6NAT()) { - rulesInstalled = true - log.Printf("Installing forwarding rules for destination %v", ea.String()) - if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil { - return fmt.Errorf("installing egress proxy rules for destination %s: %v", ea.String(), err) - } + } + backendAddrs = newBackendAddrs + resetTimer(false) + continue + case e := <-egressSvcsErrorChan: + return fmt.Errorf("egress proxy failed: %v", e) + case e := <-ingressSvcsErrorChan: + return fmt.Errorf("ingress proxy failed: %v", e) + } + if !processNetmap { + continue + } + self := nmState.self + if !self.Valid() { + continue + } + { + addrs = self.Addresses().AsSlice() + newCurrentIPs := deephash.Hash(&addrs) + ipsHaveChanged := newCurrentIPs != currentIPs + + // Store device ID in a Kubernetes Secret before + // setting up any routing rules. This ensures + // that, for containerboot instances that are + // Kubernetes operator proxies, the operator is + // able to retrieve the device ID from the + // Kubernetes Secret to clean up tailnet nodes + // for proxies whose route setup continuously + // fails. + deviceID := self.StableID() + if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceID, &deviceID) { + if err := kc.storeDeviceID(ctx, deviceID); err != nil { + return fmt.Errorf("storing device ID in Kubernetes Secret: %w", err) + } + } + if cfg.TailnetTargetFQDN != "" { + egressAddrs, err := resolveTailnetFQDN(nmState, cfg.TailnetTargetFQDN) + if err != nil { + log.Print(err.Error()) + break + } + + newCurentEgressIPs := deephash.Hash(&egressAddrs) + egressIPsHaveChanged := newCurentEgressIPs != currentEgressIPs + // The firewall rules get (re-)installed: + // - on startup + // - when the tailnet IPs of the tailnet target have changed + // - when the tailnet IPs of this node have changed + if (egressIPsHaveChanged || ipsHaveChanged) && len(egressAddrs) != 0 { + var rulesInstalled bool + for _, egressAddr := range egressAddrs { + ea := egressAddr.Addr() + if ea.Is4() || (ea.Is6() && nfr.HasIPV6NAT()) { + rulesInstalled = true + log.Printf("Installing forwarding rules for destination %v", ea.String()) + if err := installEgressForwardingRule(ctx, ea.String(), addrs, nfr); err != nil { + return fmt.Errorf("installing egress proxy rules for destination %s: %v", ea.String(), err) } } - if !rulesInstalled { - return fmt.Errorf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT()) - } } - currentEgressIPs = newCurentEgressIPs - } - if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged { - log.Printf("Installing proxy rules") - if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil { - return fmt.Errorf("installing ingress proxy rules: %w", err) + if !rulesInstalled { + return fmt.Errorf("no forwarding rules for egress addresses %v, host supports IPv6: %v", egressAddrs, nfr.HasIPV6NAT()) } } - if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged { - newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName) - if err != nil { - log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err) - resetTimer(true) - continue - } - backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool { - return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) }) - })) - if backendsHaveChanged { - log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs) - if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { - return fmt.Errorf("error installing ingress proxy rules: %w", err) - } - } - resetTimer(false) - backendAddrs = newBackendAddrs + currentEgressIPs = newCurentEgressIPs + } + if cfg.ProxyTargetIP != "" && len(addrs) != 0 && ipsHaveChanged { + log.Printf("Installing proxy rules") + if err := installIngressForwardingRule(ctx, cfg.ProxyTargetIP, addrs, nfr); err != nil { + return fmt.Errorf("installing ingress proxy rules: %w", err) } - if cfg.ServeConfigPath != "" { - cd := certDomainFromNetmap(n.NetMap) - if cd == "" { - cd = kubetypes.ValueNoHTTPS - } - prev := certDomain.Swap(ptr.To(cd)) - if prev == nil || *prev != cd { - select { - case certDomainChanged <- true: - default: - } - } + } + if cfg.ProxyTargetDNSName != "" && len(addrs) != 0 && ipsHaveChanged { + newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName) + if err != nil { + log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err) + resetTimer(true) + continue } - if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 { - log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP) - if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil { - return fmt.Errorf("installing egress proxy rules: %w", err) + backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool { + return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) }) + })) + if backendsHaveChanged { + log.Printf("installing ingress proxy rules for backends %v", newBackendAddrs) + if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { + return fmt.Errorf("error installing ingress proxy rules: %w", err) } } - // If this is a L7 cluster ingress proxy (set up - // by Kubernetes operator) and proxying of - // cluster traffic to the ingress target is - // enabled, set up proxy rule each time the - // tailnet IPs of this node change (including - // the first time they become available). - if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 { - log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP) - if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil { - return fmt.Errorf("installing rules to forward traffic to node's tailnet IP: %w", err) - } + resetTimer(false) + backendAddrs = newBackendAddrs + } + if cfg.ServeConfigPath != "" { + var cd string + if nmState.certDomains.Len() != 0 { + cd = nmState.certDomains.At(0) + } + if cd == "" { + cd = kubetypes.ValueNoHTTPS } - currentIPs = newCurrentIPs - - // Only store device FQDN and IP addresses to - // Kubernetes Secret when any required proxy - // route setup has succeeded. IPs and FQDN are - // read from the Secret by the Tailscale - // Kubernetes operator and, for some proxy - // types, such as Tailscale Ingress, advertized - // on the Ingress status. Writing them to the - // Secret only after the proxy routing has been - // set up ensures that the operator does not - // advertize endpoints of broken proxies. - // TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'. - deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()} - if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceEndpoints, &deviceEndpoints) { - if err := kc.storeDeviceEndpoints(ctx, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil { - return fmt.Errorf("storing device IPs and FQDN in Kubernetes Secret: %w", err) + prev := certDomain.Swap(new(cd)) + if prev == nil || *prev != cd { + select { + case certDomainChanged <- true: + default: } } + } + if cfg.TailnetTargetIP != "" && ipsHaveChanged && len(addrs) != 0 { + log.Printf("Installing forwarding rules for destination %v", cfg.TailnetTargetIP) + if err := installEgressForwardingRule(ctx, cfg.TailnetTargetIP, addrs, nfr); err != nil { + return fmt.Errorf("installing egress proxy rules: %w", err) + } + } + // If this is a L7 cluster ingress proxy (set up + // by Kubernetes operator) and proxying of + // cluster traffic to the ingress target is + // enabled, set up proxy rule each time the + // tailnet IPs of this node change (including + // the first time they become available). + if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) != 0 { + log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP) + if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil { + return fmt.Errorf("installing rules to forward traffic to node's tailnet IP: %w", err) + } + } + currentIPs = newCurrentIPs - if healthCheck != nil { - healthCheck.update(len(addrs) != 0) + // Only store device FQDN and IP addresses to + // Kubernetes Secret when any required proxy + // route setup has succeeded. IPs and FQDN are + // read from the Secret by the Tailscale + // Kubernetes operator and, for some proxy + // types, such as Tailscale Ingress, advertized + // on the Ingress status. Writing them to the + // Secret only after the proxy routing has been + // set up ensures that the operator does not + // advertize endpoints of broken proxies. + // TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'. + deviceEndpoints := []any{self.Name(), self.Addresses()} + if hasKubeStateStore(cfg) && deephash.Update(¤tDeviceEndpoints, &deviceEndpoints) { + if err := kc.storeDeviceEndpoints(ctx, self.Name(), addrs); err != nil { + return fmt.Errorf("storing device IPs and FQDN in Kubernetes Secret: %w", err) } + } - if cfg.ServeConfigPath != "" { - triggerWatchServeConfigChanges.Do(func() { - go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg) - }) + if healthCheck != nil { + healthCheck.Update(len(addrs) != 0) + } + + var prevServeConfig *ipn.ServeConfig + if getAutoAdvertiseBool() { + prevServeConfig, err = client.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("autoadvertisement: failed to get serve config: %w", err) } - if egressSvcsNotify != nil { - egressSvcsNotify <- n + err = refreshAdvertiseServices(ctx, prevServeConfig, klc.New(client)) + if err != nil { + return fmt.Errorf("autoadvertisement: failed to refresh advertise services: %w", err) } } - if !startupTasksDone { - // For containerboot instances that act as TCP proxies (proxying traffic to an endpoint - // passed via one of the env vars that containerboot reads) and store state in a - // Kubernetes Secret, we consider startup tasks done at the point when device info has - // been successfully stored to state Secret. For all other containerboot instances, if - // we just get to this point the startup tasks can be considered done. - if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) { - // This log message is used in tests to detect when all - // post-auth configuration is done. - log.Println("Startup complete, waiting for shutdown signal") - startupTasksDone = true - - // Configure egress proxy. Egress proxy will set up firewall rules to proxy - // traffic to tailnet targets configured in the provided configuration file. It - // will then continuously monitor the config file and netmap updates and - // reconfigure the firewall rules as needed. If any of its operations fail, it - // will crash this node. - if cfg.EgressProxiesCfgPath != "" { - log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressProxiesCfgPath) - egressSvcsNotify = make(chan ipn.Notify) - opts := egressProxyRunOpts{ - cfgPath: cfg.EgressProxiesCfgPath, - nfr: nfr, - kc: kc, - tsClient: client, - stateSecret: cfg.KubeSecret, - netmapChan: egressSvcsNotify, - podIPv4: cfg.PodIPv4, - tailnetAddrs: addrs, - } - go func() { - if err := ep.run(ctx, n, opts); err != nil { - egressSvcsErrorChan <- err - } - }() + + if cfg.ServeConfigPath != "" { + triggerWatchServeConfigChanges.Do(func() { + go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg, prevServeConfig) + }) + } + + if egressSvcsNotify != nil { + egressSvcsNotify <- nmState + } + } + if !startupTasksDone { + // For containerboot instances that act as TCP proxies (proxying traffic to an endpoint + // passed via one of the env vars that containerboot reads) and store state in a + // Kubernetes Secret, we consider startup tasks done at the point when device info has + // been successfully stored to state Secret. For all other containerboot instances, if + // we just get to this point the startup tasks can be considered done. + if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) { + // This log message is used in tests to detect when all + // post-auth configuration is done. + log.Println("Startup complete, waiting for shutdown signal") + startupTasksDone = true + + // Configure egress proxy. Egress proxy will set up firewall rules to proxy + // traffic to tailnet targets configured in the provided configuration file. It + // will then continuously monitor the config file and netmap updates and + // reconfigure the firewall rules as needed. If any of its operations fail, it + // will crash this node. + if cfg.EgressProxiesCfgPath != "" { + log.Printf("configuring egress proxy using configuration file at %s", cfg.EgressProxiesCfgPath) + egressSvcsNotify = make(chan netmapState) + opts := egressProxyRunOpts{ + cfgPath: cfg.EgressProxiesCfgPath, + nfr: nfr, + kc: kc, + tsClient: client, + stateSecret: cfg.KubeSecret, + netmapChan: egressSvcsNotify, + podIPv4: cfg.PodIPv4, + tailnetAddrs: addrs, } - ip := ingressProxy{} - if cfg.IngressProxiesCfgPath != "" { - log.Printf("configuring ingress proxy using configuration file at %s", cfg.IngressProxiesCfgPath) - opts := ingressProxyOpts{ - cfgPath: cfg.IngressProxiesCfgPath, - nfr: nfr, - kc: kc, - stateSecret: cfg.KubeSecret, - podIPv4: cfg.PodIPv4, - podIPv6: cfg.PodIPv6, + go func() { + if err := ep.run(ctx, nmState, opts); err != nil { + egressSvcsErrorChan <- err } - go func() { - if err := ip.run(ctx, opts); err != nil { - ingressSvcsErrorChan <- err - } - }() + }() + } + ip := ingressProxy{} + if cfg.IngressProxiesCfgPath != "" { + log.Printf("configuring ingress proxy using configuration file at %s", cfg.IngressProxiesCfgPath) + opts := ingressProxyOpts{ + cfgPath: cfg.IngressProxiesCfgPath, + nfr: nfr, + kc: kc, + stateSecret: cfg.KubeSecret, + podIPv4: cfg.PodIPv4, + podIPv6: cfg.PodIPv6, } + go func() { + if err := ip.run(ctx, opts); err != nil { + ingressSvcsErrorChan <- err + } + }() + } - // Wait on tailscaled process. It won't be cleaned up by default when the - // container exits as it is not PID1. TODO (irbekrm): perhaps we can replace the - // reaper by a running cmd.Wait in a goroutine immediately after starting - // tailscaled? - reaper := func() { - defer wg.Done() - for { - var status unix.WaitStatus - _, err := unix.Wait4(daemonProcess.Pid, &status, 0, nil) - if errors.Is(err, unix.EINTR) { - continue - } - if err != nil { - log.Fatalf("Waiting for tailscaled to exit: %v", err) - } - log.Print("tailscaled exited") - os.Exit(0) + // Wait on tailscaled process. It won't be cleaned up by default when the + // container exits as it is not PID1. TODO (irbekrm): perhaps we can replace the + // reaper by a running cmd.Wait in a goroutine immediately after starting + // tailscaled? + reaper := func() { + defer wg.Done() + for { + var status unix.WaitStatus + _, err := unix.Wait4(daemonProcess.Pid, &status, 0, nil) + if errors.Is(err, unix.EINTR) { + continue } + if err != nil { + log.Fatalf("Waiting for tailscaled to exit: %v", err) + } + log.Print("tailscaled exited") + os.Exit(0) } - wg.Add(1) - go reaper() } + wg.Add(1) + go reaper() } - case <-tc: - newBackendAddrs, err := resolveDNS(ctx, cfg.ProxyTargetDNSName) - if err != nil { - log.Printf("[unexpected] error resolving DNS name %s: %v", cfg.ProxyTargetDNSName, err) - resetTimer(true) - continue - } - backendsHaveChanged := !(slices.EqualFunc(backendAddrs, newBackendAddrs, func(ip1 net.IP, ip2 net.IP) bool { - return slices.ContainsFunc(newBackendAddrs, func(ip net.IP) bool { return ip.Equal(ip1) }) - })) - if backendsHaveChanged && len(addrs) != 0 { - log.Printf("Backend address change detected, installing proxy rules for backends %v", newBackendAddrs) - if err := installIngressForwardingRuleForDNSTarget(ctx, newBackendAddrs, addrs, nfr); err != nil { - return fmt.Errorf("installing ingress proxy rules for DNS target %s: %v", cfg.ProxyTargetDNSName, err) - } - } - backendAddrs = newBackendAddrs - resetTimer(false) - case e := <-egressSvcsErrorChan: - return fmt.Errorf("egress proxy failed: %v", e) - case e := <-ingressSvcsErrorChan: - return fmt.Errorf("ingress proxy failed: %v", e) } } wg.Wait() @@ -892,3 +1112,70 @@ func runHTTPServer(mux *http.ServeMux, addr string) (close func() error) { return errors.Join(err, ln.Close()) } } + +// resolveTailnetFQDN resolves a tailnet FQDN to a list of IP prefixes, which +// can be either a peer device or a Tailscale Service. +func resolveTailnetFQDN(nm netmapState, fqdn string) ([]netip.Prefix, error) { + dnsFQDN, err := dnsname.ToFQDN(fqdn) + if err != nil { + return nil, fmt.Errorf("error parsing %q as FQDN: %w", fqdn, err) + } + + // Check all peer devices first. + var ret []netip.Prefix + for p := range nm.peers() { + if strings.EqualFold(p.Name(), dnsFQDN.WithTrailingDot()) { + ret = p.Addresses().AsSlice() + break + } + } + if ret != nil { + return ret, nil + } + + // If not found yet, check for a matching Tailscale Service. + if svcIPs := serviceIPsFromNetMap(nm, dnsFQDN); len(svcIPs) != 0 { + return svcIPs, nil + } + + return nil, fmt.Errorf("could not find Tailscale node or service %q; it either does not exist, or not reachable because of ACLs", fqdn) +} + +// serviceIPsFromNetMap returns all IPs of a Tailscale Service if its FQDN is +// found in the netmap. Note that Tailscale Services are not a first-class +// object in the netmap, so we guess based on DNS ExtraRecords and AllowedIPs. +func serviceIPsFromNetMap(nm netmapState, fqdn dnsname.FQDN) []netip.Prefix { + var extraRecords []tailcfg.DNSRecord + for _, rec := range nm.dnsExtraRecords.All() { + recFQDN, err := dnsname.ToFQDN(rec.Name) + if err != nil { + continue + } + if strings.EqualFold(fqdn.WithTrailingDot(), recFQDN.WithTrailingDot()) { + extraRecords = append(extraRecords, rec) + } + } + + if len(extraRecords) == 0 { + return nil + } + + // Validate we can see a peer advertising the Tailscale Service. + var prefixes []netip.Prefix + for _, extraRecord := range extraRecords { + ip, err := netip.ParseAddr(extraRecord.Value) + if err != nil { + continue + } + ipPrefix := netip.PrefixFrom(ip, ip.BitLen()) + for ps := range nm.peers() { + for _, allowedIP := range ps.AllowedIPs().All() { + if allowedIP == ipPrefix { + prefixes = append(prefixes, ipPrefix) + } + } + } + } + + return prefixes +} diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index c7293c77a4afa..6b64f3c433db4 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -15,6 +15,7 @@ import ( "fmt" "io" "io/fs" + "maps" "net" "net/http" "net/http/httptest" @@ -31,22 +32,28 @@ import ( "github.com/google/go-cmp/cmp" "golang.org/x/sys/unix" + "tailscale.com/cmd/testwrapper/flakytest" + "tailscale.com/health" "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstest" - "tailscale.com/types/netmap" - "tailscale.com/types/ptr" + "tailscale.com/types/key" ) +const configFileAuthKey = "some-auth-key" + func TestContainerBoot(t *testing.T) { + flakytest.Mark(t, "https://github.com/tailscale/tailscale/issues/19380") boot := filepath.Join(t.TempDir(), "containerboot") if err := exec.Command("go", "build", "-ldflags", "-X main.testSleepDuration=1ms", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil { t.Fatalf("Building containerboot: %v", err) } - egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net") + egressStatus := egressSvcStatus("foo", "foo.tailnetxyz.ts.net", "100.64.0.2") + egressStatusUpdated := egressSvcStatus("foo", "foo.tailnetxyz.ts.net", "100.64.0.3") metricsURL := func(port int) string { return fmt.Sprintf("http://127.0.0.1:%d/metrics", port) @@ -77,6 +84,10 @@ func TestContainerBoot(t *testing.T) { // phase (simulates our fake tailscaled doing it). UpdateKubeSecret map[string]string + // Update files with these paths/contents at the beginning of the phase + // (simulates the operator updating mounted config files). + UpdateFiles map[string]string + // WantFiles files that should exist in the container and their // contents. WantFiles map[string]string @@ -95,13 +106,11 @@ func TestContainerBoot(t *testing.T) { EndpointStatuses map[string]int } runningNotify := &ipn.Notify{ - State: ptr.To(ipn.Running), - NetMap: &netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("myID"), - Name: "test-node.test.ts.net", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, - }).View(), + State: new(ipn.Running), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("myID"), + Name: "test-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, } type testCase struct { @@ -356,7 +365,7 @@ func TestContainerBoot(t *testing.T) { return testCase{ Env: map[string]string{ "TS_AUTHKEY": "tskey-key", - "TS_TAILNET_TARGET_FQDN": "ipv6-node.test.ts.net", // resolves to IPv6 address + "TS_TAILNET_TARGET_FQDN": "ipv6-node.test.ts.net.", // resolves to IPv6 address "TS_USERSPACE": "false", "TS_TEST_FAKE_NETFILTER_6": "false", }, @@ -373,24 +382,22 @@ func TestContainerBoot(t *testing.T) { }, { Notify: &ipn.Notify{ - State: ptr.To(ipn.Running), - NetMap: &netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("myID"), - Name: "test-node.test.ts.net", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, - }).View(), - Peers: []tailcfg.NodeView{ - (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("ipv6ID"), - Name: "ipv6-node.test.ts.net", - Addresses: []netip.Prefix{netip.MustParsePrefix("::1/128")}, - }).View(), + State: new(ipn.Running), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("myID"), + Name: "test-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, + }, + PeersChanged: []*tailcfg.Node{ + { + StableID: tailcfg.StableNodeID("ipv6ID"), + Name: "ipv6-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("::1/128")}, }, }, }, WantLog: "no forwarding rules for egress addresses [::1/128], host supports IPv6: false", - WantExitCode: ptr.To(1), + WantExitCode: new(1), }, }, } @@ -409,7 +416,7 @@ func TestContainerBoot(t *testing.T) { }, { Notify: &ipn.Notify{ - State: ptr.To(ipn.NeedsLogin), + State: new(ipn.NeedsLogin), }, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", @@ -440,7 +447,7 @@ func TestContainerBoot(t *testing.T) { }, { Notify: &ipn.Notify{ - State: ptr.To(ipn.NeedsLogin), + State: new(ipn.NeedsLogin), }, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true --authkey=tskey-key", @@ -460,6 +467,7 @@ func TestContainerBoot(t *testing.T) { Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + "POD_UID": "some-pod-uid", }, KubeSecret: map[string]string{ "authkey": "tskey-key", @@ -471,17 +479,20 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: "some-pod-uid", }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net.", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: "some-pod-uid", }, }, }, @@ -554,18 +565,20 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { Notify: &ipn.Notify{ - State: ptr.To(ipn.NeedsLogin), + State: new(ipn.NeedsLogin), }, WantCmds: []string{ "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { @@ -574,10 +587,10 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false", }, WantKubeSecret: map[string]string{ - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "device_fqdn": "test-node.test.ts.net.", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, }, @@ -599,36 +612,35 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net.", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, { Notify: &ipn.Notify{ - State: ptr.To(ipn.Running), - NetMap: &netmap.NetworkMap{ - SelfNode: (&tailcfg.Node{ - StableID: tailcfg.StableNodeID("newID"), - Name: "new-name.test.ts.net", - Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, - }).View(), + State: new(ipn.Running), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("newID"), + Name: "new-name.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, }, }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "new-name.test.ts.net", - "device_id": "newID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "new-name.test.ts.net.", + "device_id": "newID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, }, @@ -774,6 +786,127 @@ func TestContainerBoot(t *testing.T) { }, } }, + "sets_reissue_authkey_if_needs_login": func(env *testEnv) testCase { + newAuthKey := "new-reissued-auth-key" + return testCase{ + Env: map[string]string{ + "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), + "KUBERNETES_SERVICE_HOST": env.kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + }, + Phases: []phase{ + { + UpdateFiles: map[string]string{ + "etc/tailscaled/..data": "", + }, + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + }, { + Notify: &ipn.Notify{ + State: new(ipn.NeedsLogin), + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyReissueAuthkey: configFileAuthKey, + }, + WantLog: "watching for config changes via fsnotify", + }, { + UpdateFiles: map[string]string{ + "etc/tailscaled/cap-95.hujson": fmt.Sprintf(`{"Version":"alpha0","AuthKey":"%s"}`, newAuthKey), + "etc/tailscaled/..data": "updated", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + WantExitCode: new(0), + WantLog: "Successfully received new auth key, restarting to apply configuration", + }, + }, + } + }, + "sets_reissue_authkey_if_auth_fails": func(env *testEnv) testCase { + newAuthKey := "new-reissued-auth-key" + return testCase{ + Env: map[string]string{ + "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), + "KUBERNETES_SERVICE_HOST": env.kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + }, + Phases: []phase{ + { + UpdateFiles: map[string]string{ + "etc/tailscaled/..data": "", + }, + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + }, { + Notify: &ipn.Notify{ + Health: &health.State{ + Warnings: map[health.WarnableCode]health.UnhealthyState{ + health.LoginStateWarnable.Code: {}, + }, + }, + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyReissueAuthkey: configFileAuthKey, + }, + WantLog: "watching for config changes via fsnotify", + }, { + UpdateFiles: map[string]string{ + "etc/tailscaled/cap-95.hujson": fmt.Sprintf(`{"Version":"alpha0","AuthKey":"%s"}`, newAuthKey), + "etc/tailscaled/..data": "updated", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + }, + WantExitCode: new(0), + WantLog: "Successfully received new auth key, restarting to apply configuration", + }, + }, + } + }, + "clears_reissue_authkey_on_change": func(env *testEnv) testCase { + return testCase{ + Env: map[string]string{ + "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR": filepath.Join(env.d, "etc/tailscaled/"), + "KUBERNETES_SERVICE_HOST": env.kube.Host, + "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + }, + KubeSecret: map[string]string{ + kubetypes.KeyReissueAuthkey: "some-older-authkey", + "foo": "bar", // Check not everything is cleared. + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking --config=/etc/tailscaled/cap-95.hujson", + }, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + "foo": "bar", + }, + }, { + Notify: runningNotify, + WantKubeSecret: map[string]string{ + kubetypes.KeyCapVer: capver, + "foo": "bar", + kubetypes.KeyDeviceFQDN: "test-node.test.ts.net.", + kubetypes.KeyDeviceID: "myID", + kubetypes.KeyDeviceIPs: `["100.64.0.1"]`, + }, + }, + }, + } + }, "metrics_enabled": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ @@ -912,18 +1045,19 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "https_endpoint": "no-https", - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net.", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + "https_endpoint": "no-https", + kubetypes.KeyCapVer: capver, }, }, }, @@ -947,26 +1081,58 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, }, }, { - Notify: runningNotify, + Notify: &ipn.Notify{ + State: new(ipn.Running), + SelfChange: &tailcfg.Node{ + StableID: tailcfg.StableNodeID("myID"), + Name: "test-node.test.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32")}, + }, + PeersChanged: []*tailcfg.Node{ + { + StableID: tailcfg.StableNodeID("fooID"), + Name: "foo.tailnetxyz.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.2/32")}, + }, + }, + }, WantKubeSecret: map[string]string{ - "egress-services": mustBase64(t, egressStatus), - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "egress-services": string(mustJSON(t, egressStatus)), + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net.", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, }, }, + { + Notify: &ipn.Notify{ + PeersChanged: []*tailcfg.Node{{ + StableID: tailcfg.StableNodeID("fooID"), + Name: "foo.tailnetxyz.ts.net.", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.3/32")}, + }}, + }, + WantKubeSecret: map[string]string{ + "egress-services": string(mustJSON(t, egressStatusUpdated)), + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net.", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, + }, + }, }, } }, @@ -979,7 +1145,26 @@ func TestContainerBoot(t *testing.T) { Phases: []phase{ { WantLog: "TS_EGRESS_PROXIES_CONFIG_PATH is only supported for Tailscale running on Kubernetes", - WantExitCode: ptr.To(1), + WantExitCode: new(1), + }, + }, + } + }, + "serve_config_with_service_auto_advertisement": func(env *testEnv) testCase { + return testCase{ + Env: map[string]string{ + "TS_SERVE_CONFIG": filepath.Join(env.d, "etc/tailscaled/serve-config-with-services.json"), + "TS_AUTHKEY": "tskey-key", + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", + }, + }, + { + Notify: runningNotify, }, }, } @@ -1002,13 +1187,14 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { // SIGTERM before state is finished writing, should wait for // consistent state before propagating SIGTERM to tailscaled. - Signal: ptr.To(unix.SIGTERM), + Signal: new(unix.SIGTERM), UpdateKubeSecret: map[string]string{ "_machinekey": "foo", "_profiles": "foo", @@ -1016,10 +1202,11 @@ func TestContainerBoot(t *testing.T) { // Missing "_current-profile" key. }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "_machinekey": "foo", - "_profiles": "foo", - "profile-baff": "foo", + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + kubetypes.KeyCapVer: capver, }, WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"", }, @@ -1029,14 +1216,15 @@ func TestContainerBoot(t *testing.T) { "_current-profile": "foo", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "_machinekey": "foo", - "_profiles": "foo", - "profile-baff": "foo", - "_current-profile": "foo", + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + "_current-profile": "foo", + kubetypes.KeyCapVer: capver, }, WantLog: "HTTP server at [::]:9002 closed", - WantExitCode: ptr.To(0), + WantExitCode: new(0), }, }, } @@ -1061,7 +1249,7 @@ func TestContainerBoot(t *testing.T) { fmt.Sprintf("TS_TEST_SOCKET=%s", env.lapi.Path), fmt.Sprintf("TS_SOCKET=%s", env.runningSockPath), fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", env.d), - fmt.Sprint("TS_TEST_FAKE_NETFILTER=true"), + "TS_TEST_FAKE_NETFILTER=true", } for k, v := range tc.Env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) @@ -1087,19 +1275,28 @@ func TestContainerBoot(t *testing.T) { for k, v := range p.UpdateKubeSecret { env.kube.SetSecret(k, v) } + for path, content := range p.UpdateFiles { + fullPath := filepath.Join(env.d, path) + if err := os.WriteFile(fullPath, []byte(content), 0700); err != nil { + t.Fatalf("phase %d: updating file %q: %v", i, path, err) + } + // Explicitly update mtime to ensure fsnotify detects the change. + // Without this, file operations can be buffered and fsnotify events may not trigger. + now := time.Now() + if err := os.Chtimes(fullPath, now, now); err != nil { + t.Fatalf("phase %d: updating mtime for %q: %v", i, path, err) + } + } + if p.Notify != nil && p.Notify.InitialStatus == nil { + // Shallow-copy before mutating to avoid a race with + // parallel subtests that share the same *ipn.Notify. + p.Notify = new(*p.Notify) + p.Notify.InitialStatus = statusFromNotify(p.Notify) + } env.lapi.Notify(p.Notify) if p.Signal != nil { cmd.Process.Signal(*p.Signal) } - if p.WantLog != "" { - err := tstest.WaitFor(2*time.Second, func() error { - waitLogLine(t, time.Second, cbOut, p.WantLog) - return nil - }) - if err != nil { - t.Fatal(err) - } - } if p.WantExitCode != nil { state, err := cmd.Process.Wait() @@ -1109,14 +1306,19 @@ func TestContainerBoot(t *testing.T) { if state.ExitCode() != *p.WantExitCode { t.Fatalf("phase %d: want exit code %d, got %d", i, *p.WantExitCode, state.ExitCode()) } + } - // Early test return, we don't expect the successful startup log message. - return + if p.WantLog != "" { + err := tstest.WaitFor(5*time.Second, func() error { + waitLogLine(t, 5*time.Second, cbOut, p.WantLog) + return nil + }) + if err != nil { + t.Fatal(err) + } } - wantCmds = append(wantCmds, p.WantCmds...) - waitArgs(t, 2*time.Second, env.d, env.argFile, strings.Join(wantCmds, "\n")) - err := tstest.WaitFor(2*time.Second, func() error { + err := tstest.WaitFor(5*time.Second, func() error { if p.WantKubeSecret != nil { got := env.kube.Secret() if diff := cmp.Diff(got, p.WantKubeSecret); diff != "" { @@ -1131,8 +1333,18 @@ func TestContainerBoot(t *testing.T) { return nil }) if err != nil { - t.Fatalf("phase %d: %v", i, err) + t.Fatalf("test: %q phase %d: %v", name, i, err) } + + // if we provide a wanted exit code, we expect that the process is finished, + // so should return from the test. + if p.WantExitCode != nil { + return + } + + wantCmds = append(wantCmds, p.WantCmds...) + waitArgs(t, 2*time.Second, env.d, env.argFile, strings.Join(wantCmds, "\n")) + err = tstest.WaitFor(2*time.Second, func() error { for path, want := range p.WantFiles { gotBs, err := os.ReadFile(filepath.Join(env.d, path)) @@ -1203,7 +1415,7 @@ func (b *lockingBuffer) String() string { func waitLogLine(t *testing.T, timeout time.Duration, b *lockingBuffer, want string) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - for _, line := range strings.Split(b.String(), "\n") { + for line := range strings.SplitSeq(b.String(), "\n") { if !strings.HasPrefix(line, "boot: ") { continue } @@ -1275,8 +1487,8 @@ type localAPI struct { notify *ipn.Notify } -func (l *localAPI) Start() error { - path := filepath.Join(l.FSRoot, "tmp/tailscaled.sock.fake") +func (lc *localAPI) Start() error { + path := filepath.Join(lc.FSRoot, "tmp/tailscaled.sock.fake") if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { return err } @@ -1286,36 +1498,79 @@ func (l *localAPI) Start() error { return err } - l.srv = &http.Server{ - Handler: l, + lc.srv = &http.Server{ + Handler: lc, } - l.Path = path - l.cond = sync.NewCond(&l.Mutex) - go l.srv.Serve(ln) + lc.Path = path + lc.cond = sync.NewCond(&lc.Mutex) + go lc.srv.Serve(ln) return nil } -func (l *localAPI) Close() { - l.srv.Close() +func (lc *localAPI) Close() { + lc.srv.Close() } -func (l *localAPI) Notify(n *ipn.Notify) { +func (lc *localAPI) Notify(n *ipn.Notify) { if n == nil { return } - l.Lock() - defer l.Unlock() - l.notify = n - l.cond.Broadcast() + lc.Lock() + defer lc.Unlock() + lc.notify = n + lc.cond.Broadcast() +} + +func statusFromNotify(n *ipn.Notify) *ipnstate.Status { + st := new(ipnstate.Status) + if n.State != nil { + st.BackendState = n.State.String() + } + if n.SelfChange != nil { + st.Self = peerStatusFromNode(n.SelfChange.View()) + } + if len(n.PeersChanged) != 0 { + st.Peer = map[key.NodePublic]*ipnstate.PeerStatus{} + for _, p := range n.PeersChanged { + pv := p.View() + st.Peer[pv.Key()] = peerStatusFromNode(pv) + } + } + return st } -func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func peerStatusFromNode(n tailcfg.NodeView) *ipnstate.PeerStatus { + ps := &ipnstate.PeerStatus{ + ID: n.StableID(), + NodeID: n.ID(), + PublicKey: n.Key(), + DNSName: n.Name(), + } + for _, p := range n.Addresses().All() { + if p.IsSingleIP() { + ps.TailscaleIPs = append(ps.TailscaleIPs, p.Addr()) + } + } + if n.AllowedIPs().Len() != 0 { + v := n.AllowedIPs() + ps.AllowedIPs = &v + } + return ps +} + +func (lc *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/localapi/v0/serve-config": - if r.Method != "POST" { + switch r.Method { + case "GET": + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(&ipn.ServeConfig{}) + return + case "POST": + return + default: panic(fmt.Sprintf("unsupported method %q", r.Method)) } - return case "/localapi/v0/watch-ipn-bus": if r.Method != "GET" { panic(fmt.Sprintf("unsupported method %q", r.Method)) @@ -1326,6 +1581,27 @@ func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { } w.Write([]byte("fake metrics")) return + case "/localapi/v0/prefs": + switch r.Method { + case "GET": + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(&ipn.Prefs{}) + return + case "PATCH": + // EditPrefs - just return empty prefs + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(&ipn.Prefs{}) + return + default: + panic(fmt.Sprintf("unsupported method %q", r.Method)) + } + // In the localAPI ServeHTTP method + case "/localapi/v0/disconnect-control": + if r.Method != "POST" { + panic(fmt.Sprintf("unsupported method %q", r.Method)) + } + w.WriteHeader(http.StatusOK) + return default: panic(fmt.Sprintf("unsupported path %q", r.URL.Path)) } @@ -1336,11 +1612,11 @@ func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.Flush() } enc := json.NewEncoder(w) - l.Lock() - defer l.Unlock() + lc.Lock() + defer lc.Unlock() for { - if l.notify != nil { - if err := enc.Encode(l.notify); err != nil { + if lc.notify != nil { + if err := enc.Encode(lc.notify); err != nil { // Usually broken pipe as the test client disconnects. return } @@ -1348,7 +1624,7 @@ func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.Flush() } } - l.cond.Wait() + lc.cond.Wait() } } @@ -1372,9 +1648,7 @@ func (k *kubeServer) Secret() map[string]string { k.Lock() defer k.Unlock() ret := map[string]string{} - for k, v := range k.secret { - ret[k] = v - } + maps.Copy(ret, k.secret) return ret } @@ -1489,10 +1763,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { } switch r.Header.Get("Content-Type") { case "application/json-patch+json": - req := []struct { - Op string `json:"op"` - Path string `json:"path"` - }{} + req := []kubeclient.JSONPatch{} if err := json.Unmarshal(bs, &req); err != nil { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } @@ -1503,23 +1774,20 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) - case "replace": + case "add", "replace": path, ok := strings.CutPrefix(op.Path, "/data/") if !ok { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } - req := make([]kubeclient.JSONPatch, 0) - if err := json.Unmarshal(bs, &req); err != nil { - panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) + val, ok := op.Value.(string) + if !ok { + panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", op.Value)) } - - for _, patch := range req { - val, ok := patch.Value.(string) - if !ok { - panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", patch.Value)) - } - k.secret[path] = val + v, err := base64.StdEncoding.DecodeString(val) + if err != nil { + panic(fmt.Sprintf("json patch value %q is not base64 encoded: %v", val, err)) } + k.secret[path] = string(v) default: panic(fmt.Sprintf("unsupported json-patch op %q", op.Op)) } @@ -1532,7 +1800,11 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } for key, val := range req.Data { - k.secret[key] = string(val) + if val == nil { + delete(k.secret, key) + } else { + k.secret[key] = string(val) + } } default: panic(fmt.Sprintf("unknown content type %q", r.Header.Get("Content-Type"))) @@ -1542,12 +1814,6 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { } } -func mustBase64(t *testing.T, v any) string { - b := mustJSON(t, v) - s := base64.StdEncoding.WithPadding('=').EncodeToString(b) - return s -} - func mustJSON(t *testing.T, v any) []byte { b, err := json.Marshal(v) if err != nil { @@ -1557,13 +1823,14 @@ func mustJSON(t *testing.T, v any) []byte { } // egress services status given one named tailnet target specified by FQDN. As written by the proxy to its state Secret. -func egressSvcStatus(name, fqdn string) egressservices.Status { +func egressSvcStatus(name, fqdn, ip string) egressservices.Status { return egressservices.Status{ Services: map[string]*egressservices.ServiceStatus{ name: { TailnetTarget: egressservices.TailnetTarget{ FQDN: fqdn, }, + TailnetTargetIPs: []netip.Addr{netip.MustParseAddr(ip)}, }, }, } @@ -1605,8 +1872,15 @@ func newTestEnv(t *testing.T) testEnv { kube.Start(t) t.Cleanup(kube.Close) - tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"} + tailscaledConf := &ipn.ConfigVAlpha{AuthKey: new(configFileAuthKey), Version: "alpha0"} serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} + serveConfWithServices := ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:test-service-1": {}, + "svc:test-service-2": {}, + }, + } egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net") dirs := []string{ @@ -1624,15 +1898,16 @@ func newTestEnv(t *testing.T) testEnv { } } files := map[string][]byte{ - "usr/bin/tailscaled": fakeTailscaled, - "usr/bin/tailscale": fakeTailscale, - "usr/bin/iptables": fakeTailscale, - "usr/bin/ip6tables": fakeTailscale, - "dev/net/tun": []byte(""), - "proc/sys/net/ipv4/ip_forward": []byte("0"), - "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), - "etc/tailscaled/cap-95.hujson": mustJSON(t, tailscaledConf), - "etc/tailscaled/serve-config.json": mustJSON(t, serveConf), + "usr/bin/tailscaled": fakeTailscaled, + "usr/bin/tailscale": fakeTailscale, + "usr/bin/iptables": fakeTailscale, + "usr/bin/ip6tables": fakeTailscale, + "dev/net/tun": []byte(""), + "proc/sys/net/ipv4/ip_forward": []byte("0"), + "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), + "etc/tailscaled/cap-95.hujson": mustJSON(t, tailscaledConf), + "etc/tailscaled/serve-config.json": mustJSON(t, serveConf), + "etc/tailscaled/serve-config-with-services.json": mustJSON(t, serveConfWithServices), filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg), filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings): []byte("4"), } diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 37fd497779c75..f5423630f7ab2 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -9,6 +9,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "log" "os" "path/filepath" @@ -19,16 +20,19 @@ import ( "github.com/fsnotify/fsnotify" "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/kube/certs" "tailscale.com/kube/kubetypes" - "tailscale.com/types/netmap" + klc "tailscale.com/kube/localclient" + "tailscale.com/kube/services" ) // watchServeConfigChanges watches path for changes, and when it sees one, reads // the serve config from it, replacing ${TS_CERT_DOMAIN} with certDomain, and // applies it to lc. It exits when ctx is canceled. cdChanged is a channel that // is written to when the certDomain changes, causing the serve config to be -// re-read and applied. -func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings) { +// re-read and applied. prevServeConfig is the serve config that was fetched +// during startup. This will be refreshed by the goroutine when serve config changes. +func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings, prevServeConfig *ipn.ServeConfig) { if certDomainAtomic == nil { panic("certDomainAtomic must not be nil") } @@ -51,11 +55,16 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom } var certDomain string - var prevServeConfig *ipn.ServeConfig - var cm certManager + var cm *certs.CertManager if cfg.CertShareMode == "rw" { - cm = certManager{ - lc: lc, + cm = certs.NewCertManager(klc.New(lc), log.Printf) + } + + var err error + if prevServeConfig == nil { + prevServeConfig, err = lc.GetServeConfig(ctx) + if err != nil { + log.Fatalf("serve proxy: failed to get serve config: %v", err) } } for { @@ -70,49 +79,69 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom // k8s handles these mounts. So just re-read the file and apply it // if it's changed. } - sc, err := readServeConfig(cfg.ServeConfigPath, certDomain) - if err != nil { - log.Fatalf("serve proxy: failed to read serve config: %v", err) - } - if sc == nil { - log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath) - continue - } - if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) { - continue - } - if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil { - log.Fatalf("serve proxy: error updating serve config: %v", err) - } - if kc != nil && kc.canPatch { - if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil { - log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err) + + var sc *ipn.ServeConfig + if cfg.ServeConfigPath != "" { + sc, err := readServeConfig(cfg.ServeConfigPath, certDomain) + if err != nil { + log.Fatalf("serve proxy: failed to read serve config: %v", err) } + if sc == nil { + log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath) + continue + } + if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) { + continue + } + if err := updateServeConfig(ctx, sc, certDomain, klc.New(lc)); err != nil { + log.Fatalf("serve proxy: error updating serve config: %v", err) + } + if kc != nil && kc.canPatch { + if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil { + log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err) + } + } + prevServeConfig = sc + if cfg.CertShareMode != "rw" { + continue + } + if err := cm.EnsureCertLoops(ctx, sc); err != nil { + log.Fatalf("serve proxy: error ensuring cert loops: %v", err) + } + } else { + log.Printf("serve config path not provided.") + sc = prevServeConfig } - prevServeConfig = sc - if cfg.CertShareMode != "rw" { - continue - } - if err := cm.ensureCertLoops(ctx, sc); err != nil { - log.Fatalf("serve proxy: error ensuring cert loops: %v", err) + + // if we are running in kubernetes, we want to leave advertisement to the operator + // to do (by updating the serve config) + if getAutoAdvertiseBool() { + if err := refreshAdvertiseServices(ctx, sc, klc.New(lc)); err != nil { + log.Fatalf("error refreshing advertised services: %v", err) + } } } } -func certDomainFromNetmap(nm *netmap.NetworkMap) string { - if len(nm.DNS.CertDomains) == 0 { - return "" +func refreshAdvertiseServices(ctx context.Context, sc *ipn.ServeConfig, lc klc.LocalClient) error { + if sc == nil || len(sc.Services) == 0 { + return nil + } + + var svcs []string + for svc := range sc.Services { + svcs = append(svcs, svc.String()) + } + + err := services.EnsureServicesAdvertised(ctx, svcs, lc, log.Printf) + if err != nil { + return fmt.Errorf("failed to ensure services advertised: %w", err) } - return nm.DNS.CertDomains[0] -} -// localClient is a subset of [local.Client] that can be mocked for testing. -type localClient interface { - SetServeConfig(context.Context, *ipn.ServeConfig) error - CertPair(context.Context, string) ([]byte, []byte, error) + return nil } -func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error { +func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc klc.LocalClient) error { if !isValidHTTPSConfig(certDomain, sc) { return nil } diff --git a/cmd/containerboot/serve_test.go b/cmd/containerboot/serve_test.go index fc18f254dad05..5da5ef5f737c3 100644 --- a/cmd/containerboot/serve_test.go +++ b/cmd/containerboot/serve_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -12,9 +12,10 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/kube/kubetypes" + "tailscale.com/kube/localclient" + "tailscale.com/tailcfg" ) func TestUpdateServeConfig(t *testing.T) { @@ -65,13 +66,13 @@ func TestUpdateServeConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fakeLC := &fakeLocalClient{} + fakeLC := &localclient.FakeLocalClient{} err := updateServeConfig(context.Background(), tt.sc, tt.certDomain, fakeLC) if err != nil { t.Errorf("updateServeConfig() error = %v", err) } - if fakeLC.setServeCalled != tt.wantCall { - t.Errorf("SetServeConfig() called = %v, want %v", fakeLC.setServeCalled, tt.wantCall) + if fakeLC.SetServeCalled != tt.wantCall { + t.Errorf("SetServeConfig() called = %v, want %v", fakeLC.SetServeCalled, tt.wantCall) } }) } @@ -196,18 +197,114 @@ func TestReadServeConfig(t *testing.T) { } } -type fakeLocalClient struct { - *local.Client - setServeCalled bool -} +func TestRefreshAdvertiseServices(t *testing.T) { + tests := []struct { + name string + sc *ipn.ServeConfig + wantServices []string + wantEditPrefsCalled bool + wantErr bool + }{ + { + name: "nil_serve_config", + sc: nil, + wantEditPrefsCalled: false, + }, + { + name: "empty_serve_config", + sc: &ipn.ServeConfig{}, + wantEditPrefsCalled: false, + }, + { + name: "no_services_defined", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + }, + wantEditPrefsCalled: false, + }, + { + name: "single_service", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:my-service": {}, + }, + }, + wantServices: []string{"svc:my-service"}, + wantEditPrefsCalled: true, + }, + { + name: "multiple_services", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:service-a": {}, + "svc:service-b": {}, + "svc:service-c": {}, + }, + }, + wantServices: []string{"svc:service-a", "svc:service-b", "svc:service-c"}, + wantEditPrefsCalled: true, + }, + { + name: "services_with_tcp_and_web", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "example.com:443": {}, + }, + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:frontend": {}, + "svc:backend": {}, + }, + }, + wantServices: []string{"svc:frontend", "svc:backend"}, + wantEditPrefsCalled: true, + }, + } -func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConfig) error { - m.setServeCalled = true - return nil -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeLC := &localclient.FakeLocalClient{} + err := refreshAdvertiseServices(context.Background(), tt.sc, fakeLC) -func (m *fakeLocalClient) CertPair(ctx context.Context, domain string) (certPEM, keyPEM []byte, err error) { - return nil, nil, nil + if (err != nil) != tt.wantErr { + t.Errorf("refreshAdvertiseServices() error = %v, wantErr %v", err, tt.wantErr) + } + + if tt.wantEditPrefsCalled != (len(fakeLC.EditPrefsCalls) > 0) { + t.Errorf("EditPrefs called = %v, want %v", len(fakeLC.EditPrefsCalls) > 0, tt.wantEditPrefsCalled) + } + + if tt.wantEditPrefsCalled { + if len(fakeLC.EditPrefsCalls) != 1 { + t.Fatalf("expected 1 EditPrefs call, got %d", len(fakeLC.EditPrefsCalls)) + } + + mp := fakeLC.EditPrefsCalls[0] + if !mp.AdvertiseServicesSet { + t.Error("AdvertiseServicesSet should be true") + } + + if len(mp.AdvertiseServices) != len(tt.wantServices) { + t.Errorf("AdvertiseServices length = %d, want %d", len(mp.Prefs.AdvertiseServices), len(tt.wantServices)) + } + + advertised := make(map[string]bool) + for _, svc := range mp.AdvertiseServices { + advertised[svc] = true + } + + for _, want := range tt.wantServices { + if !advertised[want] { + t.Errorf("expected service %q to be advertised, but it wasn't", want) + } + } + } + }) + } } func TestHasHTTPSEndpoint(t *testing.T) { diff --git a/cmd/containerboot/services.go b/cmd/containerboot/services.go deleted file mode 100644 index 6079128c02b19..0000000000000 --- a/cmd/containerboot/services.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build linux - -package main - -import ( - "context" - "fmt" - "log" - "time" - - "tailscale.com/client/local" - "tailscale.com/ipn" -) - -// ensureServicesNotAdvertised is a function that gets called on containerboot -// termination and ensures that any currently advertised VIPServices get -// unadvertised to give clients time to switch to another node before this one -// is shut down. -func ensureServicesNotAdvertised(ctx context.Context, lc *local.Client) error { - prefs, err := lc.GetPrefs(ctx) - if err != nil { - return fmt.Errorf("error getting prefs: %w", err) - } - if len(prefs.AdvertiseServices) == 0 { - return nil - } - - log.Printf("unadvertising services: %v", prefs.AdvertiseServices) - if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{ - AdvertiseServicesSet: true, - Prefs: ipn.Prefs{ - AdvertiseServices: nil, - }, - }); err != nil { - // EditPrefs only returns an error if it fails _set_ its local prefs. - // If it fails to _persist_ the prefs in state, we don't get an error - // and we continue waiting below, as control will failover as usual. - return fmt.Errorf("error setting prefs AdvertiseServices: %w", err) - } - - // Services use the same (failover XOR regional routing) mechanism that - // HA subnet routers use. Unfortunately we don't yet get a reliable signal - // from control that it's responded to our unadvertisement, so the best we - // can do is wait for 20 seconds, where 15s is the approximate maximum time - // it should take for control to choose a new primary, and 5s is for buffer. - // - // Note: There is no guarantee that clients have been _informed_ of the new - // primary no matter how long we wait. We would need a mechanism to await - // netmap updates for peers to know for sure. - // - // See https://tailscale.com/kb/1115/high-availability for more details. - // TODO(tomhjp): Wait for a netmap update instead of sleeping when control - // supports that. - select { - case <-ctx.Done(): - return nil - case <-time.After(20 * time.Second): - return nil - } -} diff --git a/cmd/containerboot/settings.go b/cmd/containerboot/settings.go index 5a8be9036b3ca..f695f2e5db5f4 100644 --- a/cmd/containerboot/settings.go +++ b/cmd/containerboot/settings.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -22,9 +22,13 @@ import ( // settings is all the configuration for containerboot. type settings struct { - AuthKey string - Hostname string - Routes *string + AuthKey string + ClientID string + ClientSecret string + IDToken string + Audience string + Hostname string + Routes *string // ProxyTargetIP is the destination IP to which all incoming // Tailscale traffic should be proxied. If empty, no proxying // is done. This is typically a locally reachable IP. @@ -85,21 +89,30 @@ type settings struct { func configFromEnv() (*settings, error) { cfg := &settings{ - AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), - Hostname: defaultEnv("TS_HOSTNAME", ""), - Routes: defaultEnvStringPointer("TS_ROUTES"), - ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), - ProxyTargetIP: defaultEnv("TS_DEST_IP", ""), - ProxyTargetDNSName: defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""), - TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), - TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""), - DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), - ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), - InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", - UserspaceMode: defaultBool("TS_USERSPACE", true), - StateDir: defaultEnv("TS_STATE_DIR", ""), - AcceptDNS: defaultEnvBoolPointer("TS_ACCEPT_DNS"), - KubeSecret: defaultEnv("TS_KUBE_SECRET", "tailscale"), + AuthKey: defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""), + ClientID: defaultEnv("TS_CLIENT_ID", ""), + ClientSecret: defaultEnv("TS_CLIENT_SECRET", ""), + IDToken: defaultEnv("TS_ID_TOKEN", ""), + Audience: defaultEnv("TS_AUDIENCE", ""), + Hostname: defaultEnv("TS_HOSTNAME", ""), + Routes: defaultEnvStringPointer("TS_ROUTES"), + ServeConfigPath: defaultEnv("TS_SERVE_CONFIG", ""), + ProxyTargetIP: defaultEnv("TS_DEST_IP", ""), + ProxyTargetDNSName: defaultEnv("TS_EXPERIMENTAL_DEST_DNS_NAME", ""), + TailnetTargetIP: defaultEnv("TS_TAILNET_TARGET_IP", ""), + TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""), + DaemonExtraArgs: defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""), + ExtraArgs: defaultEnv("TS_EXTRA_ARGS", ""), + InKubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", + UserspaceMode: defaultBool("TS_USERSPACE", true), + StateDir: defaultEnv("TS_STATE_DIR", ""), + AcceptDNS: defaultEnvBoolPointer("TS_ACCEPT_DNS"), + KubeSecret: func() string { + if os.Getenv("KUBERNETES_SERVICE_HOST") != "" { + return defaultEnv("TS_KUBE_SECRET", "tailscale") + } + return defaultEnv("TS_KUBE_SECRET", "") + }(), SOCKSProxyAddr: defaultEnv("TS_SOCKS5_SERVER", ""), HTTPProxyAddr: defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""), Socket: defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"), @@ -118,6 +131,7 @@ func configFromEnv() (*settings, error) { IngressProxiesCfgPath: defaultEnv("TS_INGRESS_PROXIES_CONFIG_PATH", ""), PodUID: defaultEnv("POD_UID", ""), } + podIPs, ok := os.LookupEnv("POD_IPS") if ok { ips := strings.Split(podIPs, ",") @@ -136,6 +150,7 @@ func configFromEnv() (*settings, error) { cfg.PodIPv6 = parsed.String() } } + // If cert share is enabled, set the replica as read or write. Only 0th // replica should be able to write. isInCertShareMode := defaultBool("TS_EXPERIMENTAL_CERT_SHARE", false) @@ -157,9 +172,19 @@ func configFromEnv() (*settings, error) { cfg.AcceptDNS = &acceptDNSNew } + // In Kubernetes clusters, people like to use the "$(POD_IP):PORT" combination to configure the TS_LOCAL_ADDR_PORT + // environment variable (we even do this by default in the operator when enabling metrics), leading to a v6 address + // and port combo we cannot parse, as netip.ParseAddrPort expects the host segment to be enclosed in square brackets. + // We perform a check here to see if TS_LOCAL_ADDR_PORT is using the pod's IPv6 address and is not using brackets, + // adding the brackets in if need be. + if cfg.PodIPv6 != "" && strings.Contains(cfg.LocalAddrPort, cfg.PodIPv6) && !strings.ContainsAny(cfg.LocalAddrPort, "[]") { + cfg.LocalAddrPort = strings.Replace(cfg.LocalAddrPort, cfg.PodIPv6, "["+cfg.PodIPv6+"]", 1) + } + if err := cfg.validate(); err != nil { return nil, fmt.Errorf("invalid configuration: %v", err) } + return cfg, nil } @@ -241,8 +266,46 @@ func (s *settings) validate() error { if s.TailnetTargetFQDN != "" && s.TailnetTargetIP != "" { return errors.New("Both TS_TAILNET_TARGET_IP and TS_TAILNET_FQDN cannot be set") } - if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") { - return errors.New("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.") + if s.TailscaledConfigFilePath != "" && + (s.AcceptDNS != nil || + s.AuthKey != "" || + s.Routes != nil || + s.ExtraArgs != "" || + s.Hostname != "" || + s.ClientID != "" || + s.ClientSecret != "" || + s.IDToken != "" || + s.Audience != "") { + conflictingArgs := []string{ + "TS_HOSTNAME", + "TS_EXTRA_ARGS", + "TS_AUTHKEY", + "TS_ROUTES", + "TS_ACCEPT_DNS", + "TS_CLIENT_ID", + "TS_CLIENT_SECRET", + "TS_ID_TOKEN", + "TS_AUDIENCE", + } + return fmt.Errorf("TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR cannot be set in combination with %s.", strings.Join(conflictingArgs, ", ")) + } + if s.IDToken != "" && s.ClientID == "" { + return errors.New("TS_ID_TOKEN is set but TS_CLIENT_ID is not set") + } + if s.Audience != "" && s.ClientID == "" { + return errors.New("TS_AUDIENCE is set but TS_CLIENT_ID is not set") + } + if s.IDToken != "" && s.ClientSecret != "" { + return errors.New("TS_ID_TOKEN and TS_CLIENT_SECRET cannot both be set") + } + if s.IDToken != "" && s.Audience != "" { + return errors.New("TS_ID_TOKEN and TS_AUDIENCE cannot both be set") + } + if s.Audience != "" && s.ClientSecret != "" { + return errors.New("TS_AUDIENCE and TS_CLIENT_SECRET cannot both be set") + } + if s.AuthKey != "" && (s.ClientID != "" || s.ClientSecret != "" || s.IDToken != "" || s.Audience != "") { + return errors.New("TS_AUTHKEY cannot be used with TS_CLIENT_ID, TS_CLIENT_SECRET, TS_ID_TOKEN, or TS_AUDIENCE.") } if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode { return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode") @@ -312,8 +375,8 @@ func (cfg *settings) setupKube(ctx context.Context, kc *kubeClient) error { } } - // Return early if we already have an auth key. - if cfg.AuthKey != "" || isOneStepConfig(cfg) { + // Return early if we already have an auth key or are using OAuth/WIF. + if cfg.AuthKey != "" || cfg.ClientID != "" || cfg.ClientSecret != "" || isOneStepConfig(cfg) { return nil } diff --git a/cmd/containerboot/settings_test.go b/cmd/containerboot/settings_test.go index dbec066c9ab0d..eca50101b6c70 100644 --- a/cmd/containerboot/settings_test.go +++ b/cmd/containerboot/settings_test.go @@ -1,11 +1,15 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux package main -import "testing" +import ( + "net/netip" + "strings" + "testing" +) func Test_parseAcceptDNS(t *testing.T) { tests := []struct { @@ -106,3 +110,147 @@ func Test_parseAcceptDNS(t *testing.T) { }) } } + +func TestValidateAuthMethods(t *testing.T) { + tests := []struct { + name string + authKey string + clientID string + clientSecret string + idToken string + audience string + errContains string + }{ + { + name: "no_auth_method", + }, + { + name: "authkey_only", + authKey: "tskey-auth-xxx", + }, + { + name: "client_secret_only", + clientSecret: "tskey-client-xxx", + }, + { + name: "client_id_alone", + clientID: "client-id", + }, + { + name: "oauth_client_id_and_secret", + clientID: "client-id", + clientSecret: "tskey-client-xxx", + }, + { + name: "wif_client_id_and_id_token", + clientID: "client-id", + idToken: "id-token", + }, + { + name: "wif_client_id_and_audience", + clientID: "client-id", + audience: "audience", + }, + { + name: "id_token_without_client_id", + idToken: "id-token", + errContains: "TS_ID_TOKEN is set but TS_CLIENT_ID is not set", + }, + { + name: "audience_without_client_id", + audience: "audience", + errContains: "TS_AUDIENCE is set but TS_CLIENT_ID is not set", + }, + { + name: "authkey_with_client_secret", + authKey: "tskey-auth-xxx", + clientSecret: "tskey-client-xxx", + errContains: "TS_AUTHKEY cannot be used with", + }, + { + name: "authkey_with_id_token", + authKey: "tskey-auth-xxx", + clientID: "client-id", + idToken: "id-token", + errContains: "TS_AUTHKEY cannot be used with", + }, + { + name: "authkey_with_audience", + authKey: "tskey-auth-xxx", + clientID: "client-id", + audience: "audience", + errContains: "TS_AUTHKEY cannot be used with", + }, + { + name: "id_token_with_client_secret", + clientID: "client-id", + clientSecret: "tskey-client-xxx", + idToken: "id-token", + errContains: "TS_ID_TOKEN and TS_CLIENT_SECRET cannot both be set", + }, + { + name: "id_token_with_audience", + clientID: "client-id", + idToken: "id-token", + audience: "audience", + errContains: "TS_ID_TOKEN and TS_AUDIENCE cannot both be set", + }, + { + name: "audience_with_client_secret", + clientID: "client-id", + clientSecret: "tskey-client-xxx", + audience: "audience", + errContains: "TS_AUDIENCE and TS_CLIENT_SECRET cannot both be set", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &settings{ + AuthKey: tt.authKey, + ClientID: tt.clientID, + ClientSecret: tt.clientSecret, + IDToken: tt.idToken, + Audience: tt.audience, + } + err := s.validate() + if tt.errContains != "" { + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("error %q does not contain %q", err.Error(), tt.errContains) + } + } else if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +func TestHandlesKubeIPV6(t *testing.T) { + t.Setenv("TS_LOCAL_ADDR_PORT", "fd7a:115c:a1e0::6c34:352:9002") + t.Setenv("POD_IPS", "fd7a:115c:a1e0::6c34:352") + + cfg, err := configFromEnv() + if err != nil { + t.Fatal(err) + } + + if cfg.LocalAddrPort != "[fd7a:115c:a1e0::6c34:352]:9002" { + t.Errorf("LocalAddrPort is not set correctly") + } + + parsed, err := netip.ParseAddrPort(cfg.LocalAddrPort) + if err != nil { + t.Fatal(err) + } + + if !parsed.Addr().Is6() { + t.Errorf("expected v6 address but got %s", parsed) + } + + if parsed.Port() != 9002 { + t.Errorf("expected port 9002 but got %d", parsed.Port()) + } +} diff --git a/cmd/containerboot/tailscaled.go b/cmd/containerboot/tailscaled.go index f828c52573089..6f4ed77e76d72 100644 --- a/cmd/containerboot/tailscaled.go +++ b/cmd/containerboot/tailscaled.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux @@ -69,7 +69,7 @@ func startTailscaled(ctx context.Context, cfg *settings) (*local.Client, *os.Pro func tailscaledArgs(cfg *settings) []string { args := []string{"--socket=" + cfg.Socket} switch { - case cfg.InKubernetes && cfg.KubeSecret != "": + case cfg.KubeSecret != "": args = append(args, "--state=kube:"+cfg.KubeSecret) if cfg.StateDir == "" { cfg.StateDir = "/tmp" @@ -120,6 +120,18 @@ func tailscaleUp(ctx context.Context, cfg *settings) error { if cfg.AuthKey != "" { args = append(args, "--authkey="+cfg.AuthKey) } + if cfg.ClientID != "" { + args = append(args, "--client-id="+cfg.ClientID) + } + if cfg.ClientSecret != "" { + args = append(args, "--client-secret="+cfg.ClientSecret) + } + if cfg.IDToken != "" { + args = append(args, "--id-token="+cfg.IDToken) + } + if cfg.Audience != "" { + args = append(args, "--audience="+cfg.Audience) + } // --advertise-routes can be passed an empty string to configure a // device (that might have previously advertised subnet routes) to not // advertise any routes. Respect an empty string passed by a user and diff --git a/cmd/derper/ace.go b/cmd/derper/ace.go new file mode 100644 index 0000000000000..ae2d0cbebb413 --- /dev/null +++ b/cmd/derper/ace.go @@ -0,0 +1,77 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// TODO: docs about all this + +package main + +import ( + "errors" + "fmt" + "net" + "net/http" + "strings" + + "tailscale.com/derp/derpserver" + "tailscale.com/net/connectproxy" +) + +// serveConnect handles a CONNECT request for ACE support. +func serveConnect(s *derpserver.Server, w http.ResponseWriter, r *http.Request) { + if !*flagACEEnabled { + http.Error(w, "CONNECT not enabled", http.StatusForbidden) + return + } + if r.TLS == nil { + // This should already be enforced by the caller of serveConnect, but + // double check. + http.Error(w, "CONNECT requires TLS", http.StatusForbidden) + return + } + + ch := &connectproxy.Handler{ + Check: func(hostPort string) error { + host, port, err := net.SplitHostPort(hostPort) + if err != nil { + return err + } + if port != "443" && port != "80" { + // There are only two types of CONNECT requests the client makes + // via ACE: requests for /key (port 443) and requests to upgrade + // to the bidirectional ts2021 Noise protocol. + // + // The ts2021 layer can bootstrap over port 80 (http) or port + // 443 (https). + // + // Without ACE, we prefer port 80 to avoid unnecessary double + // encryption. But enough places require TLS+port 443 that we do + // support that double encryption path as a fallback. + // + // But ACE adds its own TLS layer (ACE is always CONNECT over + // https). If we don't permit port 80 here as a target, we'd + // have three layers of encryption (TLS + TLS + Noise) which is + // even more silly than two. + // + // So we permit port 80 such that we can only have two layers of + // encryption, varying by the request type: + // + // 1. TLS from client to ACE proxy (CONNECT) + // 2a. TLS from ACE proxy to https://controlplane.tailscale.com/key (port 443) + // 2b. ts2021 Noise from ACE proxy to http://controlplane.tailscale.com/ts2021 (port 80) + // + // But nothing's stopping the client from doing its ts2021 + // upgrade over https anyway and having three layers of + // encryption. But we can at least permit the client to do a + // "CONNECT controlplane.tailscale.com:80 HTTP/1.1" if it wants. + return fmt.Errorf("only ports 443 and 80 are allowed") + } + // TODO(bradfitz): make policy configurable from flags and/or come + // from local tailscaled nodeAttrs + if !strings.HasSuffix(host, ".tailscale.com") || strings.Contains(host, "derp") { + return errors.New("bad host") + } + return nil + }, + } + ch.ServeHTTP(w, r) +} diff --git a/cmd/derper/bootstrap_dns.go b/cmd/derper/bootstrap_dns.go index a58f040bae687..9abc95df56878 100644 --- a/cmd/derper/bootstrap_dns.go +++ b/cmd/derper/bootstrap_dns.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/derper/bootstrap_dns_test.go b/cmd/derper/bootstrap_dns_test.go index 9b99103abfe33..2055b97511940 100644 --- a/cmd/derper/bootstrap_dns_test.go +++ b/cmd/derper/bootstrap_dns_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -41,8 +41,28 @@ func (b *bitbucketResponseWriter) Write(p []byte) (int, error) { return len(p), func (b *bitbucketResponseWriter) WriteHeader(statusCode int) {} +// setDNSCache sets the published DNS cache for tests. +func setDNSCache(tb testing.TB, m *dnsEntryMap) { + tb.Helper() + j, err := json.Marshal(m.IPs) + if err != nil { + tb.Fatal(err) + } + tstest.AssertNotParallel(tb) + dnsCache.Store(m) + dnsCacheBytes.Store(j) + tb.Cleanup(func() { + dnsCache.Store(nil) + dnsCacheBytes.Store(nil) + }) +} + func getBootstrapDNS(t *testing.T, q string) map[string][]net.IP { t.Helper() + tstest.AssertNotParallel(t) + if dnsCache.Load() == nil { + t.Fatal("dnsCache not initialized; call setDNSCache before getBootstrapDNS") + } req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape(q), nil) w := httptest.NewRecorder() handleBootstrapDNS(w, req) @@ -100,7 +120,8 @@ func TestUnpublishedDNS(t *testing.T) { } } -func resetMetrics() { +func resetMetrics(tb testing.TB) { + tstest.AssertNotParallel(tb) publishedDNSHits.Set(0) publishedDNSMisses.Set(0) unpublishedDNSHits.Set(0) @@ -114,8 +135,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { pub := &dnsEntryMap{ IPs: map[string][]net.IP{"tailscale.com": {net.IPv4(10, 10, 10, 10)}}, } - dnsCache.Store(pub) - dnsCacheBytes.Store([]byte(`{"tailscale.com":["10.10.10.10"]}`)) + setDNSCache(t, pub) unpublishedDNSCache.Store(&dnsEntryMap{ IPs: map[string][]net.IP{ @@ -131,7 +151,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { t.Run("CacheMiss", func(t *testing.T) { // One domain in map but empty, one not in map at all for _, q := range []string{"log.tailscale.com", "login.tailscale.com"} { - resetMetrics() + resetMetrics(t) ips := getBootstrapDNS(t, q) // Expected our public map to be returned on a cache miss @@ -149,7 +169,7 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { // Verify that we do get a valid response and metric. t.Run("CacheHit", func(t *testing.T) { - resetMetrics() + resetMetrics(t) ips := getBootstrapDNS(t, "controlplane.tailscale.com") want := map[string][]net.IP{"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)}} if !reflect.DeepEqual(ips, want) { @@ -166,8 +186,10 @@ func TestUnpublishedDNSEmptyList(t *testing.T) { } func TestLookupMetric(t *testing.T) { + setDNSCache(t, &dnsEntryMap{}) + d := []string{"a.io", "b.io", "c.io", "d.io", "e.io", "e.io", "e.io", "a.io"} - resetMetrics() + resetMetrics(t) for _, q := range d { _ = getBootstrapDNS(t, q) } diff --git a/cmd/derper/cert.go b/cmd/derper/cert.go index b95755c64d2a7..979c0d671517f 100644 --- a/cmd/derper/cert.go +++ b/cmd/derper/cert.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -11,6 +11,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/base64" "encoding/json" "encoding/pem" "errors" @@ -24,6 +25,7 @@ import ( "regexp" "time" + "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" "tailscale.com/tailcfg" ) @@ -42,19 +44,42 @@ type certProvider interface { HTTPHandler(fallback http.Handler) http.Handler } -func certProviderByCertMode(mode, dir, hostname string) (certProvider, error) { +func certProviderByCertMode(mode, dir, hostname, eabKID, eabKey, email string) (certProvider, error) { if dir == "" { return nil, errors.New("missing required --certdir flag") } switch mode { - case "letsencrypt": + case "letsencrypt", "gcp": certManager := &autocert.Manager{ Prompt: autocert.AcceptTOS, HostPolicy: autocert.HostWhitelist(hostname), Cache: autocert.DirCache(dir), } + if mode == "gcp" { + if eabKID == "" || eabKey == "" { + return nil, errors.New("--certmode=gcp requires --acme-eab-kid and --acme-eab-key flags") + } + if email == "" { + return nil, errors.New("--certmode=gcp requires --acme-email flag") + } + keyBytes, err := decodeEABKey(eabKey) + if err != nil { + return nil, err + } + certManager.Client = &acme.Client{ + DirectoryURL: "https://dv.acme-v02.api.pki.goog/directory", + } + certManager.ExternalAccountBinding = &acme.ExternalAccountBinding{ + KID: eabKID, + Key: keyBytes, + } + } if hostname == "derp.tailscale.com" { certManager.HostPolicy = prodAutocertHostPolicy + } + if email != "" { + certManager.Email = email + } else if hostname == "derp.tailscale.com" { certManager.Email = "security@tailscale.com" } return certManager, nil @@ -209,3 +234,17 @@ func createSelfSignedIPCert(crtPath, keyPath, ipStr string) (*tls.Certificate, e } return &tlsCert, nil } + +// decodeEABKey decodes a base64-encoded EAB key. +// It accepts both standard base64 (with padding) and base64url (without padding). +func decodeEABKey(s string) ([]byte, error) { + // Try base64url first (no padding), then standard base64 (with padding). + // This handles both ACME spec format and gcloud output format. + if b, err := base64.RawURLEncoding.DecodeString(s); err == nil { + return b, nil + } + if b, err := base64.StdEncoding.DecodeString(s); err == nil { + return b, nil + } + return nil, errors.New("invalid base64 encoding for EAB key") +} diff --git a/cmd/derper/cert_test.go b/cmd/derper/cert_test.go index 31fd4ea446949..e111ed76b7a97 100644 --- a/cmd/derper/cert_test.go +++ b/cmd/derper/cert_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -22,8 +22,8 @@ import ( "testing" "time" - "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/derp/derpserver" "tailscale.com/net/netmon" "tailscale.com/tailcfg" "tailscale.com/types/key" @@ -91,7 +91,7 @@ func TestCertIP(t *testing.T) { t.Fatalf("Error closing key.pem: %v", err) } - cp, err := certProviderByCertMode("manual", dir, hostname) + cp, err := certProviderByCertMode("manual", dir, hostname, "", "", "") if err != nil { t.Fatal(err) } @@ -131,9 +131,9 @@ func TestPinnedCertRawIP(t *testing.T) { } defer ln.Close() - ds := derp.NewServer(key.NewNode(), t.Logf) + ds := derpserver.New(key.NewNode(), t.Logf) - derpHandler := derphttp.Handler(ds) + derpHandler := derpserver.Handler(ds) mux := http.NewServeMux() mux.Handle("/derp", derpHandler) @@ -169,3 +169,43 @@ func TestPinnedCertRawIP(t *testing.T) { } defer connClose.Close() } + +func TestGCPCertMode(t *testing.T) { + dir := t.TempDir() + + // Missing EAB credentials + _, err := certProviderByCertMode("gcp", dir, "test.example.com", "", "", "test@example.com") + if err == nil { + t.Fatal("expected error when EAB credentials are missing") + } + + // Missing email + _, err = certProviderByCertMode("gcp", dir, "test.example.com", "kid", "dGVzdC1rZXk", "") + if err == nil { + t.Fatal("expected error when email is missing") + } + + // Invalid base64 + _, err = certProviderByCertMode("gcp", dir, "test.example.com", "kid", "not-valid!", "test@example.com") + if err == nil { + t.Fatal("expected error for invalid base64") + } + + // Valid base64url (no padding) + cp, err := certProviderByCertMode("gcp", dir, "test.example.com", "kid", "dGVzdC1rZXk", "test@example.com") + if err != nil { + t.Fatalf("base64url: %v", err) + } + if cp == nil { + t.Fatal("base64url: nil certProvider") + } + + // Valid standard base64 (with padding, gcloud format) + cp, err = certProviderByCertMode("gcp", dir, "test.example.com", "kid", "dGVzdC1rZXk=", "test@example.com") + if err != nil { + t.Fatalf("base64: %v", err) + } + if cp == nil { + t.Fatal("base64: nil certProvider") + } +} diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 7adbf397f2f4f..22956a2997710 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -1,18 +1,17 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depaware) + đŸ’Ŗ crypto/internal/entropy/v1.0.0 from crypto/internal/fips140/drbg filippo.io/edwards25519 from github.com/hdevalence/ed25519consensus filippo.io/edwards25519/field from filippo.io/edwards25519 - W đŸ’Ŗ github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+ - W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate - W đŸ’Ŗ github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy + github.com/axiomhq/hyperloglog from tailscale.com/derp/derpserver github.com/beorn7/perks/quantile from github.com/prometheus/client_golang/prometheus đŸ’Ŗ github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus - github.com/coder/websocket from tailscale.com/cmd/derper+ + github.com/coder/websocket from tailscale.com/derp/derpserver+ github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket - L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw - W đŸ’Ŗ github.com/dblohm7/wingoes from tailscale.com/util/winutil+ + github.com/creachadair/msync/throttle from github.com/tailscale/setec/client/setec + W đŸ’Ŗ github.com/dblohm7/wingoes from tailscale.com/util/winutil + github.com/dgryski/go-metro from github.com/axiomhq/hyperloglog github.com/fxamacker/cbor/v2 from tailscale.com/tka github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ @@ -20,19 +19,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/v1 from tailscale.com/net/routecheck + đŸ’Ŗ github.com/go4org/hashtriemap from tailscale.com/derp/derpserver github.com/golang/groupcache/lru from tailscale.com/net/dnscache - L github.com/google/nftables from tailscale.com/util/linuxfw - L đŸ’Ŗ github.com/google/nftables/alignedbuff from github.com/google/nftables/xt - L đŸ’Ŗ github.com/google/nftables/binaryutil from github.com/google/nftables+ - L github.com/google/nftables/expr from github.com/google/nftables+ - L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ - L github.com/google/nftables/xt from github.com/google/nftables/expr+ github.com/hdevalence/ed25519consensus from tailscale.com/tka L đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink - L đŸ’Ŗ github.com/mdlayher/netlink from github.com/google/nftables+ + L đŸ’Ŗ github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+ L đŸ’Ŗ github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ - L github.com/mdlayher/netlink/nltest from github.com/google/nftables L đŸ’Ŗ github.com/mdlayher/socket from github.com/mdlayher/netlink đŸ’Ŗ github.com/mitchellh/go-ps from tailscale.com/safesocket github.com/munnerz/goautoneg from github.com/prometheus/common/expfmt @@ -41,22 +35,19 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+ - LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus - LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs - LD github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs + L github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus + L github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs + L github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs W đŸ’Ŗ github.com/tailscale/go-winio from tailscale.com/safesocket W đŸ’Ŗ github.com/tailscale/go-winio/internal/fs from github.com/tailscale/go-winio W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - L đŸ’Ŗ github.com/tailscale/netlink from tailscale.com/util/linuxfw - L đŸ’Ŗ github.com/tailscale/netlink/nl from github.com/tailscale/netlink github.com/tailscale/setec/client/setec from tailscale.com/cmd/derper github.com/tailscale/setec/types/api from github.com/tailscale/setec/client/setec - L github.com/vishvananda/netns from github.com/tailscale/netlink+ github.com/x448/float16 from github.com/fxamacker/cbor/v2 đŸ’Ŗ go4.org/mem from tailscale.com/client/local+ - go4.org/netipx from tailscale.com/net/tsaddr + go4.org/netipx from tailscale.com/net/tsaddr+ W đŸ’Ŗ golang.zx2c4.com/wireguard/windows/tunnel/winipcfg from tailscale.com/net/netmon+ google.golang.org/protobuf/encoding/protodelim from github.com/prometheus/common/expfmt google.golang.org/protobuf/encoding/prototext from github.com/prometheus/common/expfmt+ @@ -77,6 +68,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa đŸ’Ŗ google.golang.org/protobuf/internal/impl from google.golang.org/protobuf/internal/filetype+ google.golang.org/protobuf/internal/order from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/pragma from google.golang.org/protobuf/encoding/prototext+ + đŸ’Ŗ google.golang.org/protobuf/internal/protolazy from google.golang.org/protobuf/internal/impl+ google.golang.org/protobuf/internal/set from google.golang.org/protobuf/encoding/prototext đŸ’Ŗ google.golang.org/protobuf/internal/strs from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/version from google.golang.org/protobuf/runtime/protoimpl @@ -85,19 +77,20 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa google.golang.org/protobuf/reflect/protoregistry from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/runtime/protoiface from google.golang.org/protobuf/internal/impl+ google.golang.org/protobuf/runtime/protoimpl from github.com/prometheus/client_model/go+ - google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ + đŸ’Ŗ google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ tailscale.com from tailscale.com/version đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/cmd/derper+ - tailscale.com/client/local from tailscale.com/client/tailscale+ - tailscale.com/client/tailscale from tailscale.com/derp - tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ + tailscale.com/client/local from tailscale.com/derp/derpserver + tailscale.com/client/tailscale/apitype from tailscale.com/client/local tailscale.com/derp from tailscale.com/cmd/derper+ - tailscale.com/derp/derpconst from tailscale.com/derp+ + tailscale.com/derp/derpconst from tailscale.com/derp/derphttp+ tailscale.com/derp/derphttp from tailscale.com/cmd/derper - tailscale.com/disco from tailscale.com/derp + tailscale.com/derp/derpserver from tailscale.com/cmd/derper + tailscale.com/disco from tailscale.com/derp/derpserver tailscale.com/drive from tailscale.com/client/local+ tailscale.com/envknob from tailscale.com/client/local+ - tailscale.com/feature from tailscale.com/tsweb + tailscale.com/feature from tailscale.com/tsweb+ + tailscale.com/feature/buildfeatures from tailscale.com/feature+ tailscale.com/health from tailscale.com/net/tlsdial+ tailscale.com/hostinfo from tailscale.com/net/netmon+ tailscale.com/ipn from tailscale.com/client/local @@ -105,6 +98,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa tailscale.com/kube/kubetypes from tailscale.com/envknob tailscale.com/metrics from tailscale.com/cmd/derper+ tailscale.com/net/bakedroots from tailscale.com/net/tlsdial + tailscale.com/net/connectproxy from tailscale.com/cmd/derper tailscale.com/net/dnscache from tailscale.com/derp/derphttp tailscale.com/net/ktimeout from tailscale.com/cmd/derper tailscale.com/net/netaddr from tailscale.com/ipn+ @@ -113,108 +107,96 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa đŸ’Ŗ tailscale.com/net/netns from tailscale.com/derp/derphttp tailscale.com/net/netutil from tailscale.com/client/local tailscale.com/net/netx from tailscale.com/net/dnscache+ + tailscale.com/net/routecheck from tailscale.com/client/local tailscale.com/net/sockstats from tailscale.com/derp/derphttp tailscale.com/net/stun from tailscale.com/net/stunserver tailscale.com/net/stunserver from tailscale.com/cmd/derper - L tailscale.com/net/tcpinfo from tailscale.com/derp + L tailscale.com/net/tcpinfo from tailscale.com/derp/derpserver tailscale.com/net/tlsdial from tailscale.com/derp/derphttp tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/net/routecheck tailscale.com/net/tsaddr from tailscale.com/ipn+ - đŸ’Ŗ tailscale.com/net/tshttpproxy from tailscale.com/derp/derphttp+ - tailscale.com/net/wsconn from tailscale.com/cmd/derper + tailscale.com/net/udprelay/status from tailscale.com/client/local + tailscale.com/net/wsconn from tailscale.com/derp/derpserver tailscale.com/paths from tailscale.com/client/local đŸ’Ŗ tailscale.com/safesocket from tailscale.com/client/local tailscale.com/syncs from tailscale.com/cmd/derper+ tailscale.com/tailcfg from tailscale.com/client/local+ tailscale.com/tka from tailscale.com/client/local+ - W tailscale.com/tsconst from tailscale.com/net/netmon+ + tailscale.com/tsconst from tailscale.com/net/netmon+ tailscale.com/tstime from tailscale.com/derp+ tailscale.com/tstime/mono from tailscale.com/tstime/rate - tailscale.com/tstime/rate from tailscale.com/derp + tailscale.com/tstime/rate from tailscale.com/derp/derpserver tailscale.com/tsweb from tailscale.com/cmd/derper+ tailscale.com/tsweb/promvarz from tailscale.com/cmd/derper tailscale.com/tsweb/varz from tailscale.com/tsweb+ + tailscale.com/types/appctype from tailscale.com/client/local tailscale.com/types/dnstype from tailscale.com/tailcfg+ tailscale.com/types/empty from tailscale.com/ipn tailscale.com/types/ipproto from tailscale.com/tailcfg+ tailscale.com/types/key from tailscale.com/client/local+ tailscale.com/types/lazy from tailscale.com/version+ tailscale.com/types/logger from tailscale.com/cmd/derper+ - tailscale.com/types/netmap from tailscale.com/ipn - tailscale.com/types/opt from tailscale.com/client/tailscale+ - tailscale.com/types/persist from tailscale.com/ipn + tailscale.com/types/netmap from tailscale.com/ipn+ + tailscale.com/types/opt from tailscale.com/envknob+ + tailscale.com/types/persist from tailscale.com/ipn+ tailscale.com/types/preftype from tailscale.com/ipn - tailscale.com/types/ptr from tailscale.com/hostinfo+ tailscale.com/types/result from tailscale.com/util/lineiter tailscale.com/types/structs from tailscale.com/ipn+ tailscale.com/types/tkatype from tailscale.com/client/local+ tailscale.com/types/views from tailscale.com/ipn+ - tailscale.com/util/cibuild from tailscale.com/health + tailscale.com/util/bufiox from tailscale.com/derp/derpserver+ + tailscale.com/util/cibuild from tailscale.com/health+ tailscale.com/util/clientmetric from tailscale.com/net/netmon+ tailscale.com/util/cloudenv from tailscale.com/hostinfo+ - W tailscale.com/util/cmpver from tailscale.com/net/tshttpproxy tailscale.com/util/ctxkey from tailscale.com/tsweb+ đŸ’Ŗ tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting L đŸ’Ŗ tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/hostinfo+ tailscale.com/util/eventbus from tailscale.com/net/netmon+ đŸ’Ŗ tailscale.com/util/hashx from tailscale.com/util/deephash - tailscale.com/util/httpm from tailscale.com/client/tailscale tailscale.com/util/lineiter from tailscale.com/hostinfo+ - L tailscale.com/util/linuxfw from tailscale.com/net/netns tailscale.com/util/mak from tailscale.com/health+ - tailscale.com/util/multierr from tailscale.com/health+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto tailscale.com/util/rands from tailscale.com/tsweb - tailscale.com/util/set from tailscale.com/derp+ + tailscale.com/util/set from tailscale.com/derp/derpserver+ tailscale.com/util/singleflight from tailscale.com/net/dnscache tailscale.com/util/slicesx from tailscale.com/cmd/derper+ - tailscale.com/util/syspolicy from tailscale.com/ipn - tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy/setting+ - tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy/internal/metrics+ - tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source - tailscale.com/util/syspolicy/rsop from tailscale.com/util/syspolicy - tailscale.com/util/syspolicy/setting from tailscale.com/util/syspolicy+ - tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy+ - tailscale.com/util/testenv from tailscale.com/util/syspolicy+ + tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy/setting + tailscale.com/util/syspolicy/pkey from tailscale.com/ipn+ + tailscale.com/util/syspolicy/policyclient from tailscale.com/ipn + tailscale.com/util/syspolicy/ptype from tailscale.com/util/syspolicy/policyclient+ + tailscale.com/util/syspolicy/setting from tailscale.com/client/local + tailscale.com/util/testenv from tailscale.com/net/bakedroots+ tailscale.com/util/usermetric from tailscale.com/health tailscale.com/util/vizerror from tailscale.com/tailcfg+ W đŸ’Ŗ tailscale.com/util/winutil from tailscale.com/hostinfo+ - W đŸ’Ŗ tailscale.com/util/winutil/gp from tailscale.com/util/syspolicy/source W đŸ’Ŗ tailscale.com/util/winutil/winenv from tailscale.com/hostinfo+ - tailscale.com/version from tailscale.com/derp+ + tailscale.com/version from tailscale.com/cmd/derper+ tailscale.com/version/distro from tailscale.com/envknob+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap - golang.org/x/crypto/acme from golang.org/x/crypto/acme/autocert + golang.org/x/crypto/acme from golang.org/x/crypto/acme/autocert+ golang.org/x/crypto/acme/autocert from tailscale.com/cmd/derper golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from tailscale.com/tka - golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305 - golang.org/x/crypto/chacha20poly1305 from crypto/tls+ - golang.org/x/crypto/cryptobyte from crypto/ecdsa+ - golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ golang.org/x/crypto/curve25519 from golang.org/x/crypto/nacl/box+ - golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ - golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/internal/alias from golang.org/x/crypto/nacl/secretbox + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/nacl/secretbox golang.org/x/crypto/nacl/box from tailscale.com/types/key golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ golang.org/x/exp/constraints from tailscale.com/util/winutil+ - golang.org/x/exp/maps from tailscale.com/util/syspolicy/setting+ + golang.org/x/exp/maps from tailscale.com/util/syspolicy/setting L golang.org/x/net/bpf from github.com/mdlayher/netlink+ - golang.org/x/net/dns/dnsmessage from net+ - golang.org/x/net/http/httpguts from net/http - golang.org/x/net/http/httpproxy from net/http+ - golang.org/x/net/http2/hpack from net/http - golang.org/x/net/idna from golang.org/x/crypto/acme/autocert+ + golang.org/x/net/dns/dnsmessage from tailscale.com/net/dnscache + golang.org/x/net/idna from golang.org/x/crypto/acme/autocert golang.org/x/net/internal/socks from golang.org/x/net/proxy golang.org/x/net/proxy from tailscale.com/net/netns - D golang.org/x/net/route from net+ + D golang.org/x/net/route from tailscale.com/net/netmon+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ - golang.org/x/sync/singleflight from github.com/tailscale/setec/client/setec golang.org/x/sys/cpu from golang.org/x/crypto/argon2+ - LD golang.org/x/sys/unix from github.com/google/nftables+ + LD golang.org/x/sys/unix from github.com/jsimonetti/rtnetlink/internal/unix+ W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/svc from golang.org/x/sys/windows/svc/mgr+ @@ -224,6 +206,22 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ golang.org/x/text/unicode/norm from golang.org/x/net/idna golang.org/x/time/rate from tailscale.com/cmd/derper+ + vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ + vendor/golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + vendor/golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + vendor/golang.org/x/crypto/internal/alias from vendor/golang.org/x/crypto/chacha20+ + vendor/golang.org/x/crypto/internal/poly1305 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/net/dns/dnsmessage from net + vendor/golang.org/x/net/http/httpguts from net/http+ + vendor/golang.org/x/net/http/httpproxy from net/http + vendor/golang.org/x/net/http2/hpack from net/http+ + vendor/golang.org/x/net/idna from net/http+ + vendor/golang.org/x/sys/cpu from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/text/secure/bidirule from vendor/golang.org/x/net/idna + vendor/golang.org/x/text/transform from vendor/golang.org/x/text/secure/bidirule+ + vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ + vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna bufio from compress/flate+ bytes from bufio+ cmp from slices+ @@ -232,7 +230,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa container/list from crypto/tls+ context from crypto/tls+ crypto from crypto/ecdh+ - crypto/aes from crypto/internal/hpke+ + crypto/aes from crypto/tls+ crypto/cipher from crypto/aes+ crypto/des from crypto/tls+ crypto/dsa from crypto/x509 @@ -240,11 +238,14 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa crypto/ecdsa from crypto/tls+ crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ + crypto/fips140 from crypto/tls/internal/fips140tls+ + crypto/hkdf from crypto/hpke+ crypto/hmac from crypto/tls+ + crypto/hpke from crypto/tls crypto/internal/boring from crypto/aes+ crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring - crypto/internal/entropy from crypto/internal/fips140/drbg + crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ crypto/internal/fips140 from crypto/internal/fips140/aes+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ @@ -259,7 +260,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/mlkem crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -269,22 +270,25 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ crypto/internal/fips140/tls12 from crypto/tls crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140cache from crypto/ecdsa+ crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140deps/time from crypto/internal/entropy/v1.0.0 crypto/internal/fips140hash from crypto/ecdsa+ crypto/internal/fips140only from crypto/cipher+ - crypto/internal/hpke from crypto/tls crypto/internal/impl from crypto/internal/fips140/aes+ - crypto/internal/randutil from crypto/dsa+ - crypto/internal/sysrand from crypto/internal/entropy+ + crypto/internal/rand from crypto/dsa+ + crypto/internal/randutil from crypto/internal/rand + crypto/internal/sysrand from crypto/internal/fips140/drbg crypto/md5 from crypto/tls+ + crypto/mlkem from crypto/hpke+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls crypto/rsa from crypto/tls+ crypto/sha1 from crypto/tls+ crypto/sha256 from crypto/tls+ - crypto/sha3 from crypto/internal/fips140hash + crypto/sha3 from crypto/internal/fips140hash+ crypto/sha512 from crypto/ecdsa+ crypto/subtle from crypto/cipher+ crypto/tls from golang.org/x/crypto/acme+ @@ -308,8 +312,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa go/token from google.golang.org/protobuf/internal/strs hash from crypto+ hash/crc32 from compress/gzip+ - hash/fnv from google.golang.org/protobuf/internal/detrand - hash/maphash from go4.org/mem + hash/fnv from google.golang.org/protobuf/internal/detrand+ + hash/maphash from go4.org/mem+ html from net/http/pprof+ html/template from tailscale.com/cmd/derper+ internal/abi from crypto/x509/internal/macos+ @@ -325,9 +329,8 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa internal/goarch from crypto/internal/fips140deps/cpu+ internal/godebug from crypto/internal/fips140deps/godebug+ internal/godebugs from internal/godebug+ - internal/goexperiment from hash/maphash+ + internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ - internal/itoa from internal/poll+ internal/msan from internal/runtime/maps+ internal/nettrace from net+ internal/oserror from io/fs+ @@ -336,25 +339,35 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa internal/profilerecord from runtime+ internal/race from internal/poll+ internal/reflectlite from context+ + D internal/routebsd from net internal/runtime/atomic from internal/runtime/exithook+ + L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime + internal/runtime/gc from runtime+ + internal/runtime/gc/scan from runtime internal/runtime/maps from reflect+ internal/runtime/math from internal/runtime/maps+ + internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ - L internal/runtime/syscall from runtime+ + L internal/runtime/syscall/linux from internal/runtime/cgroup+ + W internal/runtime/syscall/windows from internal/syscall/windows+ + internal/saferio from encoding/asn1 internal/singleflight from net + internal/strconv from internal/poll+ internal/stringslite from embed+ internal/sync from sync+ + internal/synctest from sync internal/syscall/execenv from os+ LD internal/syscall/unix from crypto/internal/sysrand+ W internal/syscall/windows from crypto/internal/sysrand+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os + internal/trace/tracev2 from runtime+ internal/unsafeheader from internal/reflectlite+ io from bufio+ io/fs from crypto/x509+ - L io/ioutil from github.com/mitchellh/go-ps+ + L io/ioutil from github.com/mitchellh/go-ps iter from maps+ log from expvar+ log/internal from log @@ -372,18 +385,19 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa net/http/httptrace from net/http+ net/http/internal from net/http net/http/internal/ascii from net/http + net/http/internal/httpcommon from net/http net/http/pprof from tailscale.com/tsweb net/netip from go4.org/netipx+ - net/textproto from golang.org/x/net/http/httpguts+ + net/textproto from github.com/coder/websocket+ net/url from crypto/x509+ os from crypto/internal/sysrand+ - os/exec from github.com/coreos/go-iptables/iptables+ + os/exec from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+ os/signal from tailscale.com/cmd/derper - W os/user from tailscale.com/util/winutil+ + W os/user from tailscale.com/util/winutil path from github.com/prometheus/client_golang/prometheus/internal+ path/filepath from crypto/x509+ - reflect from crypto/x509+ - regexp from github.com/coreos/go-iptables/iptables+ + reflect from encoding/asn1+ + regexp from github.com/prometheus/client_golang/prometheus/internal+ regexp/syntax from regexp runtime from crypto/internal/fips140+ runtime/debug from github.com/prometheus/client_golang/prometheus+ @@ -394,6 +408,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa sort from compress/flate+ strconv from compress/flate+ strings from bufio+ + W structs from internal/syscall/windows sync from compress/flate+ sync/atomic from context+ syscall from crypto/internal/sysrand+ @@ -406,4 +421,4 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa unicode/utf8 from bufio+ unique from net/netip unsafe from bytes+ - weak from unique + weak from unique+ diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 7ea404beb50af..0e3ab1bc21c0b 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The derper binary is a simple DERP server. @@ -40,8 +40,7 @@ import ( "github.com/tailscale/setec/client/setec" "golang.org/x/time/rate" "tailscale.com/atomicfile" - "tailscale.com/derp" - "tailscale.com/derp/derphttp" + "tailscale.com/derp/derpserver" "tailscale.com/metrics" "tailscale.com/net/ktimeout" "tailscale.com/net/stunserver" @@ -61,9 +60,12 @@ var ( httpPort = flag.Int("http-port", 80, "The port on which to serve HTTP. Set to -1 to disable. The listener is bound to the same IP (if any) as specified in the -a flag.") stunPort = flag.Int("stun-port", 3478, "The UDP port on which to serve STUN. The listener is bound to the same IP (if any) as specified in the -a flag.") configPath = flag.String("c", "", "config file path") - certMode = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt") - certDir = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443") - hostname = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks") + certMode = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt, gcp") + certDir = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store ACME (e.g. LetsEncrypt) certs, if addr's port is :443") + hostname = flag.String("hostname", "derp.tailscale.com", "TLS host name for certs, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks") + acmeEABKid = flag.String("acme-eab-kid", "", "ACME External Account Binding (EAB) Key ID (required for --certmode=gcp)") + acmeEABKey = flag.String("acme-eab-key", "", "ACME External Account Binding (EAB) HMAC key, base64-encoded (required for --certmode=gcp)") + acmeEmail = flag.String("acme-email", "", "ACME account contact email address (required for --certmode=gcp, optional for letsencrypt)") runSTUN = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.") runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.") flagHome = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to") @@ -85,27 +87,27 @@ var ( acceptConnLimit = flag.Float64("accept-connection-limit", math.Inf(+1), "rate limit for accepting new connection") acceptConnBurst = flag.Int("accept-connection-burst", math.MaxInt, "burst limit for accepting new connection") + rateConfigPath = flag.String("rate-config", "", "if non-empty, path to JSON rate limit config file. Rate limiting is experimental and subject to change. Configuration is reloaded on SIGHUP.") + // tcpKeepAlive is intentionally long, to reduce battery cost. There is an L7 keepalive on a higher frequency schedule. tcpKeepAlive = flag.Duration("tcp-keepalive-time", 10*time.Minute, "TCP keepalive time") // tcpUserTimeout is intentionally short, so that hung connections are cleaned up promptly. DERPs should be nearby users. tcpUserTimeout = flag.Duration("tcp-user-timeout", 15*time.Second, "TCP user timeout") // tcpWriteTimeout is the timeout for writing to client TCP connections. It does not apply to mesh connections. - tcpWriteTimeout = flag.Duration("tcp-write-timeout", derp.DefaultTCPWiteTimeout, "TCP write timeout; 0 results in no timeout being set on writes") + tcpWriteTimeout = flag.Duration("tcp-write-timeout", derpserver.DefaultTCPWiteTimeout, "TCP write timeout; 0 results in no timeout being set on writes") + + // ACE + flagACEEnabled = flag.Bool("ace", false, "whether to enable embedded ACE server [experimental + in-development as of 2025-09-12; not yet documented]") ) var ( - tlsRequestVersion = &metrics.LabelMap{Label: "version"} - tlsActiveVersion = &metrics.LabelMap{Label: "version"} + tlsRequestVersion = metrics.NewLabelMap("derper_tls_request_version", "version") + tlsActiveVersion = metrics.NewLabelMap("gauge_derper_tls_active_version", "version") ) const setecMeshKeyName = "meshkey" const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY" -func init() { - expvar.Publish("derper_tls_request_version", tlsRequestVersion) - expvar.Publish("gauge_derper_tls_active_version", tlsActiveVersion) -} - type config struct { PrivateKey key.NodePrivate } @@ -186,12 +188,18 @@ func main() { serveTLS := tsweb.IsProd443(*addr) || *certMode == "manual" - s := derp.NewServer(cfg.PrivateKey, log.Printf) + s := derpserver.New(cfg.PrivateKey, log.Printf) s.SetVerifyClient(*verifyClients) s.SetTailscaledSocketPath(*socket) s.SetVerifyClientURL(*verifyClientURL) s.SetVerifyClientURLFailOpen(*verifyFailOpen) s.SetTCPWriteTimeout(*tcpWriteTimeout) + if *rateConfigPath != "" { + if err := s.LoadAndApplyRateConfig(*rateConfigPath); err != nil { + log.Fatalf("derper: loading rate config: %v", err) + } + go watchRateConfig(ctx, s, *rateConfigPath) + } var meshKey string if *dev { @@ -244,7 +252,7 @@ func main() { if err := startMesh(s); err != nil { log.Fatalf("startMesh: %v", err) } - expvar.Publish("derp", s.ExpVar()) + expvar.Publish("derp", s.ExpVar(*rateConfigPath != "")) handleHome, ok := getHomeHandler(*flagHome) if !ok { @@ -253,8 +261,8 @@ func main() { mux := http.NewServeMux() if *runDERP { - derpHandler := derphttp.Handler(s) - derpHandler = addWebSocketSupport(s, derpHandler) + derpHandler := derpserver.Handler(s) + derpHandler = derpserver.AddWebSocketSupport(s, derpHandler) mux.Handle("/derp", derpHandler) } else { mux.Handle("/derp", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -264,8 +272,8 @@ func main() { // These two endpoints are the same. Different versions of the clients // have assumes different paths over time so we support both. - mux.HandleFunc("/derp/probe", derphttp.ProbeHandler) - mux.HandleFunc("/derp/latency-check", derphttp.ProbeHandler) + mux.HandleFunc("/derp/probe", derpserver.ProbeHandler) + mux.HandleFunc("/derp/latency-check", derpserver.ProbeHandler) go refreshBootstrapDNSLoop() mux.HandleFunc("/bootstrap-dns", tsweb.BrowserHeaderHandlerFunc(handleBootstrapDNS)) @@ -277,7 +285,7 @@ func main() { tsweb.AddBrowserHeaders(w) io.WriteString(w, "User-agent: *\nDisallow: /\n") })) - mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent)) + mux.Handle("/generate_204", http.HandlerFunc(derpserver.ServeNoContent)) debug := tsweb.Debugger(mux) debug.KV("TLS hostname", *hostname) debug.KV("Mesh key", s.HasMeshKey()) @@ -341,7 +349,7 @@ func main() { if serveTLS { log.Printf("derper: serving on %s with TLS", *addr) var certManager certProvider - certManager, err = certProviderByCertMode(*certMode, *certDir, *hostname) + certManager, err = certProviderByCertMode(*certMode, *certDir, *hostname, *acmeEABKid, *acmeEABKey, *acmeEmail) if err != nil { log.Fatalf("derper: can not start cert provider: %v", err) } @@ -373,6 +381,11 @@ func main() { tlsRequestVersion.Add(label, 1) tlsActiveVersion.Add(label, 1) defer tlsActiveVersion.Add(label, -1) + + if r.Method == "CONNECT" { + serveConnect(s, w, r) + return + } } mux.ServeHTTP(w, r) @@ -380,7 +393,7 @@ func main() { if *httpPort > -1 { go func() { port80mux := http.NewServeMux() - port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent) + port80mux.HandleFunc("/generate_204", derpserver.ServeNoContent) port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux})) port80srv := &http.Server{ Addr: net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)), @@ -421,6 +434,27 @@ func main() { } } +// watchRateConfig listens for SIGHUP signals and reloads the rate config +// file on each signal, applying it to the server. It returns when ctx is done. +func watchRateConfig(ctx context.Context, s *derpserver.Server, path string) { + sighup := make(chan os.Signal, 1) + signal.Notify(sighup, syscall.SIGHUP) + defer signal.Stop(sighup) + for { + select { + case <-ctx.Done(): + return + case <-sighup: + log.Printf("derper: received SIGHUP, reloading rate config from %s", path) + if err := s.LoadAndApplyRateConfig(path); err != nil { + log.Printf("derper: rate config reload failed: %v", err) + continue + } + log.Printf("derper: rate config reloaded successfully") + } + } +} + var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`) func prodAutocertHostPolicy(_ context.Context, host string) error { @@ -474,32 +508,32 @@ func newRateLimitedListener(ln net.Listener, limit rate.Limit, burst int) *rateL return &rateLimitedListener{Listener: ln, lim: rate.NewLimiter(limit, burst)} } -func (l *rateLimitedListener) ExpVar() expvar.Var { +func (ln *rateLimitedListener) ExpVar() expvar.Var { m := new(metrics.Set) - m.Set("counter_accepted_connections", &l.numAccepts) - m.Set("counter_rejected_connections", &l.numRejects) + m.Set("counter_accepted_connections", &ln.numAccepts) + m.Set("counter_rejected_connections", &ln.numRejects) return m } var errLimitedConn = errors.New("cannot accept connection; rate limited") -func (l *rateLimitedListener) Accept() (net.Conn, error) { +func (ln *rateLimitedListener) Accept() (net.Conn, error) { // Even under a rate limited situation, we accept the connection immediately // and close it, rather than being slow at accepting new connections. // This provides two benefits: 1) it signals to the client that something // is going on on the server, and 2) it prevents new connections from // piling up and occupying resources in the OS kernel. // The client will retry as needing (with backoffs in place). - cn, err := l.Listener.Accept() + cn, err := ln.Listener.Accept() if err != nil { return nil, err } - if !l.lim.Allow() { - l.numRejects.Add(1) + if !ln.lim.Allow() { + ln.numRejects.Add(1) cn.Close() return nil, errLimitedConn } - l.numAccepts.Add(1) + ln.numAccepts.Add(1) return cn, nil } diff --git a/cmd/derper/derper_test.go b/cmd/derper/derper_test.go index 6dce1fcdfebdd..fc1ebd6930dd6 100644 --- a/cmd/derper/derper_test.go +++ b/cmd/derper/derper_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -11,7 +11,7 @@ import ( "strings" "testing" - "tailscale.com/derp/derphttp" + "tailscale.com/derp/derpserver" "tailscale.com/tstest/deptest" ) @@ -46,30 +46,30 @@ func TestNoContent(t *testing.T) { want string }{ { - name: "no challenge", + name: "no-challenge", }, { - name: "valid challenge", + name: "valid-challenge", input: "input", want: "response input", }, { - name: "valid challenge hostname", + name: "valid-challenge-hostname", input: "ts_derp99b.tailscale.com", want: "response ts_derp99b.tailscale.com", }, { - name: "invalid challenge", + name: "invalid-challenge", input: "foo\x00bar", want: "", }, { - name: "whitespace invalid challenge", + name: "whitespace-invalid-challenge", input: "foo bar", want: "", }, { - name: "long challenge", + name: "long-challenge", input: strings.Repeat("x", 65), want: "", }, @@ -78,20 +78,20 @@ func TestNoContent(t *testing.T) { t.Run(tt.name, func(t *testing.T) { req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil) if tt.input != "" { - req.Header.Set(derphttp.NoContentChallengeHeader, tt.input) + req.Header.Set(derpserver.NoContentChallengeHeader, tt.input) } w := httptest.NewRecorder() - derphttp.ServeNoContent(w, req) + derpserver.ServeNoContent(w, req) resp := w.Result() if tt.want == "" { - if h, found := resp.Header[derphttp.NoContentResponseHeader]; found { + if h, found := resp.Header[derpserver.NoContentResponseHeader]; found { t.Errorf("got %+v; expected no response header", h) } return } - if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want { + if got := resp.Header.Get(derpserver.NoContentResponseHeader); got != tt.want { t.Errorf("got %q; want %q", got, tt.want) } }) diff --git a/cmd/derper/mesh.go b/cmd/derper/mesh.go index cbb2fa59ac030..c07cfe969d9e3 100644 --- a/cmd/derper/mesh.go +++ b/cmd/derper/mesh.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -13,18 +13,19 @@ import ( "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/derp/derpserver" "tailscale.com/net/netmon" "tailscale.com/types/logger" ) -func startMesh(s *derp.Server) error { +func startMesh(s *derpserver.Server) error { if *meshWith == "" { return nil } if !s.HasMeshKey() { return errors.New("--mesh-with requires --mesh-psk-file") } - for _, hostTuple := range strings.Split(*meshWith, ",") { + for hostTuple := range strings.SplitSeq(*meshWith, ",") { if err := startMeshWithHost(s, hostTuple); err != nil { return err } @@ -32,7 +33,7 @@ func startMesh(s *derp.Server) error { return nil } -func startMeshWithHost(s *derp.Server, hostTuple string) error { +func startMeshWithHost(s *derpserver.Server, hostTuple string) error { var host string var dialHost string hostParts := strings.Split(hostTuple, "/") diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 25159d649408e..549364e5e8f6a 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The derpprobe binary probes derpers. @@ -107,6 +107,7 @@ func main() { mux := http.NewServeMux() d := tsweb.Debugger(mux) d.Handle("probe-run", "Run a probe", tsweb.StdHandler(tsweb.ReturnHandlerFunc(p.RunHandler), tsweb.HandlerOptions{Logf: log.Printf})) + d.Handle("probe-all", "Run all configured probes", tsweb.StdHandler(tsweb.ReturnHandlerFunc(p.RunAllHandler), tsweb.HandlerOptions{Logf: log.Printf})) mux.Handle("/", tsweb.StdHandler(p.StatusHandler( prober.WithTitle("DERP Prober"), prober.WithPageLink("Prober metrics", "/debug/varz"), diff --git a/cmd/dist/dist.go b/cmd/dist/dist.go index 038ced708e0f0..88b9e6fba9133 100644 --- a/cmd/dist/dist.go +++ b/cmd/dist/dist.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The dist command builds Tailscale release packages for distribution. @@ -21,12 +21,13 @@ import ( ) var ( - synologyPackageCenter bool - gcloudCredentialsBase64 string - gcloudProject string - gcloudKeyring string - qnapKeyName string - qnapCertificateBase64 string + synologyPackageCenter bool + gcloudCredentialsBase64 string + gcloudProject string + gcloudKeyring string + qnapKeyName string + qnapCertificateBase64 string + qnapCertificateIntermediariesBase64 string ) func getTargets() ([]dist.Target, error) { @@ -47,11 +48,11 @@ func getTargets() ([]dist.Target, error) { // To build for package center, run // ./tool/go run ./cmd/dist build --synology-package-center synology ret = append(ret, synology.Targets(synologyPackageCenter, nil)...) - qnapSigningArgs := []string{gcloudCredentialsBase64, gcloudProject, gcloudKeyring, qnapKeyName, qnapCertificateBase64} + qnapSigningArgs := []string{gcloudCredentialsBase64, gcloudProject, gcloudKeyring, qnapKeyName, qnapCertificateBase64, qnapCertificateIntermediariesBase64} if cmp.Or(qnapSigningArgs...) != "" && slices.Contains(qnapSigningArgs, "") { - return nil, errors.New("all of --gcloud-credentials, --gcloud-project, --gcloud-keyring, --qnap-key-name and --qnap-certificate must be set") + return nil, errors.New("all of --gcloud-credentials, --gcloud-project, --gcloud-keyring, --qnap-key-name, --qnap-certificate and --qnap-certificate-intermediaries must be set") } - ret = append(ret, qnap.Targets(gcloudCredentialsBase64, gcloudProject, gcloudKeyring, qnapKeyName, qnapCertificateBase64)...) + ret = append(ret, qnap.Targets(gcloudCredentialsBase64, gcloudProject, gcloudKeyring, qnapKeyName, qnapCertificateBase64, qnapCertificateIntermediariesBase64)...) return ret, nil } @@ -65,6 +66,7 @@ func main() { subcmd.FlagSet.StringVar(&gcloudKeyring, "gcloud-keyring", "", "path to keyring in GCP KMS (used when signing QNAP builds)") subcmd.FlagSet.StringVar(&qnapKeyName, "qnap-key-name", "", "name of GCP key to use when signing QNAP builds") subcmd.FlagSet.StringVar(&qnapCertificateBase64, "qnap-certificate", "", "base64 encoded certificate to use when signing QNAP builds") + subcmd.FlagSet.StringVar(&qnapCertificateIntermediariesBase64, "qnap-certificate-intermediaries", "", "base64 encoded intermediary certificate to use when signing QNAP builds") } } diff --git a/cmd/distsign/distsign.go b/cmd/distsign/distsign.go new file mode 100644 index 0000000000000..e0dba27206be9 --- /dev/null +++ b/cmd/distsign/distsign.go @@ -0,0 +1,42 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Command distsign tests downloads and signature validating for packages +// published by Tailscale on pkgs.tailscale.com. +package main + +import ( + "context" + "flag" + "log" + "os" + "path/filepath" + + "tailscale.com/clientupdate/distsign" +) + +var ( + pkgsURL = flag.String("pkgs-url", "https://pkgs.tailscale.com/", "URL of the packages server") + pkgName = flag.String("pkg-name", "", "name of the package on the packages server, including the stable/unstable track prefix") +) + +func main() { + flag.Parse() + + if *pkgName == "" { + log.Fatalf("--pkg-name is required") + } + + c, err := distsign.NewClient(log.Printf, *pkgsURL) + if err != nil { + log.Fatal(err) + } + tempDir := filepath.Join(os.TempDir(), "distsign") + if err := os.MkdirAll(tempDir, 0755); err != nil { + log.Fatal(err) + } + if err := c.Download(context.Background(), *pkgName, filepath.Join(os.TempDir(), "distsign", filepath.Base(*pkgName))); err != nil { + log.Fatal(err) + } + log.Printf("%q ok", *pkgName) +} diff --git a/cmd/featuretags/featuretags.go b/cmd/featuretags/featuretags.go new file mode 100644 index 0000000000000..f3aae68cc8b17 --- /dev/null +++ b/cmd/featuretags/featuretags.go @@ -0,0 +1,86 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// The featuretags command helps other build tools select Tailscale's Go build +// tags to use. +package main + +import ( + "flag" + "fmt" + "log" + "maps" + "slices" + "strings" + + "tailscale.com/feature/featuretags" + "tailscale.com/util/set" +) + +var ( + min = flag.Bool("min", false, "remove all features not mentioned in --add") + remove = flag.String("remove", "", "a comma-separated list of features to remove from the build. (without the 'ts_omit_' prefix)") + add = flag.String("add", "", "a comma-separated list of features or tags to add, if --min is used.") + list = flag.Bool("list", false, "if true, list all known features and what they do") +) + +func main() { + flag.Parse() + + features := featuretags.Features + + if *list { + for _, f := range slices.Sorted(maps.Keys(features)) { + fmt.Printf("%20s: %s\n", f, features[f].Desc) + } + return + } + + var keep = map[featuretags.FeatureTag]bool{} + for t := range strings.SplitSeq(*add, ",") { + if t != "" { + for ft := range featuretags.Requires(featuretags.FeatureTag(t)) { + keep[ft] = true + } + } + } + var tags []string + if keep[featuretags.CLI] { + tags = append(tags, "ts_include_cli") + } + if *min { + for _, f := range slices.Sorted(maps.Keys(features)) { + if f == "" { + continue + } + if !keep[f] && f.IsOmittable() { + tags = append(tags, f.OmitTag()) + } + } + } + removeSet := set.Set[featuretags.FeatureTag]{} + for v := range strings.SplitSeq(*remove, ",") { + if v == "" { + continue + } + f := featuretags.FeatureTag(v) + if _, ok := features[f]; !ok { + log.Fatalf("unknown feature %q in --remove", f) + } + removeSet.Add(f) + } + for ft := range removeSet { + set := featuretags.RequiredBy(ft) + for dependent := range set { + if !removeSet.Contains(dependent) { + log.Fatalf("cannot remove %q without also removing %q, which depends on it", ft, dependent) + } + } + tags = append(tags, ft.OmitTag()) + } + slices.Sort(tags) + tags = slices.Compact(tags) + if len(tags) != 0 { + fmt.Println(strings.Join(tags, ",")) + } +} diff --git a/cmd/get-authkey/main.go b/cmd/get-authkey/main.go index ec7ab5d2c6158..da98decda6ae5 100644 --- a/cmd/get-authkey/main.go +++ b/cmd/get-authkey/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // get-authkey allocates an authkey using an OAuth API client diff --git a/cmd/gitops-pusher/cache.go b/cmd/gitops-pusher/cache.go index 6792e5e63e9cc..af5c4606c0d50 100644 --- a/cmd/gitops-pusher/cache.go +++ b/cmd/gitops-pusher/cache.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/gitops-pusher/gitops-pusher.go b/cmd/gitops-pusher/gitops-pusher.go index 690ca287056d3..9ea115a1585e7 100644 --- a/cmd/gitops-pusher/gitops-pusher.go +++ b/cmd/gitops-pusher/gitops-pusher.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Command gitops-pusher allows users to use a GitOps flow for managing Tailscale ACLs. @@ -19,12 +19,15 @@ import ( "os" "regexp" "strings" + "sync" "time" "github.com/peterbourgon/ff/v3/ffcli" "github.com/tailscale/hujson" "golang.org/x/oauth2/clientcredentials" - "tailscale.com/client/tailscale" + tsclient "tailscale.com/client/tailscale" + _ "tailscale.com/feature/identityfederation" + "tailscale.com/internal/client/tailscale" "tailscale.com/util/httpm" ) @@ -38,6 +41,12 @@ var ( failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed") ) +var ( + getCredentialsOnce sync.Once + client *http.Client + apiKey string +) + func modifiedExternallyError() error { if *githubSyntax { return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname) @@ -46,9 +55,9 @@ func modifiedExternallyError() error { } } -func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(context.Context, []string) error { +func apply(cache *Cache, tailnet string) func(context.Context, []string) error { return func(ctx context.Context, args []string) error { - controlEtag, err := getACLETag(ctx, client, tailnet, apiKey) + controlEtag, err := getACLETag(ctx, tailnet) if err != nil { return err } @@ -83,7 +92,7 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte } } - if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil { + if err := applyNewACL(ctx, tailnet, *policyFname, controlEtag); err != nil { return err } @@ -93,9 +102,9 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte } } -func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(context.Context, []string) error { +func test(cache *Cache, tailnet string) func(context.Context, []string) error { return func(ctx context.Context, args []string) error { - controlEtag, err := getACLETag(ctx, client, tailnet, apiKey) + controlEtag, err := getACLETag(ctx, tailnet) if err != nil { return err } @@ -129,16 +138,16 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex } } - if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil { + if err := testNewACLs(ctx, tailnet, *policyFname); err != nil { return err } return nil } } -func getChecksums(cache *Cache, client *http.Client, tailnet, apiKey string) func(context.Context, []string) error { +func getChecksums(cache *Cache, tailnet string) func(context.Context, []string) error { return func(ctx context.Context, args []string) error { - controlEtag, err := getACLETag(ctx, client, tailnet, apiKey) + controlEtag, err := getACLETag(ctx, tailnet) if err != nil { return err } @@ -166,28 +175,7 @@ func main() { if !ok { log.Fatal("set envvar TS_TAILNET to your tailnet's name") } - apiKey, ok := os.LookupEnv("TS_API_KEY") - oauthId, oiok := os.LookupEnv("TS_OAUTH_ID") - oauthSecret, osok := os.LookupEnv("TS_OAUTH_SECRET") - if !ok && (!oiok || !osok) { - log.Fatal("set envvar TS_API_KEY to your Tailscale API key or TS_OAUTH_ID and TS_OAUTH_SECRET to your Tailscale OAuth ID and Secret") - } - if apiKey != "" && (oauthId != "" || oauthSecret != "") { - log.Fatal("set either the envvar TS_API_KEY or TS_OAUTH_ID and TS_OAUTH_SECRET") - } - var client *http.Client - if oiok && (oauthId != "" || oauthSecret != "") { - // Both should ideally be set, but if either are non-empty it means the user had an intent - // to set _something_, so they should receive the oauth error flow. - oauthConfig := &clientcredentials.Config{ - ClientID: oauthId, - ClientSecret: oauthSecret, - TokenURL: fmt.Sprintf("https://%s/api/v2/oauth/token", *apiServer), - } - client = oauthConfig.Client(context.Background()) - } else { - client = http.DefaultClient - } + cache, err := LoadCache(*cacheFname) if err != nil { if os.IsNotExist(err) { @@ -203,7 +191,7 @@ func main() { ShortUsage: "gitops-pusher [options] apply", ShortHelp: "Pushes changes to CONTROL", LongHelp: `Pushes changes to CONTROL`, - Exec: apply(cache, client, tailnet, apiKey), + Exec: apply(cache, tailnet), } testCmd := &ffcli.Command{ @@ -211,7 +199,7 @@ func main() { ShortUsage: "gitops-pusher [options] test", ShortHelp: "Tests ACL changes", LongHelp: "Tests ACL changes", - Exec: test(cache, client, tailnet, apiKey), + Exec: test(cache, tailnet), } cksumCmd := &ffcli.Command{ @@ -219,7 +207,7 @@ func main() { ShortUsage: "Shows checksums of ACL files", ShortHelp: "Fetch checksum of CONTROL's ACL and the local ACL for comparison", LongHelp: "Fetch checksum of CONTROL's ACL and the local ACL for comparison", - Exec: getChecksums(cache, client, tailnet, apiKey), + Exec: getChecksums(cache, tailnet), } root := &ffcli.Command{ @@ -242,6 +230,47 @@ func main() { } } +func getCredentials() (*http.Client, string) { + getCredentialsOnce.Do(func() { + apiKeyEnv, ok := os.LookupEnv("TS_API_KEY") + oauthId, oiok := os.LookupEnv("TS_OAUTH_ID") + oauthSecret, osok := os.LookupEnv("TS_OAUTH_SECRET") + idToken, idok := os.LookupEnv("TS_ID_TOKEN") + + if !ok && (!oiok || (!osok && !idok)) { + log.Fatal("set envvar TS_API_KEY to your Tailscale API key, TS_OAUTH_ID and TS_OAUTH_SECRET to a Tailscale OAuth ID and Secret, or TS_OAUTH_ID and TS_ID_TOKEN to a Tailscale federated identity Client ID and OIDC identity token") + } + if apiKeyEnv != "" && (oauthId != "" || (oauthSecret != "" && idToken != "")) { + log.Fatal("set either the envvar TS_API_KEY, TS_OAUTH_ID and TS_OAUTH_SECRET, or TS_OAUTH_ID and TS_ID_TOKEN") + } + if oiok && ((oauthId != "" && !idok) || oauthSecret != "") { + // Both should ideally be set, but if either are non-empty it means the user had an intent + // to set _something_, so they should receive the oauth error flow. + oauthConfig := &clientcredentials.Config{ + ClientID: oauthId, + ClientSecret: oauthSecret, + TokenURL: fmt.Sprintf("https://%s/api/v2/oauth/token", *apiServer), + } + client = oauthConfig.Client(context.Background()) + } else if idok && idToken != "" && oiok && oauthId != "" { + if exchangeJWTForToken, ok := tailscale.HookExchangeJWTForTokenViaWIF.GetOk(); ok { + var err error + apiKeyEnv, err = exchangeJWTForToken(context.Background(), fmt.Sprintf("https://%s", *apiServer), oauthId, idToken) + if err != nil { + log.Fatal(err) + } + } + client = http.DefaultClient + } else { + client = http.DefaultClient + } + + apiKey = apiKeyEnv + }) + + return client, apiKey +} + func sumFile(fname string) (string, error) { data, err := os.ReadFile(fname) if err != nil { @@ -262,7 +291,9 @@ func sumFile(fname string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -func applyNewACL(ctx context.Context, client *http.Client, tailnet, apiKey, policyFname, oldEtag string) error { +func applyNewACL(ctx context.Context, tailnet, policyFname, oldEtag string) error { + client, apiKey := getCredentials() + fin, err := os.Open(policyFname) if err != nil { return err @@ -299,7 +330,9 @@ func applyNewACL(ctx context.Context, client *http.Client, tailnet, apiKey, poli return nil } -func testNewACLs(ctx context.Context, client *http.Client, tailnet, apiKey, policyFname string) error { +func testNewACLs(ctx context.Context, tailnet, policyFname string) error { + client, apiKey := getCredentials() + data, err := os.ReadFile(policyFname) if err != nil { return err @@ -346,7 +379,7 @@ var lineColMessageSplit = regexp.MustCompile(`line ([0-9]+), column ([0-9]+): (. // ACLGitopsTestError is redefined here so we can add a custom .Error() response type ACLGitopsTestError struct { - tailscale.ACLTestError + tsclient.ACLTestError } func (ate ACLGitopsTestError) Error() string { @@ -388,7 +421,9 @@ func (ate ACLGitopsTestError) Error() string { return sb.String() } -func getACLETag(ctx context.Context, client *http.Client, tailnet, apiKey string) (string, error) { +func getACLETag(ctx context.Context, tailnet string) (string, error) { + client, apiKey := getCredentials() + req, err := http.NewRequestWithContext(ctx, httpm.GET, fmt.Sprintf("https://%s/api/v2/tailnet/%s/acl", *apiServer, tailnet), nil) if err != nil { return "", err diff --git a/cmd/gitops-pusher/gitops-pusher_test.go b/cmd/gitops-pusher/gitops-pusher_test.go index e08b06c9cd194..8d785e8cf793a 100644 --- a/cmd/gitops-pusher/gitops-pusher_test.go +++ b/cmd/gitops-pusher/gitops-pusher_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -30,7 +30,7 @@ func TestEmbeddedTypeUnmarshal(t *testing.T) { }, } - t.Run("unmarshal gitops type from acl type", func(t *testing.T) { + t.Run("unmarshal-gitops-from-acl", func(t *testing.T) { b, _ := json.Marshal(aclTestErr) var e ACLGitopsTestError err := json.Unmarshal(b, &e) @@ -41,7 +41,7 @@ func TestEmbeddedTypeUnmarshal(t *testing.T) { t.Fatalf("user heading for 'ACLError' not found in gitops error: %v", e.Error()) } }) - t.Run("unmarshal acl type from gitops type", func(t *testing.T) { + t.Run("unmarshal-acl-from-gitops", func(t *testing.T) { b, _ := json.Marshal(gitopsErr) var e tailscale.ACLTestError err := json.Unmarshal(b, &e) diff --git a/cmd/hello/hello.go b/cmd/hello/hello.go index fa116b28b15ab..45eb7751c3790 100644 --- a/cmd/hello/hello.go +++ b/cmd/hello/hello.go @@ -1,216 +1,20 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The hello binary runs hello.ts.net. package main // import "tailscale.com/cmd/hello" import ( - "context" - "crypto/tls" - _ "embed" - "encoding/json" - "errors" - "flag" - "html/template" "log" - "net/http" - "os" - "strings" - "time" - "tailscale.com/client/local" - "tailscale.com/client/tailscale/apitype" - "tailscale.com/tailcfg" + "tailscale.com/cmd/hello/helloserver" ) -var ( - httpAddr = flag.String("http", ":80", "address to run an HTTP server on, or empty for none") - httpsAddr = flag.String("https", ":443", "address to run an HTTPS server on, or empty for none") - testIP = flag.String("test-ip", "", "if non-empty, look up IP and exit before running a server") -) - -//go:embed hello.tmpl.html -var embeddedTemplate string - -var localClient local.Client - func main() { - flag.Parse() - if *testIP != "" { - res, err := localClient.WhoIs(context.Background(), *testIP) - if err != nil { - log.Fatal(err) - } - e := json.NewEncoder(os.Stdout) - e.SetIndent("", "\t") - e.Encode(res) - return + s := &helloserver.Server{ + HTTPAddr: ":80", + HTTPSAddr: ":443", } - if devMode() { - // Parse it optimistically - var err error - tmpl, err = template.New("home").Parse(embeddedTemplate) - if err != nil { - log.Printf("ignoring template error in dev mode: %v", err) - } - } else { - if embeddedTemplate == "" { - log.Fatalf("embeddedTemplate is empty; must be build with Go 1.16+") - } - tmpl = template.Must(template.New("home").Parse(embeddedTemplate)) - } - - http.HandleFunc("/", root) log.Printf("Starting hello server.") - - errc := make(chan error, 1) - if *httpAddr != "" { - log.Printf("running HTTP server on %s", *httpAddr) - go func() { - errc <- http.ListenAndServe(*httpAddr, nil) - }() - } - if *httpsAddr != "" { - log.Printf("running HTTPS server on %s", *httpsAddr) - go func() { - hs := &http.Server{ - Addr: *httpsAddr, - TLSConfig: &tls.Config{ - GetCertificate: func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) { - switch hi.ServerName { - case "hello.ts.net": - return localClient.GetCertificate(hi) - case "hello.ipn.dev": - c, err := tls.LoadX509KeyPair( - "/etc/hello/hello.ipn.dev.crt", - "/etc/hello/hello.ipn.dev.key", - ) - if err != nil { - return nil, err - } - return &c, nil - } - return nil, errors.New("invalid SNI name") - }, - }, - IdleTimeout: 30 * time.Second, - ReadHeaderTimeout: 20 * time.Second, - MaxHeaderBytes: 10 << 10, - } - errc <- hs.ListenAndServeTLS("", "") - }() - } - log.Fatal(<-errc) -} - -func devMode() bool { return *httpsAddr == "" && *httpAddr != "" } - -func getTmpl() (*template.Template, error) { - if devMode() { - tmplData, err := os.ReadFile("hello.tmpl.html") - if os.IsNotExist(err) { - log.Printf("using baked-in template in dev mode; can't find hello.tmpl.html in current directory") - return tmpl, nil - } - return template.New("home").Parse(string(tmplData)) - } - return tmpl, nil -} - -// tmpl is the template used in prod mode. -// In dev mode it's only used if the template file doesn't exist on disk. -// It's initialized by main after flag parsing. -var tmpl *template.Template - -type tmplData struct { - DisplayName string // "Foo Barberson" - LoginName string // "foo@bar.com" - ProfilePicURL string // "https://..." - MachineName string // "imac5k" - MachineOS string // "Linux" - IP string // "100.2.3.4" -} - -func tailscaleIP(who *apitype.WhoIsResponse) string { - if who == nil { - return "" - } - vals, err := tailcfg.UnmarshalNodeCapJSON[string](who.Node.CapMap, tailcfg.NodeAttrNativeIPV4) - if err == nil && len(vals) > 0 { - return vals[0] - } - for _, nodeIP := range who.Node.Addresses { - if nodeIP.Addr().Is4() && nodeIP.IsSingleIP() { - return nodeIP.Addr().String() - } - } - for _, nodeIP := range who.Node.Addresses { - if nodeIP.IsSingleIP() { - return nodeIP.Addr().String() - } - } - return "" -} - -func root(w http.ResponseWriter, r *http.Request) { - if r.TLS == nil && *httpsAddr != "" { - host := r.Host - if strings.Contains(r.Host, "100.101.102.103") || - strings.Contains(r.Host, "hello.ipn.dev") { - host = "hello.ts.net" - } - http.Redirect(w, r, "https://"+host, http.StatusFound) - return - } - if r.RequestURI != "/" { - http.Redirect(w, r, "/", http.StatusFound) - return - } - if r.TLS != nil && *httpsAddr != "" && strings.Contains(r.Host, "hello.ipn.dev") { - http.Redirect(w, r, "https://hello.ts.net", http.StatusFound) - return - } - tmpl, err := getTmpl() - if err != nil { - w.Header().Set("Content-Type", "text/plain") - http.Error(w, "template error: "+err.Error(), 500) - return - } - - who, err := localClient.WhoIs(r.Context(), r.RemoteAddr) - var data tmplData - if err != nil { - if devMode() { - log.Printf("warning: using fake data in dev mode due to whois lookup error: %v", err) - data = tmplData{ - DisplayName: "Taily Scalerson", - LoginName: "taily@scaler.son", - ProfilePicURL: "https://placekitten.com/200/200", - MachineName: "scaled", - MachineOS: "Linux", - IP: "100.1.2.3", - } - } else { - log.Printf("whois(%q) error: %v", r.RemoteAddr, err) - http.Error(w, "Your Tailscale works, but we failed to look you up.", 500) - return - } - } else { - data = tmplData{ - DisplayName: who.UserProfile.DisplayName, - LoginName: who.UserProfile.LoginName, - ProfilePicURL: who.UserProfile.ProfilePicURL, - MachineName: firstLabel(who.Node.ComputedName), - MachineOS: who.Node.Hostinfo.OS(), - IP: tailscaleIP(who), - } - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - tmpl.Execute(w, data) -} - -// firstLabel s up until the first period, if any. -func firstLabel(s string) string { - s, _, _ = strings.Cut(s, ".") - return s + log.Fatal(s.Run()) } diff --git a/cmd/hello/hello.tmpl.html b/cmd/hello/hello.tmpl.html deleted file mode 100644 index 3ecd1b58ab9b5..0000000000000 --- a/cmd/hello/hello.tmpl.html +++ /dev/null @@ -1,438 +0,0 @@ - - - - - - Hello from Tailscale - - - - -
- -
-

You're connected over Tailscale!

-

This device is signed in asâ€Ļ

-
-
- -
-
-
- {{ with .DisplayName }} -

{{.}}

- {{ end }} -
{{.LoginName}}
-
-
-
-
- - - - - - -

{{.MachineName}}

-
-
{{.IP}}
-
-
- -
- - diff --git a/cmd/hello/helloserver/hello.tmpl.html b/cmd/hello/helloserver/hello.tmpl.html new file mode 100644 index 0000000000000..0f74d116f74cc --- /dev/null +++ b/cmd/hello/helloserver/hello.tmpl.html @@ -0,0 +1,71 @@ + + + + + + Hello from Tailscale + + + + +
+ +
+

You're connected over Tailscale!

+

This device is signed in asâ€Ļ

+
+
+ +
+
+ Profile picture +
+
+ {{ with .DisplayName }} +

{{.}}

+ {{ end }} +
{{.LoginName}}
+
+
+
+
+ + + + + + +

{{.MachineName}}

+
+
{{.IP}}
+
+
+ +
+ + diff --git a/cmd/hello/helloserver/helloserver.go b/cmd/hello/helloserver/helloserver.go new file mode 100644 index 0000000000000..41e7dbce20cda --- /dev/null +++ b/cmd/hello/helloserver/helloserver.go @@ -0,0 +1,157 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package helloserver implements the HTTP server behind hello.ts.net. +package helloserver + +import ( + "crypto/tls" + "embed" + "html/template" + "log" + "net/http" + "strings" + "time" + + "tailscale.com/client/local" + "tailscale.com/client/tailscale/apitype" + "tailscale.com/tailcfg" +) + +//go:embed hello.tmpl.html +var embeddedTemplate string + +//go:embed static/* +var staticFiles embed.FS + +var staticHandler = http.FileServerFS(staticFiles) + +var tmpl = template.Must(template.New("home").Parse(embeddedTemplate)) + +// Server is an HTTP server for hello.ts.net. +// +// The zero value is not valid; populate at least one of HTTPAddr or HTTPSAddr +// before calling Run. +type Server struct { + // HTTPAddr is the address to run an HTTP server on, or empty for none. + HTTPAddr string + + // HTTPSAddr is the address to run an HTTPS server on, or empty for none. + HTTPSAddr string + + // LocalClient is used to look up the identity of incoming requests and + // to obtain TLS certificates. If nil, the zero value of local.Client is + // used. + LocalClient *local.Client +} + +func (s *Server) localClient() *local.Client { + if s.LocalClient != nil { + return s.LocalClient + } + return &local.Client{} +} + +// Run starts the configured HTTP and HTTPS servers and blocks until one of +// them returns an error. +func (s *Server) Run() error { + errc := make(chan error, 1) + if s.HTTPAddr != "" { + log.Printf("running HTTP server on %s", s.HTTPAddr) + go func() { + errc <- http.ListenAndServe(s.HTTPAddr, s) + }() + } + if s.HTTPSAddr != "" { + log.Printf("running HTTPS server on %s", s.HTTPSAddr) + go func() { + hs := &http.Server{ + Addr: s.HTTPSAddr, + Handler: s, + TLSConfig: &tls.Config{ + GetCertificate: s.localClient().GetCertificate, + }, + IdleTimeout: 30 * time.Second, + ReadHeaderTimeout: 20 * time.Second, + MaxHeaderBytes: 10 << 10, + } + errc <- hs.ListenAndServeTLS("", "") + }() + } + return <-errc +} + +type tmplData struct { + DisplayName string // "Foo Barberson" + LoginName string // "foo@bar.com" + ProfilePicURL string // "https://..." + MachineName string // "imac5k" + MachineOS string // "Linux" + IP string // "100.2.3.4" +} + +func tailscaleIP(who *apitype.WhoIsResponse) string { + if who == nil { + return "" + } + vals, err := tailcfg.UnmarshalNodeCapJSON[string](who.Node.CapMap, tailcfg.NodeAttrNativeIPV4) + if err == nil && len(vals) > 0 { + return vals[0] + } + for _, nodeIP := range who.Node.Addresses { + if nodeIP.Addr().Is4() && nodeIP.IsSingleIP() { + return nodeIP.Addr().String() + } + } + for _, nodeIP := range who.Node.Addresses { + if nodeIP.IsSingleIP() { + return nodeIP.Addr().String() + } + } + return "" +} + +// ServeHTTP implements http.Handler. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.TLS == nil && s.HTTPSAddr != "" { + host := r.Host + if strings.Contains(r.Host, "100.101.102.103") { + host = "hello.ts.net" + } + http.Redirect(w, r, "https://"+host, http.StatusFound) + return + } + + if strings.HasPrefix(r.RequestURI, "/static/") { + staticHandler.ServeHTTP(w, r) + return + } + + if r.RequestURI != "/" { + http.Redirect(w, r, "/", http.StatusFound) + return + } + + who, err := s.localClient().WhoIs(r.Context(), r.RemoteAddr) + if err != nil { + log.Printf("whois(%q) error: %v", r.RemoteAddr, err) + http.Error(w, "Your Tailscale works, but we failed to look you up.", 500) + return + } + data := tmplData{ + DisplayName: who.UserProfile.DisplayName, + LoginName: who.UserProfile.LoginName, + ProfilePicURL: who.UserProfile.ProfilePicURL, + MachineName: firstLabel(who.Node.ComputedName), + MachineOS: who.Node.Hostinfo.OS(), + IP: tailscaleIP(who), + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl.Execute(w, data) +} + +// firstLabel returns s up until the first period, if any. +func firstLabel(s string) string { + s, _, _ = strings.Cut(s, ".") + return s +} diff --git a/cmd/hello/helloserver/static/script.js b/cmd/hello/helloserver/static/script.js new file mode 100644 index 0000000000000..db9bcd0f3426e --- /dev/null +++ b/cmd/hello/helloserver/static/script.js @@ -0,0 +1,12 @@ +(function () { + var lastSeen = localStorage.getItem("lastSeen"); + if (!lastSeen) { + document.body.classList.add("animate"); + window.addEventListener("load", function () { + setTimeout(function () { + document.body.classList.add("animating"); + localStorage.setItem("lastSeen", Date.now()); + }, 100); + }); + } +})(); diff --git a/cmd/hello/helloserver/static/style.css b/cmd/hello/helloserver/static/style.css new file mode 100644 index 0000000000000..8ad55edc666c2 --- /dev/null +++ b/cmd/hello/helloserver/static/style.css @@ -0,0 +1,366 @@ +html, +body { + margin: 0; + padding: 0; +} + +body { + font-family: Inter, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + font-size: 100%; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +html, +body, +main { + height: 100%; +} + +*, +::before, +::after { + box-sizing: border-box; + border-width: 0; + border-style: solid; + border-color: #dad6d5; +} + +h1, +h2, +h3, +h4, +h5, +h6 { + margin: 0; + font-size: 1rem; + font-weight: inherit; +} + +a { + color: inherit; +} + +p { + margin: 0; +} + +main { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + max-width: 24rem; + width: 95%; + margin-left: auto; + margin-right: auto; +} + +.p-2 { + padding: 0.5rem; +} + +.p-4 { + padding: 1rem; +} + +.px-2 { + padding-left: 0.5rem; + padding-right: 0.5rem; +} + +.pl-3 { + padding-left: 0.75rem; +} + +.pr-3 { + padding-right: 0.75rem; +} + +.pt-4 { + padding-top: 1rem; +} + +.mr-2 { + margin-right: 0.5rem; +; +} + +.mb-1 { + margin-bottom: 0.25rem; +} + +.mb-2 { + margin-bottom: 0.5rem; +} + +.mb-4 { + margin-bottom: 1rem; +} + +.mb-6 { + margin-bottom: 1.5rem; +} + +.mb-8 { + margin-bottom: 2rem; +} + +.mb-12 { + margin-bottom: 3rem; +} + +.width-full { + width: 100%; +} + +.min-width-0 { + min-width: 0; +} + +.rounded-lg { + border-radius: 0.5rem; +} + +.relative { + position: relative; +} + +.flex { + display: flex; +} + +.justify-between { + justify-content: space-between; +} + +.items-center { + align-items: center; +} + +.border { + border-width: 1px; +} + +.border-t-1 { + border-top-width: 1px; +} + +.border-gray-100 { + border-color: #f7f5f4; +} + +.border-gray-200 { + border-color: #eeebea; +} + +.border-gray-300 { + border-color: #dad6d5; +} + +.bg-white { + background-color: white; +} + +.bg-gray-0 { + background-color: #faf9f8; +} + +.bg-gray-100 { + background-color: #f7f5f4; +} + +.text-green-600 { + color: #0d4b3b; +} + +.text-blue-600 { + color: #3f5db3; +} + +.hover\:text-blue-800:hover { + color: #253570; +} + +.text-gray-600 { + color: #444342; +} + +.text-gray-700 { + color: #2e2d2d; +} + +.text-gray-800 { + color: #232222; +} + +.text-center { + text-align: center; +} + +.text-sm { + font-size: 0.875rem; +} + +.font-title { + font-size: 1.25rem; + letter-spacing: -0.025em; +} + +.font-semibold { + font-weight: 600; +} + +.font-medium { + font-weight: 500; +} + +.font-regular { + font-weight: 400; +} + +.truncate { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.overflow-hidden { + overflow: hidden; +} + +.profile-pic { + width: 2.5rem; + height: 2.5rem; + background-size: cover; + margin-right: 0.5rem; + flex-shrink: 0; +} + +.profile-pic-img { + width: 100%; + height: 100%; + object-fit: cover; + display: block; + border-radius: 9999px; +} + +.panel { + box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04); +} + +.animate .panel { + transform: translateY(10%); + box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.0), 0 10px 10px -5px rgba(0, 0, 0, 0.0); + transition: transform 1200ms ease, opacity 1200ms ease, box-shadow 1200ms ease; +} + +.animate .panel-interior { + opacity: 0.0; + transition: opacity 1200ms ease; +} + +.animate .logo { + transform: translateY(2rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animate .header-title { + transform: translateY(1.6rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animate .header-text { + transform: translateY(1.2rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animate .footer { + transform: translateY(-0.5rem); + opacity: 0.0; + transition: transform 1200ms ease, opacity 1200ms ease; +} + +.animating .panel { + transform: translateY(0); + opacity: 1.0; + box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04); +} + +.animating .panel-interior { + opacity: 1.0; +} + +.animating .spinner { + opacity: 0.0; +} + +.animating .logo, +.animating .header-title, +.animating .header-text, +.animating .footer { + transform: translateY(0); + opacity: 1.0; +} + +.spinner { + display: inline-flex; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + align-items: center; + transition: opacity 200ms ease; +} + +.spinner span { + display: inline-block; + background-color: currentColor; + border-radius: 9999px; + animation-name: loading-dots-blink; + animation-duration: 1.4s; + animation-iteration-count: infinite; + animation-fill-mode: both; + width: 0.35em; + height: 0.35em; + margin: 0 0.15em; +} + +.spinner span:nth-child(2) { + animation-delay: 200ms; +} + +.spinner span:nth-child(3) { + animation-delay: 400ms; +} + +.spinner { + display: none; +} + +.animate .spinner { + display: inline-flex; +} + +@keyframes loading-dots-blink { + 0% { + opacity: 0.2; + } + 20% { + opacity: 1; + } + 100% { + opacity: 0.2; + } +} + +@media (prefers-reduced-motion) { + * { + animation-duration: 0ms !important; + transition-duration: 0ms !important; + transition-delay: 0ms !important; + } +} diff --git a/cmd/jsonimports/format.go b/cmd/jsonimports/format.go new file mode 100644 index 0000000000000..e990d0e6745c3 --- /dev/null +++ b/cmd/jsonimports/format.go @@ -0,0 +1,175 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "bytes" + "go/ast" + "go/format" + "go/parser" + "go/token" + "go/types" + "path" + "slices" + "strconv" + "strings" + + "tailscale.com/util/must" +) + +// mustFormatFile formats a Go source file and adjust "json" imports. +// It panics if there are any parsing errors. +// +// - "encoding/json" is imported under the name "jsonv1" or "jsonv1std" +// - "encoding/json/v2" is rewritten to import "github.com/go-json-experiment/json" instead +// - "encoding/json/jsontext" is rewritten to import "github.com/go-json-experiment/json/jsontext" instead +// - "github.com/go-json-experiment/json" is imported under the name "jsonv2" +// - "github.com/go-json-experiment/json/v1" is imported under the name "jsonv1" +// +// If no changes to the file is made, it returns input. +func mustFormatFile(in []byte) (out []byte) { + fset := token.NewFileSet() + f := must.Get(parser.ParseFile(fset, "", in, parser.ParseComments)) + + // Check for the existence of "json" imports. + jsonImports := make(map[string][]*ast.ImportSpec) + for _, imp := range f.Imports { + switch pkgPath := must.Get(strconv.Unquote(imp.Path.Value)); pkgPath { + case + "encoding/json", + "encoding/json/v2", + "encoding/json/jsontext", + "github.com/go-json-experiment/json", + "github.com/go-json-experiment/json/v1", + "github.com/go-json-experiment/json/jsontext": + jsonImports[pkgPath] = append(jsonImports[pkgPath], imp) + } + } + if len(jsonImports) == 0 { + return in + } + + // Best-effort local type-check of the file + // to resolve local declarations to detect shadowed variables. + typeInfo := &types.Info{Uses: make(map[*ast.Ident]types.Object)} + (&types.Config{ + Error: func(err error) {}, + }).Check("", fset, []*ast.File{f}, typeInfo) + + // Rewrite imports to instead use "github.com/go-json-experiment/json". + // This ensures that code continues to build even if + // goexperiment.jsonv2 is *not* specified. + // As of https://github.com/go-json-experiment/json/pull/186, + // imports to "github.com/go-json-experiment/json" are identical + // to the standard library if built with goexperiment.jsonv2. + for fromPath, toPath := range map[string]string{ + "encoding/json/v2": "github.com/go-json-experiment/json", + "encoding/json/jsontext": "github.com/go-json-experiment/json/jsontext", + } { + for _, imp := range jsonImports[fromPath] { + imp.Path.Value = strconv.Quote(toPath) + jsonImports[toPath] = append(jsonImports[toPath], imp) + } + delete(jsonImports, fromPath) + } + + // While in a transitory state, where both v1 and v2 json imports + // may exist in our codebase, always explicitly import with + // either jsonv1 or jsonv2 in the package name to avoid ambiguities + // when looking at a particular Marshal or Unmarshal call site. + renames := make(map[string]string) // mapping of old names to new names + deletes := make(map[*ast.ImportSpec]bool) // set of imports to delete + for pkgPath, imps := range jsonImports { + var newName string + switch pkgPath { + case "encoding/json": + newName = "jsonv1" + // If "github.com/go-json-experiment/json/v1" is also imported, + // then use jsonv1std for "encoding/json" to avoid a conflict. + if len(jsonImports["github.com/go-json-experiment/json/v1"]) > 0 { + newName += "std" + } + case "github.com/go-json-experiment/json": + newName = "jsonv2" + case "github.com/go-json-experiment/json/v1": + newName = "jsonv1" + } + + // Rename the import if different than expected. + if oldName := importName(imps[0]); oldName != newName && newName != "" { + renames[oldName] = newName + pos := imps[0].Pos() // preserve original positioning + imps[0].Name = ast.NewIdent(newName) + imps[0].Name.NamePos = pos + } + + // For all redundant imports, use the first imported name. + for _, imp := range imps[1:] { + renames[importName(imp)] = importName(imps[0]) + deletes[imp] = true + } + } + if len(deletes) > 0 { + f.Imports = slices.DeleteFunc(f.Imports, func(imp *ast.ImportSpec) bool { + return deletes[imp] + }) + for _, decl := range f.Decls { + if genDecl, ok := decl.(*ast.GenDecl); ok && genDecl.Tok == token.IMPORT { + genDecl.Specs = slices.DeleteFunc(genDecl.Specs, func(spec ast.Spec) bool { + return deletes[spec.(*ast.ImportSpec)] + }) + } + } + } + if len(renames) > 0 { + ast.Walk(astVisitor(func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok { + // Just because the selector looks like "json.Marshal" + // does not mean that it is referencing the "json" package. + // There could be a local "json" declaration that shadows + // the package import. Check partial type information + // to see if there was a local declaration. + if obj, ok := typeInfo.Uses[id]; ok { + if _, ok := obj.(*types.PkgName); !ok { + return true + } + } + + if newName, ok := renames[id.String()]; ok { + id.Name = newName + } + } + } + return true + }), f) + } + + bb := new(bytes.Buffer) + must.Do(format.Node(bb, fset, f)) + return must.Get(format.Source(bb.Bytes())) +} + +// importName is the local package name used for an import. +// If no explicit local name is used, then it uses string parsing +// to derive the package name from the path, relying on the convention +// that the package name is the base name of the package path. +func importName(imp *ast.ImportSpec) string { + if imp.Name != nil { + return imp.Name.String() + } + pkgPath, _ := strconv.Unquote(imp.Path.Value) + pkgPath = strings.TrimRight(pkgPath, "/v0123456789") // exclude version directories + return path.Base(pkgPath) +} + +// astVisitor is a function that implements [ast.Visitor]. +type astVisitor func(ast.Node) bool + +func (f astVisitor) Visit(node ast.Node) ast.Visitor { + if !f(node) { + return nil + } + return f +} diff --git a/cmd/jsonimports/format_test.go b/cmd/jsonimports/format_test.go new file mode 100644 index 0000000000000..fb3d6fa09698d --- /dev/null +++ b/cmd/jsonimports/format_test.go @@ -0,0 +1,162 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "go/format" + "testing" + + "tailscale.com/util/must" + "tailscale.com/util/safediff" +) + +func TestFormatFile(t *testing.T) { + tests := []struct{ in, want string }{{ + in: `package foobar + + import ( + "encoding/json" + jsonv2exp "github.com/go-json-experiment/json" + ) + + func main() { + json.Marshal() + jsonv2exp.Marshal() + { + var json T // deliberately shadow "json" package name + json.Marshal() // should not be re-written + } + } + `, + want: `package foobar + + import ( + jsonv1 "encoding/json" + jsonv2 "github.com/go-json-experiment/json" + ) + + func main() { + jsonv1.Marshal() + jsonv2.Marshal() + { + var json T // deliberately shadow "json" package name + json.Marshal() // should not be re-written + } + } + `, + }, { + in: `package foobar + + import ( + "github.com/go-json-experiment/json" + jsonv2exp "github.com/go-json-experiment/json" + ) + + func main() { + json.Marshal() + jsonv2exp.Marshal() + } + `, + want: `package foobar + import ( + jsonv2 "github.com/go-json-experiment/json" + ) + func main() { + jsonv2.Marshal() + jsonv2.Marshal() + } + `, + }, { + in: `package foobar + import "github.com/go-json-experiment/json/v1" + func main() { + json.Marshal() + } + `, + want: `package foobar + import jsonv1 "github.com/go-json-experiment/json/v1" + func main() { + jsonv1.Marshal() + } + `, + }, { + in: `package foobar + import ( + "encoding/json" + jsonv1in2 "github.com/go-json-experiment/json/v1" + ) + func main() { + json.Marshal() + jsonv1in2.Marshal() + } + `, + want: `package foobar + import ( + jsonv1std "encoding/json" + jsonv1 "github.com/go-json-experiment/json/v1" + ) + func main() { + jsonv1std.Marshal() + jsonv1.Marshal() + } + `, + }, { + in: `package foobar + import ( + "encoding/json" + jsonv1in2 "github.com/go-json-experiment/json/v1" + ) + func main() { + json.Marshal() + jsonv1in2.Marshal() + } + `, + want: `package foobar + import ( + jsonv1std "encoding/json" + jsonv1 "github.com/go-json-experiment/json/v1" + ) + func main() { + jsonv1std.Marshal() + jsonv1.Marshal() + } + `, + }, { + in: `package foobar + import ( + "encoding/json" + j2 "encoding/json/v2" + "encoding/json/jsontext" + ) + func main() { + json.Marshal() + j2.Marshal() + jsontext.NewEncoder + } + `, + want: `package foobar + import ( + jsonv1 "encoding/json" + jsonv2 "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + ) + func main() { + jsonv1.Marshal() + jsonv2.Marshal() + jsontext.NewEncoder + } + `, + }} + for _, tt := range tests { + got := string(must.Get(format.Source([]byte(tt.in)))) + got = string(mustFormatFile([]byte(got))) + want := string(must.Get(format.Source([]byte(tt.want)))) + if got != want { + diff, _ := safediff.Lines(got, want, -1) + t.Errorf("mismatch (-got +want)\n%s", diff) + t.Error(got) + t.Error(want) + } + } +} diff --git a/cmd/jsonimports/jsonimports.go b/cmd/jsonimports/jsonimports.go new file mode 100644 index 0000000000000..6903844e121ca --- /dev/null +++ b/cmd/jsonimports/jsonimports.go @@ -0,0 +1,124 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// The jsonimports tool formats all Go source files in the repository +// to enforce that "json" imports are consistent. +// +// With Go 1.25, the "encoding/json/v2" and "encoding/json/jsontext" +// packages are now available under goexperiment.jsonv2. +// This leads to possible confusion over the following: +// +// - "encoding/json" +// - "encoding/json/v2" +// - "encoding/json/jsontext" +// - "github.com/go-json-experiment/json/v1" +// - "github.com/go-json-experiment/json" +// - "github.com/go-json-experiment/json/jsontext" +// +// In order to enforce consistent usage, we apply the following rules: +// +// - Until the Go standard library formally accepts "encoding/json/v2" +// and "encoding/json/jsontext" into the standard library +// (i.e., they are no longer considered experimental), +// we forbid any code from directly importing those packages. +// Go code should instead import "github.com/go-json-experiment/json" +// and "github.com/go-json-experiment/json/jsontext". +// The latter packages contain aliases to the standard library +// if built on Go 1.25 with the goexperiment.jsonv2 tag specified. +// +// - Imports of "encoding/json" or "github.com/go-json-experiment/json/v1" +// must be explicitly imported under the package name "jsonv1". +// If both packages need to be imported, then the former should +// be imported under the package name "jsonv1std". +// +// - Imports of "github.com/go-json-experiment/json" +// must be explicitly imported under the package name "jsonv2". +// +// The latter two rules exist to provide clarity when reading code. +// Without them, it is unclear whether "json.Marshal" refers to v1 or v2. +// With them, however, it is clear that "jsonv1.Marshal" is calling v1 and +// that "jsonv2.Marshal" is calling v2. +// +// TODO(@joetsai): At this present moment, there is no guidance given on +// whether to use v1 or v2 for newly written Go source code. +// I will write a document in the near future providing more guidance. +// Feel free to continue using v1 "encoding/json" as you are accustomed to. +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "sync" + + "tailscale.com/syncs" + "tailscale.com/util/must" + "tailscale.com/util/safediff" +) + +func main() { + update := flag.Bool("update", false, "update all Go source files") + flag.Parse() + + // Change working directory to Git repository root. + repoRoot := strings.TrimSuffix(string(must.Get(exec.Command( + "git", "rev-parse", "--show-toplevel", + ).Output())), "\n") + must.Do(os.Chdir(repoRoot)) + + // Iterate over all indexed files in the Git repository. + var printMu sync.Mutex + var group sync.WaitGroup + sema := syncs.NewSemaphore(runtime.NumCPU()) + var numDiffs int + files := string(must.Get(exec.Command("git", "ls-files").Output())) + for file := range strings.Lines(files) { + sema.Acquire() + group.Go(func() { + defer sema.Release() + + // Ignore non-Go source files. + file = strings.TrimSuffix(file, "\n") + if !strings.HasSuffix(file, ".go") { + return + } + + // Format all "json" imports in the Go source file. + srcIn := must.Get(os.ReadFile(file)) + srcOut := mustFormatFile(srcIn) + + // Print differences with each formatted file. + if !bytes.Equal(srcIn, srcOut) { + numDiffs++ + + printMu.Lock() + fmt.Println(file) + lines, _ := safediff.Lines(string(srcIn), string(srcOut), -1) + for line := range strings.Lines(lines) { + fmt.Print("\t", line) + } + fmt.Println() + printMu.Unlock() + + // If -update is specified, write out the changes. + if *update { + mode := must.Get(os.Stat(file)).Mode() + must.Do(os.WriteFile(file, srcOut, mode)) + } + } + }) + } + group.Wait() + + // Report whether any differences were detected. + if numDiffs > 0 && !*update { + fmt.Printf(`%d files with "json" imports that need formatting`+"\n", numDiffs) + fmt.Println("Please run:") + fmt.Println("\t./tool/go run tailscale.com/cmd/jsonimports -update") + os.Exit(1) + } +} diff --git a/cmd/k8s-nameserver/main.go b/cmd/k8s-nameserver/main.go index ca4b449358083..1b219fb1ab924 100644 --- a/cmd/k8s-nameserver/main.go +++ b/cmd/k8s-nameserver/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -31,6 +31,9 @@ const ( tsNetDomain = "ts.net" // addr is the the address that the UDP and TCP listeners will listen on. addr = ":1053" + // defaultTTL is the default TTL for DNS records in seconds. + // Set to 0 to disable caching. Can be increased when usage patterns are better understood. + defaultTTL = 0 // The following constants are specific to the nameserver configuration // provided by a mounted Kubernetes Configmap. The Configmap mounted at @@ -39,9 +42,9 @@ const ( kubeletMountedConfigLn = "..data" ) -// nameserver is a simple nameserver that responds to DNS queries for A records +// nameserver is a simple nameserver that responds to DNS queries for A and AAAA records // for ts.net domain names over UDP or TCP. It serves DNS responses from -// in-memory IPv4 host records. It is intended to be deployed on Kubernetes with +// in-memory IPv4 and IPv6 host records. It is intended to be deployed on Kubernetes with // a ConfigMap mounted at /config that should contain the host records. It // dynamically reconfigures its in-memory mappings as the contents of the // mounted ConfigMap changes. @@ -56,10 +59,13 @@ type nameserver struct { // in-memory records. configWatcher <-chan string - mu sync.Mutex // protects following + mu sync.RWMutex // protects following // ip4 are the in-memory hostname -> IP4 mappings that the nameserver // uses to respond to A record queries. ip4 map[dnsname.FQDN][]net.IP + // ip6 are the in-memory hostname -> IP6 mappings that the nameserver + // uses to respond to AAAA record queries. + ip6 map[dnsname.FQDN][]net.IP } func main() { @@ -98,16 +104,13 @@ func main() { tcpSig <- s // stop the TCP listener } -// handleFunc is a DNS query handler that can respond to A record queries from +// handleFunc is a DNS query handler that can respond to A and AAAA record queries from // the nameserver's in-memory records. -// - If an A record query is received and the -// nameserver's in-memory records contain records for the queried domain name, -// return a success response. -// - If an A record query is received, but the -// nameserver's in-memory records do not contain records for the queried domain name, -// return NXDOMAIN. -// - If an A record query is received, but the queried domain name is not valid, return Format Error. -// - If a query is received for any other record type than A, return Not Implemented. +// - For A queries: returns IPv4 addresses if available, NXDOMAIN if the name doesn't exist +// - For AAAA queries: returns IPv6 addresses if available, NOERROR with no data if only +// IPv4 exists (per RFC 4074), or NXDOMAIN if the name doesn't exist at all +// - For invalid domain names: returns Format Error +// - For other record types: returns Not Implemented func (n *nameserver) handleFunc() func(w dns.ResponseWriter, r *dns.Msg) { h := func(w dns.ResponseWriter, r *dns.Msg) { m := new(dns.Msg) @@ -135,35 +138,19 @@ func (n *nameserver) handleFunc() func(w dns.ResponseWriter, r *dns.Msg) { m.RecursionAvailable = false ips := n.lookupIP4(fqdn) - if ips == nil || len(ips) == 0 { + if len(ips) == 0 { // As we are the authoritative nameserver for MagicDNS // names, if we do not have a record for this MagicDNS // name, it does not exist. m = m.SetRcode(r, dns.RcodeNameError) return } - // TODO (irbekrm): TTL is currently set to 0, meaning - // that cluster workloads will not cache the DNS - // records. Revisit this in future when we understand - // the usage patterns better- is it putting too much - // load on kube DNS server or is this fine? for _, ip := range ips { - rr := &dns.A{Hdr: dns.RR_Header{Name: q, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 0}, A: ip} + rr := &dns.A{Hdr: dns.RR_Header{Name: q, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: defaultTTL}, A: ip} m.SetRcode(r, dns.RcodeSuccess) m.Answer = append(m.Answer, rr) } case dns.TypeAAAA: - // TODO (irbekrm): add IPv6 support. - // The nameserver currently does not support IPv6 - // (records are not being created for IPv6 Pod addresses). - // However, we can expect that some callers will - // nevertheless send AAAA queries. - // We have to return NOERROR if a query is received for - // an AAAA record for a DNS name that we have an A - // record for- else the caller might not follow with an - // A record query. - // https://github.com/tailscale/tailscale/issues/12321 - // https://datatracker.ietf.org/doc/html/rfc4074 q := r.Question[0].Name fqdn, err := dnsname.ToFQDN(q) if err != nil { @@ -174,14 +161,27 @@ func (n *nameserver) handleFunc() func(w dns.ResponseWriter, r *dns.Msg) { // single source of truth for MagicDNS names by // non-tailnet Kubernetes workloads. m.Authoritative = true - ips := n.lookupIP4(fqdn) - if len(ips) == 0 { + m.RecursionAvailable = false + + ips := n.lookupIP6(fqdn) + // Also check if we have IPv4 records to determine correct response code. + // If the name exists (has A records) but no AAAA records, we return NOERROR + // per RFC 4074. If the name doesn't exist at all, we return NXDOMAIN. + ip4s := n.lookupIP4(fqdn) + + if len(ips) == 0 && len(ip4s) == 0 { // As we are the authoritative nameserver for MagicDNS - // names, if we do not have a record for this MagicDNS + // names, if we do not have any record for this MagicDNS // name, it does not exist. m = m.SetRcode(r, dns.RcodeNameError) return } + + // Return IPv6 addresses if available + for _, ip := range ips { + rr := &dns.AAAA{Hdr: dns.RR_Header{Name: q, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: defaultTTL}, AAAA: ip} + m.Answer = append(m.Answer, rr) + } m.SetRcode(r, dns.RcodeSuccess) default: log.Printf("[unexpected] nameserver received a query for an unsupported record type: %s", r.Question[0].String()) @@ -231,10 +231,11 @@ func (n *nameserver) resetRecords() error { log.Printf("error reading nameserver's configuration: %v", err) return err } - if dnsCfgBytes == nil || len(dnsCfgBytes) < 1 { + if len(dnsCfgBytes) == 0 { log.Print("nameserver's configuration is empty, any in-memory records will be unset") n.mu.Lock() n.ip4 = make(map[dnsname.FQDN][]net.IP) + n.ip6 = make(map[dnsname.FQDN][]net.IP) n.mu.Unlock() return nil } @@ -249,30 +250,63 @@ func (n *nameserver) resetRecords() error { } ip4 := make(map[dnsname.FQDN][]net.IP) + ip6 := make(map[dnsname.FQDN][]net.IP) defer func() { n.mu.Lock() defer n.mu.Unlock() n.ip4 = ip4 + n.ip6 = ip6 }() - if len(dnsCfg.IP4) == 0 { + if len(dnsCfg.IP4) == 0 && len(dnsCfg.IP6) == 0 { log.Print("nameserver's configuration contains no records, any in-memory records will be unset") return nil } + // Process IPv4 records for fqdn, ips := range dnsCfg.IP4 { fqdn, err := dnsname.ToFQDN(fqdn) if err != nil { log.Printf("invalid nameserver's configuration: %s is not a valid FQDN: %v; skipping this record", fqdn, err) continue // one invalid hostname should not break the whole nameserver } + var validIPs []net.IP for _, ipS := range ips { ip := net.ParseIP(ipS).To4() if ip == nil { // To4 returns nil if IP is not a IPv4 address log.Printf("invalid nameserver's configuration: %v does not appear to be an IPv4 address; skipping this record", ipS) continue // one invalid IP address should not break the whole nameserver } - ip4[fqdn] = []net.IP{ip} + validIPs = append(validIPs, ip) + } + if len(validIPs) > 0 { + ip4[fqdn] = validIPs + } + } + + // Process IPv6 records + for fqdn, ips := range dnsCfg.IP6 { + fqdn, err := dnsname.ToFQDN(fqdn) + if err != nil { + log.Printf("invalid nameserver's configuration: %s is not a valid FQDN: %v; skipping this record", fqdn, err) + continue // one invalid hostname should not break the whole nameserver + } + var validIPs []net.IP + for _, ipS := range ips { + ip := net.ParseIP(ipS) + if ip == nil { + log.Printf("invalid nameserver's configuration: %v does not appear to be a valid IP address; skipping this record", ipS) + continue + } + // Check if it's a valid IPv6 address + if ip.To4() != nil { + log.Printf("invalid nameserver's configuration: %v appears to be IPv4 but was in IPv6 records; skipping this record", ipS) + continue + } + validIPs = append(validIPs, ip.To16()) + } + if len(validIPs) > 0 { + ip6[fqdn] = validIPs } } return nil @@ -372,8 +406,20 @@ func (n *nameserver) lookupIP4(fqdn dnsname.FQDN) []net.IP { if n.ip4 == nil { return nil } - n.mu.Lock() - defer n.mu.Unlock() + n.mu.RLock() + defer n.mu.RUnlock() f := n.ip4[fqdn] return f } + +// lookupIP6 returns any IPv6 addresses for the given FQDN from nameserver's +// in-memory records. +func (n *nameserver) lookupIP6(fqdn dnsname.FQDN) []net.IP { + if n.ip6 == nil { + return nil + } + n.mu.RLock() + defer n.mu.RUnlock() + f := n.ip6[fqdn] + return f +} diff --git a/cmd/k8s-nameserver/main_test.go b/cmd/k8s-nameserver/main_test.go index d9a33c4faffe5..b5cd8c907d522 100644 --- a/cmd/k8s-nameserver/main_test.go +++ b/cmd/k8s-nameserver/main_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -19,11 +19,12 @@ func TestNameserver(t *testing.T) { tests := []struct { name string ip4 map[dnsname.FQDN][]net.IP + ip6 map[dnsname.FQDN][]net.IP query *dns.Msg wantResp *dns.Msg }{ { - name: "A record query, record exists", + name: "A-record-exists", ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}}, query: &dns.Msg{ Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeA}}, @@ -45,7 +46,7 @@ func TestNameserver(t *testing.T) { }}, }, { - name: "A record query, record does not exist", + name: "A-record-not-exists", ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}}, query: &dns.Msg{ Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeA}}, @@ -63,7 +64,7 @@ func TestNameserver(t *testing.T) { }}, }, { - name: "A record query, but the name is not a valid FQDN", + name: "A-record-invalid-FQDN", ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}}, query: &dns.Msg{ Question: []dns.Question{{Name: "foo..bar.com", Qtype: dns.TypeA}}, @@ -79,7 +80,7 @@ func TestNameserver(t *testing.T) { }}, }, { - name: "AAAA record query, A record exists", + name: "AAAA-query-A-record-exists", ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}}, query: &dns.Msg{ Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}}, @@ -96,7 +97,7 @@ func TestNameserver(t *testing.T) { }}, }, { - name: "AAAA record query, A record does not exist", + name: "AAAA-query-A-record-not-exists", ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}}, query: &dns.Msg{ Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeAAAA}}, @@ -113,7 +114,50 @@ func TestNameserver(t *testing.T) { }}, }, { - name: "CNAME record query", + name: "AAAA-query-ipv6-record", + ip6: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {net.ParseIP("2001:db8::1")}}, + query: &dns.Msg{ + Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}}, + MsgHdr: dns.MsgHdr{Id: 1, RecursionDesired: true}, + }, + wantResp: &dns.Msg{ + Answer: []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{ + Name: "foo.bar.com", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 0}, + AAAA: net.ParseIP("2001:db8::1")}}, + Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}}, + MsgHdr: dns.MsgHdr{ + Id: 1, + Rcode: dns.RcodeSuccess, + RecursionAvailable: false, + RecursionDesired: true, + Response: true, + Opcode: dns.OpcodeQuery, + Authoritative: true, + }}, + }, + { + name: "dual-stack-A-and-AAAA", + ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("dual.bar.com."): {{10, 0, 0, 1}}}, + ip6: map[dnsname.FQDN][]net.IP{dnsname.FQDN("dual.bar.com."): {net.ParseIP("2001:db8::1")}}, + query: &dns.Msg{ + Question: []dns.Question{{Name: "dual.bar.com", Qtype: dns.TypeAAAA}}, + MsgHdr: dns.MsgHdr{Id: 1}, + }, + wantResp: &dns.Msg{ + Answer: []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{ + Name: "dual.bar.com", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 0}, + AAAA: net.ParseIP("2001:db8::1")}}, + Question: []dns.Question{{Name: "dual.bar.com", Qtype: dns.TypeAAAA}}, + MsgHdr: dns.MsgHdr{ + Id: 1, + Rcode: dns.RcodeSuccess, + Response: true, + Opcode: dns.OpcodeQuery, + Authoritative: true, + }}, + }, + { + name: "CNAME-query", ip4: map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}}, query: &dns.Msg{ Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeCNAME}}, @@ -133,6 +177,7 @@ func TestNameserver(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ns := &nameserver{ ip4: tt.ip4, + ip6: tt.ip6, } handler := ns.handleFunc() fakeRespW := &fakeResponseWriter{} @@ -149,43 +194,63 @@ func TestResetRecords(t *testing.T) { name string config []byte hasIp4 map[dnsname.FQDN][]net.IP + hasIp6 map[dnsname.FQDN][]net.IP wantsIp4 map[dnsname.FQDN][]net.IP + wantsIp6 map[dnsname.FQDN][]net.IP wantsErr bool }{ { - name: "previously empty nameserver.ip4 gets set", + name: "previously-empty-nameserver-ip4-gets-set", config: []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`), wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}}, + wantsIp6: make(map[dnsname.FQDN][]net.IP), }, { - name: "nameserver.ip4 gets reset", + name: "nameserver-ip4-gets-reset", hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}}, config: []byte(`{"version": "v1alpha1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`), wantsIp4: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {{1, 2, 3, 4}}}, + wantsIp6: make(map[dnsname.FQDN][]net.IP), }, { - name: "configuration with incompatible version", + name: "configuration-with-incompatible-version", hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}}, config: []byte(`{"version": "v1beta1", "ip4": {"foo.bar.com": ["1.2.3.4"]}}`), wantsIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}}, + wantsIp6: nil, wantsErr: true, }, { - name: "nameserver.ip4 gets reset to empty config when no configuration is provided", + name: "nameserver-ip4-gets-reset-to-empty-config-when-no-configuration-is-provided", hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}}, wantsIp4: make(map[dnsname.FQDN][]net.IP), + wantsIp6: make(map[dnsname.FQDN][]net.IP), }, { - name: "nameserver.ip4 gets reset to empty config when the provided configuration is empty", + name: "nameserver-ip4-gets-reset-to-empty-config-when-the-provided-configuration-is-empty", hasIp4: map[dnsname.FQDN][]net.IP{"baz.bar.com.": {{1, 1, 3, 3}}}, config: []byte(`{"version": "v1alpha1", "ip4": {}}`), wantsIp4: make(map[dnsname.FQDN][]net.IP), + wantsIp6: make(map[dnsname.FQDN][]net.IP), + }, + { + name: "nameserver-ip6-gets-set", + config: []byte(`{"version": "v1alpha1", "ip6": {"foo.bar.com": ["2001:db8::1"]}}`), + wantsIp4: make(map[dnsname.FQDN][]net.IP), + wantsIp6: map[dnsname.FQDN][]net.IP{"foo.bar.com.": {net.ParseIP("2001:db8::1")}}, + }, + { + name: "dual-stack-configuration", + config: []byte(`{"version": "v1alpha1", "ip4": {"dual.bar.com": ["10.0.0.1"]}, "ip6": {"dual.bar.com": ["2001:db8::1"]}}`), + wantsIp4: map[dnsname.FQDN][]net.IP{"dual.bar.com.": {{10, 0, 0, 1}}}, + wantsIp6: map[dnsname.FQDN][]net.IP{"dual.bar.com.": {net.ParseIP("2001:db8::1")}}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ns := &nameserver{ ip4: tt.hasIp4, + ip6: tt.hasIp6, configReader: func() ([]byte, error) { return tt.config, nil }, } if err := ns.resetRecords(); err == nil == tt.wantsErr { @@ -194,6 +259,9 @@ func TestResetRecords(t *testing.T) { if diff := cmp.Diff(ns.ip4, tt.wantsIp4); diff != "" { t.Fatalf("unexpected nameserver.ip4 contents (-got +want): \n%s", diff) } + if diff := cmp.Diff(ns.ip6, tt.wantsIp6); diff != "" { + t.Fatalf("unexpected nameserver.ip6 contents (-got +want): \n%s", diff) + } }) } } diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go new file mode 100644 index 0000000000000..37260c7a0cbb2 --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -0,0 +1,484 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/client/tailscale/v2" + + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/tstime" +) + +const ( + proxyPGFinalizerName = "tailscale.com/kube-apiserver-finalizer" + + // Reasons for KubeAPIServerProxyValid condition. + reasonKubeAPIServerProxyInvalid = "KubeAPIServerProxyInvalid" + reasonKubeAPIServerProxyValid = "KubeAPIServerProxyValid" + + // Reasons for KubeAPIServerProxyConfigured condition. + reasonKubeAPIServerProxyConfigured = "KubeAPIServerProxyConfigured" + reasonKubeAPIServerProxyNoBackends = "KubeAPIServerProxyNoBackends" +) + +// KubeAPIServerTSServiceReconciler reconciles the Tailscale Services required for an +// HA deployment of the API Server Proxy. +type KubeAPIServerTSServiceReconciler struct { + client.Client + recorder record.EventRecorder + logger *zap.SugaredLogger + clients ClientProvider + tsNamespace string + defaultTags []string + operatorID string // stableID of the operator's Tailscale device + + clock tstime.Clock +} + +// Reconcile is the entry point for the controller. +func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + logger := r.logger.With("ProxyGroup", req.Name) + logger.Debugf("starting reconcile") + defer logger.Debugf("reconcile finished") + + pg := new(tsapi.ProxyGroup) + err = r.Get(ctx, req.NamespacedName, pg) + if apierrors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + logger.Debugf("ProxyGroup not found, assuming it was deleted") + return res, nil + } else if err != nil { + return res, fmt.Errorf("failed to get ProxyGroup: %w", err) + } + + serviceName := serviceNameForAPIServerProxy(pg) + logger = logger.With("Tailscale Service", serviceName) + tsClient, err := r.clients.For(pg.Spec.Tailnet) + if err != nil { + return res, fmt.Errorf("failed to get tailscale client: %w", err) + } + + if markedForDeletion(pg) { + logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up") + if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return res, nil + } + + return res, err + } + + err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient) + if err != nil { + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// maybeProvision ensures that a Tailscale Service for this ProxyGroup exists +// and is up to date. +// +// Returns true if the operation resulted in a Tailscale Service update. +func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, tsClient tsclient.Client) (err error) { + var dnsName string + oldPGStatus := pg.Status.DeepCopy() + defer func() { + podsAdvertising, podsErr := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName.String()) + if podsErr != nil { + err = errors.Join(err, fmt.Errorf("failed to get number of advertised Pods: %w", podsErr)) + // Continue, updating the status with the best available information. + } + + // Update the ProxyGroup status with the Tailscale Service information + // Update the condition based on how many pods are advertising the service + conditionStatus := metav1.ConditionFalse + conditionReason := reasonKubeAPIServerProxyNoBackends + conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, pgReplicas(pg)) + + pg.Status.URL = "" + if podsAdvertising > 0 { + // At least one pod is advertising the service, consider it configured + conditionStatus = metav1.ConditionTrue + conditionReason = reasonKubeAPIServerProxyConfigured + if dnsName != "" { + pg.Status.URL = "https://" + dnsName + } + } + + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger) + + if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { + // An error encountered here should get returned by the Reconcile function. + err = errors.Join(err, r.Client.Status().Update(ctx, pg)) + } + }() + + if !tsoperator.ProxyGroupAvailable(pg) { + return nil + } + + if !slices.Contains(pg.Finalizers, proxyPGFinalizerName) { + // This log line is printed exactly once during initial provisioning, + // because once the finalizer is in place this block gets skipped. So, + // this is a nice place to tell the operator that the high level, + // multi-reconcile operation is underway. + logger.Info("provisioning Tailscale Service for ProxyGroup") + pg.Finalizers = append(pg.Finalizers, proxyPGFinalizerName) + if err := r.Update(ctx, pg); err != nil { + return fmt.Errorf("failed to add finalizer: %w", err) + } + } + + // 1. Check there isn't a Tailscale Service with the same hostname + // already created and not owned by this ProxyGroup. + existingTSSvc, err := tsClient.VIPServices().Get(ctx, serviceName.String()) + if err != nil && !tailscale.IsNotFound(err) { + return fmt.Errorf("error getting Tailscale Service %q: %w", serviceName, err) + } + + updatedAnnotations, err := exclusiveOwnerAnnotations(pg, r.operatorID, existingTSSvc) + if err != nil { + const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different Service name in the ProxyGroup's spec.kubeAPIServer.serviceName field" + msg := fmt.Sprintf("error ensuring exclusive ownership of Tailscale Service %s: %v. %s", serviceName, err, instr) + logger.Warn(msg) + r.recorder.Event(pg, corev1.EventTypeWarning, "InvalidTailscaleService", msg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger) + return nil + } + + // After getting this far, we know the Tailscale Service is valid. + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger) + + // Service tags are limited to matching the ProxyGroup's tags until we have + // support for querying peer caps for a Service-bound request. + serviceTags := r.defaultTags + if len(pg.Spec.Tags) > 0 { + serviceTags = pg.Spec.Tags.Stringify() + } + + tsSvc := tailscale.VIPService{ + Name: serviceName.String(), + Tags: serviceTags, + Ports: []string{"tcp:443"}, + Comment: managedTSServiceComment, + Annotations: updatedAnnotations, + } + if existingTSSvc != nil { + tsSvc.Addrs = existingTSSvc.Addrs + } + + // 2. Ensure the Tailscale Service exists and is up to date. + if existingTSSvc == nil || + !slices.Equal(tsSvc.Tags, existingTSSvc.Tags) || + !ownersAreSetAndEqual(tsSvc, *existingTSSvc) || + !slices.Equal(tsSvc.Ports, existingTSSvc.Ports) { + logger.Infof("Ensuring Tailscale Service exists and is up to date") + if err = tsClient.VIPServices().CreateOrUpdate(ctx, tsSvc); err != nil { + return fmt.Errorf("error creating Tailscale Service: %w", err) + } + } + + // 3. Ensure that TLS Secret and RBAC exists. + dnsName, err = dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) + if err != nil { + return fmt.Errorf("error determining service DNS name: %w", err) + } + + if err = r.ensureCertResources(ctx, pg, dnsName); err != nil { + return fmt.Errorf("error ensuring cert resources: %w", err) + } + + // 4. Configure the Pods to advertise the Tailscale Service. + if err = r.maybeAdvertiseServices(ctx, pg, serviceName, logger); err != nil { + return fmt.Errorf("error updating advertised Tailscale Services: %w", err) + } + + // 5. Clean up any stale Tailscale Services from previous resource versions. + if err = r.maybeDeleteStaleServices(ctx, pg, logger, tsClient); err != nil { + return fmt.Errorf("failed to delete stale Tailscale Services: %w", err) + } + + return nil +} + +// maybeCleanup ensures that any resources, such as a Tailscale Service created for this Service, are cleaned up when the +// Service is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup. The Tailscale Service is only +// deleted if it does not contain any other owner references. If it does, the cleanup only removes the owner reference +// corresponding to this Service. +func (r *KubeAPIServerTSServiceReconciler) maybeCleanup(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, client tsclient.Client) (err error) { + ix := slices.Index(pg.Finalizers, proxyPGFinalizerName) + if ix < 0 { + logger.Debugf("no finalizer, nothing to do") + return nil + } + logger.Infof("Ensuring that Service %q is cleaned up", serviceName) + + defer func() { + if err == nil { + err = r.deleteFinalizer(ctx, pg, logger) + } + }() + + if _, err = cleanupTailscaleService(ctx, client, serviceName.String(), r.operatorID, logger); err != nil { + return fmt.Errorf("error deleting Tailscale Service: %w", err) + } + + if err = cleanupCertResources(ctx, r.Client, r.tsNamespace, serviceName, pg); err != nil { + return fmt.Errorf("failed to clean up cert resources: %w", err) + } + + return nil +} + +// maybeDeleteStaleServices deletes Services that have previously been created for +// this ProxyGroup but are no longer needed. +func (r *KubeAPIServerTSServiceReconciler) maybeDeleteStaleServices(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, tsClient tsclient.Client) error { + serviceName := serviceNameForAPIServerProxy(pg) + + svcs, err := tsClient.VIPServices().List(ctx) + if err != nil { + return fmt.Errorf("error listing Tailscale Services: %w", err) + } + + for _, svc := range svcs { + if svc.Name == serviceName.String() { + continue + } + + owners, err := parseOwnerAnnotation(&svc) + if err != nil { + logger.Warnf("error parsing owner annotation for Tailscale Service %s: %v", svc.Name, err) + continue + } + if owners == nil || len(owners.OwnerRefs) != 1 || owners.OwnerRefs[0].OperatorID != r.operatorID { + continue + } + + owner := owners.OwnerRefs[0] + if owner.Resource == nil || owner.Resource.Kind != "ProxyGroup" || owner.Resource.UID != string(pg.UID) { + continue + } + + logger.Infof("Deleting Tailscale Service %s", svc.Name) + if err = tsClient.VIPServices().Delete(ctx, svc.Name); err != nil && !tailscale.IsNotFound(err) { + return fmt.Errorf("error deleting Tailscale Service %s: %w", svc.Name, err) + } + + if err = cleanupCertResources(ctx, r.Client, r.tsNamespace, tailcfg.ServiceName(svc.Name), pg); err != nil { + return fmt.Errorf("failed to clean up cert resources: %w", err) + } + } + + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) deleteFinalizer(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error { + pg.Finalizers = slices.DeleteFunc(pg.Finalizers, func(f string) bool { + return f == proxyPGFinalizerName + }) + logger.Debugf("ensure %q finalizer is removed", proxyPGFinalizerName) + + if err := r.Update(ctx, pg); err != nil { + return fmt.Errorf("failed to remove finalizer %q: %w", proxyPGFinalizerName, err) + } + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string) error { + secret := certSecret(pg.Name, r.tsNamespace, domain, pg) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { + s.Labels = secret.Labels + }); err != nil { + return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) + } + role := certSecretRole(pg.Name, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + r.Labels = role.Labels + r.Rules = role.Rules + }); err != nil { + return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) + } + rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) { + rb.Labels = rolebinding.Labels + rb.Subjects = rolebinding.Subjects + rb.RoleRef = rolebinding.RoleRef + }); err != nil { + return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err) + } + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) maybeAdvertiseServices(ctx context.Context, pg *tsapi.ProxyGroup, serviceName tailcfg.ServiceName, logger *zap.SugaredLogger) error { + // Get all config Secrets for this ProxyGroup + cfgSecrets := &corev1.SecretList{} + if err := r.List(ctx, cfgSecrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { + return fmt.Errorf("failed to list config Secrets: %w", err) + } + + // Only advertise a Tailscale Service once the TLS certs required for + // serving it are available. + shouldBeAdvertised, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) + if err != nil { + return fmt.Errorf("error checking TLS credentials provisioned for Tailscale Service %q: %w", serviceName, err) + } + var advertiseServices []string + if shouldBeAdvertised { + advertiseServices = []string{serviceName.String()} + } + + for _, s := range cfgSecrets.Items { + if len(s.Data[kubetypes.KubeAPIServerConfigFile]) == 0 { + continue + } + + // Parse the existing config. + cfg, err := conf.Load(s.Data[kubetypes.KubeAPIServerConfigFile]) + if err != nil { + return fmt.Errorf("error loading config from Secret %q: %w", s.Name, err) + } + + if cfg.Parsed.APIServerProxy == nil { + return fmt.Errorf("config Secret %q does not contain APIServerProxy config", s.Name) + } + + existingCfgSecret := s.DeepCopy() + + var updated bool + if cfg.Parsed.APIServerProxy.ServiceName == nil || *cfg.Parsed.APIServerProxy.ServiceName != serviceName { + cfg.Parsed.APIServerProxy.ServiceName = &serviceName + updated = true + } + + // Update the services to advertise if required. + if !slices.Equal(cfg.Parsed.AdvertiseServices, advertiseServices) { + cfg.Parsed.AdvertiseServices = advertiseServices + updated = true + } + + if !updated { + continue + } + + // Update the config Secret. + cfgB, err := json.Marshal(conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &cfg.Parsed, + }) + if err != nil { + return err + } + + s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + if !apiequality.Semantic.DeepEqual(existingCfgSecret, s) { + logger.Debugf("Updating the Tailscale Services in ProxyGroup config Secret %s", s.Name) + if err := r.Update(ctx, &s); err != nil { + return err + } + } + } + + return nil +} + +func serviceNameForAPIServerProxy(pg *tsapi.ProxyGroup) tailcfg.ServiceName { + if pg.Spec.KubeAPIServer != nil && pg.Spec.KubeAPIServer.Hostname != "" { + return tailcfg.ServiceName("svc:" + pg.Spec.KubeAPIServer.Hostname) + } + + return tailcfg.ServiceName("svc:" + pg.Name) +} + +// exclusiveOwnerAnnotations returns the updated annotations required to ensure this +// instance of the operator is the exclusive owner. If the Tailscale Service is not +// nil, but does not contain an owner reference we return an error as this likely means +// that the Service was created by something other than a Tailscale Kubernetes operator. +// We also error if it is already owned by another operator instance, as we do not +// want to load balance a kube-apiserver ProxyGroup across multiple clusters. +func exclusiveOwnerAnnotations(pg *tsapi.ProxyGroup, operatorID string, svc *tailscale.VIPService) (map[string]string, error) { + ref := OwnerRef{ + OperatorID: operatorID, + Resource: &Resource{ + Kind: "ProxyGroup", + Name: pg.Name, + UID: string(pg.UID), + }, + } + if svc == nil { + c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} + data, err := json.Marshal(c) + if err != nil { + return nil, fmt.Errorf("failed to marshal Tailscale Service's owner annotation contents: %w", err) + } + + return map[string]string{ + ownerAnnotation: string(data), + }, nil + } + + o, err := parseOwnerAnnotation(svc) + if err != nil { + return nil, err + } + if o == nil || len(o.OwnerRefs) == 0 { + return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) + } + + if len(o.OwnerRefs) > 1 || o.OwnerRefs[0].OperatorID != operatorID { + return nil, fmt.Errorf("Tailscale Service %s is already owned by other operator(s) and cannot be shared across multiple clusters; configure a difference Service name to continue", svc.Name) + } + + if o.OwnerRefs[0].Resource == nil { + return nil, fmt.Errorf("Tailscale Service %s exists, but does not reference an owning resource; not proceeding as this is likely a Service already owned by an Ingress", svc.Name) + } + + if o.OwnerRefs[0].Resource.Kind != "ProxyGroup" || o.OwnerRefs[0].Resource.UID != string(pg.UID) { + return nil, fmt.Errorf("Tailscale Service %s is already owned by another resource: %#v; configure a difference Service name to continue", svc.Name, o.OwnerRefs[0].Resource) + } + + if o.OwnerRefs[0].Resource.Name != pg.Name { + // ProxyGroup name can be updated in place. + o.OwnerRefs[0].Resource.Name = pg.Name + } + + oBytes, err := json.Marshal(o) + if err != nil { + return nil, err + } + + newAnnots := make(map[string]string, len(svc.Annotations)+1) + maps.Copy(newAnnots, svc.Annotations) + newAnnots[ownerAnnotation] = string(oBytes) + + return newAnnots, nil +} diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go new file mode 100644 index 0000000000000..889ef064b05d9 --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -0,0 +1,388 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/client/tailscale/v2" + + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/types/opt" +) + +func TestAPIServerProxyReconciler(t *testing.T) { + const ( + pgName = "test-pg" + pgUID = "test-pg-uid" + ns = "operator-ns" + defaultDomain = "test-pg.ts.net" + ) + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgName, + Generation: 1, + UID: pgUID, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + }, + Status: tsapi.ProxyGroupStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ProxyGroupAvailable), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + }, + }, + }, + } + initialCfg := &conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: new("test-key"), + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + }, + }, + } + expectedCfg := *initialCfg + initialCfgB, err := json.Marshal(initialCfg) + if err != nil { + t.Fatalf("marshaling initial config: %v", err) + } + pgCfgSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pgName, 0), + Namespace: ns, + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), + }, + Data: map[string][]byte{ + // Existing config should be preserved. + kubetypes.KubeAPIServerConfigFile: initialCfgB, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, pgCfgSecret). + WithStatusSubresource(pg). + Build() + expectCfg := func(c *conf.VersionedConfig) { + t.Helper() + cBytes, err := json.Marshal(c) + if err != nil { + t.Fatalf("marshaling expected config: %v", err) + } + pgCfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cBytes + expectEqual(t, fc, pgCfgSecret) + } + + ft := &fakeTSClient{ + vipServices: make(map[string]tailscale.VIPService), + } + ingressTSSvc := tailscale.VIPService{ + Name: "svc:some-ingress-hostname", + Comment: managedTSServiceComment, + Annotations: map[string]string{ + // No resource field. + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, + }, + Ports: []string{"tcp:443"}, + Tags: []string{"tag:k8s"}, + Addrs: []string{"5.6.7.8"}, + } + ft.VIPServices().CreateOrUpdate(t.Context(), ingressTSSvc) + + r := &KubeAPIServerTSServiceReconciler{ + Client: fc, + clients: tsclient.NewProvider(ft), + defaultTags: []string{"tag:k8s"}, + tsNamespace: ns, + logger: zap.Must(zap.NewDevelopment()).Sugar(), + recorder: record.NewFakeRecorder(10), + clock: tstest.NewClock(tstest.ClockOpts{}), + operatorID: "self-id", + } + + // Create a Tailscale Service that will conflict with the initial config. + if err := ft.VIPServices().CreateOrUpdate(t.Context(), tailscale.VIPService{ + Name: "svc:" + pgName, + }); err != nil { + t.Fatalf("creating initial Tailscale Service: %v", err) + } + expectReconciled(t, r, "", pgName) + pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName} + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + expectMissing[corev1.Secret](t, fc, ns, defaultDomain) + expectMissing[rbacv1.Role](t, fc, ns, defaultDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain) + expectEqual(t, fc, pgCfgSecret) // Unchanged. + + // Delete Tailscale Service; should see Service created and valid condition updated to true. + if err := ft.VIPServices().Delete(t.Context(), "svc:"+pgName); err != nil { + t.Fatalf("deleting initial Tailscale Service: %v", err) + } + + // Create the state secret for the ProxyGroup without services being advertised. + mustCreate(t, fc, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg-0", + Namespace: ns, + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState), + }, + Data: map[string][]byte{ + "_current-profile": []byte("test"), + "test": []byte(`{"Config":{"NodeID":"node-foo", "UserProfile": {"LoginName": "test-pg.ts.net" }}}`), + }, + }) + + expectReconciled(t, r, "", pgName) + + tsSvc, err := ft.VIPServices().Get(t.Context(), "svc:"+pgName) + if err != nil { + t.Fatalf("getting Tailscale Service: %v", err) + } + if tsSvc == nil { + t.Fatalf("expected Tailscale Service to be created, but got nil") + } + expectedTSSvc := &tailscale.VIPService{ + Name: "svc:" + pgName, + Comment: managedTSServiceComment, + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"test-pg-uid"}}]}`, + }, + Ports: []string{"tcp:443"}, + Tags: []string{"tag:k8s"}, + Addrs: []string{"5.6.7.8"}, + } + if !reflect.DeepEqual(tsSvc, expectedTSSvc) { + t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc) + } + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + expectedCfg.APIServerProxy.ServiceName = new(tailcfg.ServiceName("svc:" + pgName)) + expectCfg(&expectedCfg) + + expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg)) + expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain)) + expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain)) + + // Simulate certs being issued; should observe AdvertiseServices config change. + populateTLSSecret(t, fc, pgName, defaultDomain) + expectReconciled(t, r, "", pgName) + + expectedCfg.AdvertiseServices = []string{"svc:" + pgName} + expectCfg(&expectedCfg) + + expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Unchanged status. + + // Simulate Pod prefs updated with advertised services; should see Configured condition updated to true. + mustUpdate(t, fc, ns, "test-pg-0", func(o *corev1.Secret) { + var p prefs + if err = json.Unmarshal(o.Data["test"], &p); err != nil { + t.Errorf("failed to unmarshal preferences: %v", err) + } + + p.AdvertiseServices = []string{"svc:test-pg"} + o.Data["test"], err = json.Marshal(p) + if err != nil { + t.Errorf("failed to marshal preferences: %v", err) + } + }) + + expectReconciled(t, r, "", pgName) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + pg.Status.URL = "https://" + defaultDomain + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Rename the Tailscale Service - old one + cert resources should be cleaned up. + updatedServiceName := tailcfg.ServiceName("svc:test-pg-renamed") + updatedDomain := "test-pg-renamed.ts.net" + pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{ + Hostname: updatedServiceName.WithoutPrefix(), + } + mustUpdate(t, fc, "", pgName, func(p *tsapi.ProxyGroup) { + p.Spec.KubeAPIServer = pg.Spec.KubeAPIServer + }) + expectReconciled(t, r, "", pgName) + _, err = ft.VIPServices().Get(t.Context(), "svc:"+pgName) + if !tailscale.IsNotFound(err) { + t.Fatalf("Expected 404, got: %v", err) + } + tsSvc, err = ft.VIPServices().Get(t.Context(), updatedServiceName.String()) + if err != nil { + t.Fatalf("Expected renamed svc, got error: %v", err) + } + expectedTSSvc.Name = updatedServiceName.String() + if !reflect.DeepEqual(tsSvc, expectedTSSvc) { + t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc) + } + // Check cfg and status reset until TLS certs are available again. + expectedCfg.APIServerProxy.ServiceName = new(updatedServiceName) + expectedCfg.AdvertiseServices = nil + expectCfg(&expectedCfg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + pg.Status.URL = "" + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg)) + expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain)) + expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain)) + expectMissing[corev1.Secret](t, fc, ns, defaultDomain) + expectMissing[rbacv1.Role](t, fc, ns, defaultDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain) + + // Check we get the new hostname in the status once ready. + populateTLSSecret(t, fc, pgName, updatedDomain) + mustUpdate(t, fc, "operator-ns", "test-pg-0", func(s *corev1.Secret) { + s.Data["profile-foo"] = []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`) + }) + expectReconciled(t, r, "", pgName) + expectedCfg.AdvertiseServices = []string{updatedServiceName.String()} + expectCfg(&expectedCfg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + pg.Status.URL = "https://" + updatedDomain + + // Delete the ProxyGroup and verify Tailscale Service and cert resources are cleaned up. + if err := fc.Delete(t.Context(), pg); err != nil { + t.Fatalf("deleting ProxyGroup: %v", err) + } + expectReconciled(t, r, "", pgName) + expectMissing[corev1.Secret](t, fc, ns, updatedDomain) + expectMissing[rbacv1.Role](t, fc, ns, updatedDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, updatedDomain) + _, err = ft.VIPServices().Get(t.Context(), updatedServiceName.String()) + if !tailscale.IsNotFound(err) { + t.Fatalf("Expected 404, got: %v", err) + } + + // Ingress Tailscale Service should not be affected. + svc, err := ft.VIPServices().Get(t.Context(), ingressTSSvc.Name) + if err != nil { + t.Fatalf("getting ingress Tailscale Service: %v", err) + } + if !reflect.DeepEqual(svc, &ingressTSSvc) { + t.Fatalf("expected ingress Tailscale Service to be unmodified %+v, got %+v", ingressTSSvc, svc) + } +} + +func TestExclusiveOwnerAnnotations(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + UID: "pg1-uid", + }, + } + const ( + pg1Owner = `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg1","uid":"pg1-uid"}}]}` + ) + + for name, tc := range map[string]struct { + svc *tailscale.VIPService + wantErr string + }{ + "no_svc": { + svc: nil, + }, + "empty_svc": { + svc: &tailscale.VIPService{}, + wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator", + }, + "already_owner": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: pg1Owner, + }, + }, + }, + "already_owner_name_updated": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"old-pg1-name","uid":"pg1-uid"}}]}`, + }, + }, + }, + "preserves_existing_annotations": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + "existing": "annotation", + ownerAnnotation: pg1Owner, + }, + }, + }, + "owned_by_another_operator": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`, + }, + }, + wantErr: "already owned by other operator(s)", + }, + "owned_by_an_ingress": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, // Ingress doesn't set Resource field (yet). + }, + }, + wantErr: "does not reference an owning resource", + }, + "owned_by_another_pg": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg2","uid":"pg2-uid"}}]}`, + }, + }, + wantErr: "already owned by another resource", + }, + } { + t.Run(name, func(t *testing.T) { + got, err := exclusiveOwnerAnnotations(pg, "self-id", tc.svc) + if tc.wantErr != "" { + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("exclusiveOwnerAnnotations() error = %v, wantErr %v", err, tc.wantErr) + } + } else if diff := cmp.Diff(pg1Owner, got[ownerAnnotation]); diff != "" { + t.Errorf("exclusiveOwnerAnnotations() mismatch (-want +got):\n%s", diff) + } + if tc.svc == nil { + return // Don't check annotations being preserved. + } + for k, v := range tc.svc.Annotations { + if k == ownerAnnotation { + continue + } + if got[k] != v { + t.Errorf("exclusiveOwnerAnnotations() did not preserve annotation %q: got %q, want %q", k, got[k], v) + } + } + }) + } +} + +func omitPGStatusConditionMessages(p *tsapi.ProxyGroup) { + for i := range p.Status.Conditions { + // Don't bother validating the message. + p.Status.Conditions[i].Message = "" + } +} diff --git a/cmd/k8s-operator/api-server-proxy.go b/cmd/k8s-operator/api-server-proxy.go new file mode 100644 index 0000000000000..b8d87cf0aa38a --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy.go @@ -0,0 +1,42 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "fmt" + "log" + "os" + + "tailscale.com/kube/kubetypes" +) + +func parseAPIProxyMode() *kubetypes.APIServerProxyMode { + haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" + haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" + switch { + case haveAPIProxyEnv && haveAuthProxyEnv: + log.Fatal("AUTH_PROXY (deprecated) and APISERVER_PROXY are mutually exclusive, please unset AUTH_PROXY") + case haveAuthProxyEnv: + var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated + if authProxyEnv { + return new(kubetypes.APIServerProxyModeAuth) + } + return nil + case haveAPIProxyEnv: + var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" + switch apiProxyEnv { + case "true": + return new(kubetypes.APIServerProxyModeAuth) + case "false", "": + return nil + case "noauth": + return new(kubetypes.APIServerProxyModeNoAuth) + default: + panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) + } + } + return nil +} diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index c243036cbabd9..0c2d32482e78b 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,6 +7,7 @@ package main import ( "context" + "errors" "fmt" "net/netip" "slices" @@ -14,8 +15,6 @@ import ( "sync" "time" - "errors" - "go.uber.org/zap" xslices "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" @@ -26,6 +25,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -176,6 +176,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge if cn.Spec.Hostname != "" { hostname = string(cn.Spec.Hostname) } + crl := childResourceLabels(cn.Name, a.tsnamespace, "connector") proxyClass := cn.Spec.ProxyClass @@ -188,10 +189,17 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge } } + var replicas int32 = 1 + if cn.Spec.Replicas != nil { + replicas = *cn.Spec.Replicas + } + sts := &tailscaleSTSConfig{ + Replicas: replicas, ParentResourceName: cn.Name, ParentResourceUID: string(cn.UID), Hostname: hostname, + HostnamePrefix: string(cn.Spec.HostnamePrefix), ChildResourceLabels: crl, Tags: cn.Spec.Tags.Stringify(), Connector: &connector{ @@ -199,6 +207,8 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge }, ProxyClassName: proxyClass, proxyType: proxyTypeConnector, + LoginServer: a.ssr.loginServer, + Tailnet: cn.Spec.Tailnet, } if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 { @@ -218,16 +228,19 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge } else { a.exitNodes.Remove(cn.UID) } + if cn.Spec.SubnetRouter != nil { a.subnetRouters.Add(cn.GetUID()) } else { a.subnetRouters.Remove(cn.GetUID()) } + if cn.Spec.AppConnector != nil { a.appConnectors.Add(cn.GetUID()) } else { a.appConnectors.Remove(cn.GetUID()) } + a.mu.Unlock() gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len())) gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len())) @@ -243,27 +256,29 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge return err } - dev, err := a.ssr.DeviceInfo(ctx, crl, logger) + devices, err := a.ssr.DeviceInfo(ctx, crl, logger) if err != nil { return err } - if dev == nil || dev.hostname == "" { - logger.Debugf("no Tailscale hostname known yet, waiting for Connector Pod to finish auth") - // No hostname yet. Wait for the connector pod to auth. - cn.Status.TailnetIPs = nil - cn.Status.Hostname = "" - return nil + cn.Status.Devices = make([]tsapi.ConnectorDevice, len(devices)) + for i, dev := range devices { + cn.Status.Devices[i] = tsapi.ConnectorDevice{ + Hostname: dev.hostname, + TailnetIPs: dev.ips, + } } - cn.Status.TailnetIPs = dev.ips - cn.Status.Hostname = dev.hostname + if len(cn.Status.Devices) > 0 { + cn.Status.Hostname = cn.Status.Devices[0].Hostname + cn.Status.TailnetIPs = cn.Status.Devices[0].TailnetIPs + } return nil } func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) { - if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil { + if done, err := a.ssr.Cleanup(ctx, cn.Spec.Tailnet, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil { return false, fmt.Errorf("failed to cleanup Connector resources: %w", err) } else if !done { logger.Debugf("Connector cleanup not done yet, waiting for next reconcile") @@ -301,6 +316,15 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error { if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil { return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node") } + + // These two checks should be caught by the Connector schema validation. + if cn.Spec.Replicas != nil && *cn.Spec.Replicas > 1 && cn.Spec.Hostname != "" { + return errors.New("invalid spec: a Connector that is configured with multiple replicas cannot specify a hostname. Instead, use a hostnamePrefix") + } + if cn.Spec.HostnamePrefix != "" && cn.Spec.Hostname != "" { + return errors.New("invalid spec: a Connect cannot use both a hostname and hostname prefix") + } + if cn.Spec.AppConnector != nil { return validateAppConnector(cn.Spec.AppConnector) } diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index f32fe3282020c..69e8e287d07d1 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,6 +7,8 @@ package main import ( "context" + "strconv" + "strings" "testing" "time" @@ -17,7 +19,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/util/mak" @@ -36,6 +40,7 @@ func TestConnector(t *testing.T) { APIVersion: "tailscale.com/v1alpha1", }, Spec: tsapi.ConnectorSpec{ + Replicas: new(int32(1)), SubnetRouter: &tsapi.SubnetRouter{ AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"}, }, @@ -55,10 +60,11 @@ func TestConnector(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) cr := &ConnectorReconciler{ - Client: fc, + Client: fc, + recorder: record.NewFakeRecorder(10), ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -78,9 +84,10 @@ func TestConnector(t *testing.T) { isExitNode: true, subnetRoutes: "10.40.0.0/14", app: kubetypes.AppConnector, + replicas: cn.Spec.Replicas, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Connector status should get updated with the IP/hostname info when available. const hostname = "foo.tailnetxyz.ts.net" @@ -94,6 +101,10 @@ func TestConnector(t *testing.T) { cn.Status.IsExitNode = cn.Spec.ExitNode cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify() cn.Status.Hostname = hostname + cn.Status.Devices = []tsapi.ConnectorDevice{{ + Hostname: hostname, + TailnetIPs: []string{"127.0.0.1", "::1"}, + }} cn.Status.TailnetIPs = []string{"127.0.0.1", "::1"} expectEqual(t, fc, cn, func(o *tsapi.Connector) { o.Status.Conditions = nil @@ -106,7 +117,7 @@ func TestConnector(t *testing.T) { opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Remove a route. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -114,7 +125,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Remove the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -122,7 +133,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Re-add the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -132,7 +143,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -156,6 +167,7 @@ func TestConnector(t *testing.T) { APIVersion: "tailscale.io/v1alpha1", }, Spec: tsapi.ConnectorSpec{ + Replicas: new(int32(1)), SubnetRouter: &tsapi.SubnetRouter{ AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"}, }, @@ -174,9 +186,10 @@ func TestConnector(t *testing.T) { subnetRoutes: "10.40.0.0/14", hostname: "test-connector", app: kubetypes.AppConnector, + replicas: cn.Spec.Replicas, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Add an exit node. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -184,7 +197,7 @@ func TestConnector(t *testing.T) { }) opts.isExitNode = true expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -217,9 +230,11 @@ func TestConnectorWithProxyClass(t *testing.T) { APIVersion: "tailscale.io/v1alpha1", }, Spec: tsapi.ConnectorSpec{ + Replicas: new(int32(1)), SubnetRouter: &tsapi.SubnetRouter{ AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"}, }, + ExitNode: true, }, } @@ -239,7 +254,7 @@ func TestConnectorWithProxyClass(t *testing.T) { clock: cl, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -260,9 +275,10 @@ func TestConnectorWithProxyClass(t *testing.T) { isExitNode: true, subnetRoutes: "10.40.0.0/14", app: kubetypes.AppConnector, + replicas: cn.Spec.Replicas, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Update Connector to specify a ProxyClass. ProxyClass is not yet // ready, so its configuration is NOT applied to the Connector @@ -271,7 +287,7 @@ func TestConnectorWithProxyClass(t *testing.T) { conn.Spec.ProxyClass = "custom-metadata" }) expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 3. ProxyClass is set to Ready by proxy-class reconciler. Connector // get reconciled and configuration from the ProxyClass is applied to @@ -286,7 +302,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = pc.Name expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 4. Connector.spec.proxyClass field is unset, Connector gets // reconciled and configuration from the ProxyClass is removed from the @@ -296,7 +312,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func TestConnectorWithAppConnector(t *testing.T) { @@ -311,6 +327,7 @@ func TestConnectorWithAppConnector(t *testing.T) { APIVersion: "tailscale.io/v1alpha1", }, Spec: tsapi.ConnectorSpec{ + Replicas: new(int32(1)), AppConnector: &tsapi.AppConnector{}, }, } @@ -331,7 +348,7 @@ func TestConnectorWithAppConnector(t *testing.T) { clock: cl, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -340,7 +357,7 @@ func TestConnectorWithAppConnector(t *testing.T) { recorder: fr, } - // 1. Connector with app connnector is created and becomes ready + // 1. Connector with app connector is created and becomes ready expectReconciled(t, cr, "", "test") fullName, shortName := findGenName(t, fc, "", "test", "connector") opts := configOpts{ @@ -350,13 +367,15 @@ func TestConnectorWithAppConnector(t *testing.T) { hostname: "test-connector", app: kubetypes.AppConnector, isAppConnector: true, + replicas: cn.Spec.Replicas, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Connector's ready condition should be set to true cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer") cn.Status.IsAppConnector = true + cn.Status.Devices = []tsapi.ConnectorDevice{} cn.Status.Conditions = []metav1.Condition{{ Type: string(tsapi.ConnectorReady), Status: metav1.ConditionTrue, @@ -368,9 +387,9 @@ func TestConnectorWithAppConnector(t *testing.T) { // 2. Connector with invalid app connector routes has status set to invalid mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { - conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")} + conn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"} }) - cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")} + cn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"} expectReconciled(t, cr, "", "test") cn.Status.Conditions = []metav1.Condition{{ Type: string(tsapi.ConnectorReady), @@ -383,9 +402,9 @@ func TestConnectorWithAppConnector(t *testing.T) { // 3. Connector with valid app connnector routes becomes ready mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { - conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")} + conn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"} }) - cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")} + cn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"} cn.Status.Conditions = []metav1.Condition{{ Type: string(tsapi.ConnectorReady), Status: metav1.ConditionTrue, @@ -395,3 +414,94 @@ func TestConnectorWithAppConnector(t *testing.T) { }} expectReconciled(t, cr, "", "test") } + +func TestConnectorWithMultipleReplicas(t *testing.T) { + cn := &tsapi.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("1234-UID"), + }, + TypeMeta: metav1.TypeMeta{ + Kind: tsapi.ConnectorKind, + APIVersion: "tailscale.io/v1alpha1", + }, + Spec: tsapi.ConnectorSpec{ + Replicas: new(int32(3)), + AppConnector: &tsapi.AppConnector{}, + HostnamePrefix: "test-connector", + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(cn). + WithStatusSubresource(cn). + Build() + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + cl := tstest.NewClock(tstest.ClockOpts{}) + fr := record.NewFakeRecorder(1) + cr := &ConnectorReconciler{ + Client: fc, + clock: cl, + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(ft), + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + recorder: fr, + } + + // 1. Ensure that our connector resource is reconciled. + expectReconciled(t, cr, "", "test") + + // 2. Ensure we have a number of secrets matching the number of replicas. + names := findGenNames(t, fc, "", "test", "connector") + if int32(len(names)) != *cn.Spec.Replicas { + t.Fatalf("expected %d secrets, got %d", *cn.Spec.Replicas, len(names)) + } + + // 3. Ensure each device has the correct hostname prefix and ordinal suffix. + for i, name := range names { + expected := expectedSecret(t, fc, configOpts{ + secretName: name, + hostname: string(cn.Spec.HostnamePrefix) + "-" + strconv.Itoa(i), + isAppConnector: true, + parentType: "connector", + namespace: cr.tsnamespace, + }) + + expectEqual(t, fc, expected) + } + + // 4. Ensure the generated stateful set has the matching number of replicas + shortName := strings.TrimSuffix(names[0], "-0") + + var sts appsv1.StatefulSet + if err = fc.Get(t.Context(), types.NamespacedName{Namespace: "operator-ns", Name: shortName}, &sts); err != nil { + t.Fatalf("failed to get StatefulSet %q: %v", shortName, err) + } + + if sts.Spec.Replicas == nil { + t.Fatalf("actual StatefulSet %q does not have replicas set", shortName) + } + + if *sts.Spec.Replicas != *cn.Spec.Replicas { + t.Fatalf("expected %d replicas, got %d", *cn.Spec.Replicas, *sts.Spec.Replicas) + } + + // 5. We'll scale the connector down by 1 replica and make sure its secret is cleaned up + mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.Replicas = new(int32(2)) + }) + expectReconciled(t, cr, "", "test") + names = findGenNames(t, fc, "", "test", "connector") + if len(names) != 2 { + t.Fatalf("expected 2 secrets, got %d", len(names)) + } +} diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 36c5184c3b44a..a1c9371f4c617 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -1,120 +1,50 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/depaware) + đŸ’Ŗ crypto/internal/entropy/v1.0.0 from crypto/internal/fips140/drbg filippo.io/edwards25519 from github.com/hdevalence/ed25519consensus filippo.io/edwards25519/field from filippo.io/edwards25519 W đŸ’Ŗ github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+ W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate W đŸ’Ŗ github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy - L github.com/aws/aws-sdk-go-v2/aws from github.com/aws/aws-sdk-go-v2/aws/defaults+ - L github.com/aws/aws-sdk-go-v2/aws/arn from tailscale.com/ipn/store/awsstore - L github.com/aws/aws-sdk-go-v2/aws/defaults from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/aws-sdk-go-v2/aws/middleware from github.com/aws/aws-sdk-go-v2/aws/retry+ - L github.com/aws/aws-sdk-go-v2/aws/protocol/query from github.com/aws/aws-sdk-go-v2/service/sts - L github.com/aws/aws-sdk-go-v2/aws/protocol/restjson from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/aws-sdk-go-v2/aws/protocol/xml from github.com/aws/aws-sdk-go-v2/service/sts - L github.com/aws/aws-sdk-go-v2/aws/ratelimit from github.com/aws/aws-sdk-go-v2/aws/retry - L github.com/aws/aws-sdk-go-v2/aws/retry from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client+ - L github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 from github.com/aws/aws-sdk-go-v2/aws/signer/v4 - L github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ - L github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/config from tailscale.com/ipn/store/awsstore - L github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds - L github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/feature/ec2/imds from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds - L github.com/aws/aws-sdk-go-v2/internal/auth from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ - L github.com/aws/aws-sdk-go-v2/internal/auth/smithy from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/aws-sdk-go-v2/internal/configsources from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/aws-sdk-go-v2/internal/context from github.com/aws/aws-sdk-go-v2/aws/retry+ - L github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+ - L github.com/aws/aws-sdk-go-v2/internal/ini from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/internal/middleware from github.com/aws/aws-sdk-go-v2/service/sso+ - L github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ - L github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ - L github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds - L github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 - L github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws - L github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry - L github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts - L github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts - L github.com/aws/aws-sdk-go-v2/service/ssm from tailscale.com/ipn/store/awsstore - L github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm - L github.com/aws/aws-sdk-go-v2/service/ssm/types from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/aws-sdk-go-v2/service/sso from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso - L github.com/aws/aws-sdk-go-v2/service/sso/types from github.com/aws/aws-sdk-go-v2/service/sso - L github.com/aws/aws-sdk-go-v2/service/ssooidc from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssooidc - L github.com/aws/aws-sdk-go-v2/service/ssooidc/types from github.com/aws/aws-sdk-go-v2/service/ssooidc - L github.com/aws/aws-sdk-go-v2/service/sts from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sts - L github.com/aws/aws-sdk-go-v2/service/sts/types from github.com/aws/aws-sdk-go-v2/credentials/stscreds+ - L github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ - L github.com/aws/smithy-go/auth from github.com/aws/aws-sdk-go-v2/internal/auth+ - L github.com/aws/smithy-go/auth/bearer from github.com/aws/aws-sdk-go-v2/aws+ - L github.com/aws/smithy-go/context from github.com/aws/smithy-go/auth/bearer - L github.com/aws/smithy-go/document from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/smithy-go/encoding from github.com/aws/smithy-go/encoding/json+ - L github.com/aws/smithy-go/encoding/httpbinding from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ - L github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts - L github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer - L github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ - L github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ - L github.com/aws/smithy-go/metrics from github.com/aws/aws-sdk-go-v2/aws/retry+ - L github.com/aws/smithy-go/middleware from github.com/aws/aws-sdk-go-v2/aws+ - L github.com/aws/smithy-go/private/requestcompression from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/smithy-go/ptr from github.com/aws/aws-sdk-go-v2/aws+ - L github.com/aws/smithy-go/rand from github.com/aws/aws-sdk-go-v2/aws/middleware+ - L github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/ssm+ - L github.com/aws/smithy-go/tracing from github.com/aws/aws-sdk-go-v2/aws/middleware+ - L github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws/middleware+ - L github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http - L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm github.com/beorn7/perks/quantile from github.com/prometheus/client_golang/prometheus - đŸ’Ŗ github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus + github.com/blang/semver/v4 from k8s.io/component-base/metrics + đŸ’Ŗ github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus+ github.com/coder/websocket from tailscale.com/util/eventbus github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket - L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw + github.com/creachadair/msync/trigger from tailscale.com/logtail đŸ’Ŗ github.com/davecgh/go-spew/spew from k8s.io/apimachinery/pkg/util/dump - W đŸ’Ŗ github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/com+ + W đŸ’Ŗ github.com/dblohm7/wingoes from tailscale.com/net/tshttpproxy+ W đŸ’Ŗ github.com/dblohm7/wingoes/com from tailscale.com/util/osdiag+ W đŸ’Ŗ github.com/dblohm7/wingoes/com/automation from tailscale.com/util/osdiag/internal/wsc W github.com/dblohm7/wingoes/internal from github.com/dblohm7/wingoes/com W đŸ’Ŗ github.com/dblohm7/wingoes/pe from tailscale.com/util/osdiag+ - LW đŸ’Ŗ github.com/digitalocean/go-smbios/smbios from tailscale.com/posture github.com/distribution/reference from tailscale.com/cmd/k8s-operator github.com/emicklei/go-restful/v3 from k8s.io/kube-openapi/pkg/common github.com/emicklei/go-restful/v3/log from github.com/emicklei/go-restful/v3 github.com/evanphx/json-patch/v5 from sigs.k8s.io/controller-runtime/pkg/client github.com/evanphx/json-patch/v5/internal/json from github.com/evanphx/json-patch/v5 đŸ’Ŗ github.com/fsnotify/fsnotify from sigs.k8s.io/controller-runtime/pkg/certwatcher + github.com/fsnotify/fsnotify/internal from github.com/fsnotify/fsnotify github.com/fxamacker/cbor/v2 from tailscale.com/tka+ github.com/gaissmai/bart from tailscale.com/net/ipset+ + github.com/gaissmai/bart/internal/allot from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/art from github.com/gaissmai/bart+ github.com/gaissmai/bart/internal/bitset from github.com/gaissmai/bart+ - github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart + github.com/gaissmai/bart/internal/lpm from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/nodes from github.com/gaissmai/bart + github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/value from github.com/gaissmai/bart+ github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json/internal/jsonflags+ github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json/internal/jsonopts+ github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/jsontext from tailscale.com/logtail+ + github.com/go-json-experiment/json/v1 from tailscale.com/net/routecheck github.com/go-logr/logr from github.com/go-logr/logr/slogr+ github.com/go-logr/logr/slogr from github.com/go-logr/zapr github.com/go-logr/zapr from sigs.k8s.io/controller-runtime/pkg/log/zap+ - W đŸ’Ŗ github.com/go-ole/go-ole from github.com/go-ole/go-ole/oleutil+ - W đŸ’Ŗ github.com/go-ole/go-ole/oleutil from tailscale.com/wgengine/winnet github.com/go-openapi/jsonpointer from github.com/go-openapi/jsonreference github.com/go-openapi/jsonreference from k8s.io/kube-openapi/pkg/internal+ github.com/go-openapi/jsonreference/internal from github.com/go-openapi/jsonreference @@ -123,87 +53,69 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ đŸ’Ŗ github.com/gogo/protobuf/proto from k8s.io/api/admission/v1+ github.com/gogo/protobuf/sortkeys from k8s.io/api/admission/v1+ github.com/golang/groupcache/lru from tailscale.com/net/dnscache - github.com/golang/protobuf/proto from k8s.io/client-go/discovery+ - github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+ + github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/transport/tcp github.com/google/gnostic-models/compiler from github.com/google/gnostic-models/openapiv2+ github.com/google/gnostic-models/extensions from github.com/google/gnostic-models/compiler github.com/google/gnostic-models/jsonschema from github.com/google/gnostic-models/compiler github.com/google/gnostic-models/openapiv2 from k8s.io/client-go/discovery+ github.com/google/gnostic-models/openapiv3 from k8s.io/kube-openapi/pkg/handler3+ - đŸ’Ŗ github.com/google/go-cmp/cmp from k8s.io/apimachinery/pkg/util/diff+ - github.com/google/go-cmp/cmp/internal/diff from github.com/google/go-cmp/cmp - github.com/google/go-cmp/cmp/internal/flags from github.com/google/go-cmp/cmp+ - github.com/google/go-cmp/cmp/internal/function from github.com/google/go-cmp/cmp - đŸ’Ŗ github.com/google/go-cmp/cmp/internal/value from github.com/google/go-cmp/cmp - github.com/google/gofuzz from k8s.io/apimachinery/pkg/apis/meta/v1+ - github.com/google/gofuzz/bytesource from github.com/google/gofuzz - L github.com/google/nftables from tailscale.com/util/linuxfw - L đŸ’Ŗ github.com/google/nftables/alignedbuff from github.com/google/nftables/xt - L đŸ’Ŗ github.com/google/nftables/binaryutil from github.com/google/nftables+ - L github.com/google/nftables/expr from github.com/google/nftables+ - L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ - L github.com/google/nftables/xt from github.com/google/nftables/expr+ - github.com/google/uuid from github.com/prometheus-community/pro-bing+ - github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ - L đŸ’Ŗ github.com/illarion/gonotify/v3 from tailscale.com/net/dns - L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 - L github.com/jmespath/go-jmespath from github.com/aws/aws-sdk-go-v2/service/ssm + github.com/google/uuid from k8s.io/apimachinery/pkg/util/uuid+ + github.com/hdevalence/ed25519consensus from tailscale.com/tka + github.com/huin/goupnp from github.com/huin/goupnp/dcps/internetgateway2+ + github.com/huin/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper + github.com/huin/goupnp/httpu from github.com/huin/goupnp+ + github.com/huin/goupnp/scpd from github.com/huin/goupnp + github.com/huin/goupnp/soap from github.com/huin/goupnp+ + github.com/huin/goupnp/ssdp from github.com/huin/goupnp github.com/josharian/intern from github.com/mailru/easyjson/jlexer L đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink - đŸ’Ŗ github.com/json-iterator/go from sigs.k8s.io/structured-merge-diff/v4/fieldpath+ + đŸ’Ŗ github.com/json-iterator/go from sigs.k8s.io/structured-merge-diff/v6/fieldpath+ github.com/klauspost/compress from github.com/klauspost/compress/zstd github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0 github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ + đŸ’Ŗ github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+ github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd github.com/mailru/easyjson/buffer from github.com/mailru/easyjson/jwriter đŸ’Ŗ github.com/mailru/easyjson/jlexer from github.com/go-openapi/swag github.com/mailru/easyjson/jwriter from github.com/go-openapi/swag - L github.com/mdlayher/genetlink from tailscale.com/net/tstun - L đŸ’Ŗ github.com/mdlayher/netlink from github.com/google/nftables+ + L đŸ’Ŗ github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+ L đŸ’Ŗ github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ - L github.com/mdlayher/netlink/nltest from github.com/google/nftables - L github.com/mdlayher/sdnotify from tailscale.com/util/systemd L đŸ’Ŗ github.com/mdlayher/socket from github.com/mdlayher/netlink+ - github.com/miekg/dns from tailscale.com/net/dns/recursive đŸ’Ŗ github.com/mitchellh/go-ps from tailscale.com/safesocket github.com/modern-go/concurrent from github.com/json-iterator/go đŸ’Ŗ github.com/modern-go/reflect2 from github.com/json-iterator/go github.com/munnerz/goautoneg from k8s.io/kube-openapi/pkg/handler3+ github.com/opencontainers/go-digest from github.com/distribution/reference + github.com/pires/go-proxyproto from tailscale.com/ipn/ipnlocal+ github.com/pkg/errors from github.com/evanphx/json-patch/v5+ - D github.com/prometheus-community/pro-bing from tailscale.com/wgengine/netstack + github.com/pmezard/go-difflib/difflib from k8s.io/apimachinery/pkg/util/diff + github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil from github.com/prometheus/client_golang/prometheus/promhttp + github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil/header from github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil đŸ’Ŗ github.com/prometheus/client_golang/prometheus from github.com/prometheus/client_golang/prometheus/collectors+ - github.com/prometheus/client_golang/prometheus/collectors from sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics + github.com/prometheus/client_golang/prometheus/collectors from sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics+ github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/client_golang/prometheus/promhttp from sigs.k8s.io/controller-runtime/pkg/metrics/server+ + github.com/prometheus/client_golang/prometheus/promhttp/internal from github.com/prometheus/client_golang/prometheus/promhttp github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+ - LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus + LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus+ LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs LD github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs - L đŸ’Ŗ github.com/safchain/ethtool from tailscale.com/doctor/ethtool+ - github.com/spf13/pflag from k8s.io/client-go/tools/clientcmd - W đŸ’Ŗ github.com/tailscale/certstore from tailscale.com/control/controlclient + L đŸ’Ŗ github.com/safchain/ethtool from tailscale.com/net/netkernelconf + github.com/spf13/pflag from k8s.io/client-go/tools/clientcmd+ + DW đŸ’Ŗ github.com/tailscale/certstore from tailscale.com/control/controlclient W đŸ’Ŗ github.com/tailscale/go-winio from tailscale.com/safesocket W đŸ’Ŗ github.com/tailscale/go-winio/internal/fs from github.com/tailscale/go-winio W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ - github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper - github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ - github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp - github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ - github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp - github.com/tailscale/hujson from tailscale.com/ipn/conffile - L đŸ’Ŗ github.com/tailscale/netlink from tailscale.com/net/routetable+ - L đŸ’Ŗ github.com/tailscale/netlink/nl from github.com/tailscale/netlink - github.com/tailscale/peercred from tailscale.com/ipn/ipnauth + github.com/tailscale/hujson from tailscale.com/ipn/conffile+ + LD github.com/tailscale/peercred from tailscale.com/ipn/ipnauth github.com/tailscale/web-client-prebuilt from tailscale.com/client/web đŸ’Ŗ github.com/tailscale/wireguard-go/conn from github.com/tailscale/wireguard-go/device+ W đŸ’Ŗ github.com/tailscale/wireguard-go/conn/winrio from github.com/tailscale/wireguard-go/conn @@ -215,8 +127,15 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/tailscale/wireguard-go/rwcancel from github.com/tailscale/wireguard-go/device+ github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device đŸ’Ŗ github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+ - L github.com/vishvananda/netns from github.com/tailscale/netlink+ github.com/x448/float16 from github.com/fxamacker/cbor/v2 + go.opentelemetry.io/otel/attribute from go.opentelemetry.io/otel/trace+ + go.opentelemetry.io/otel/attribute/internal from go.opentelemetry.io/otel/attribute + go.opentelemetry.io/otel/attribute/internal/xxhash from go.opentelemetry.io/otel/attribute + go.opentelemetry.io/otel/codes from go.opentelemetry.io/otel/trace + go.opentelemetry.io/otel/semconv/v1.37.0 from go.opentelemetry.io/otel/trace + go.opentelemetry.io/otel/trace from k8s.io/component-base/metrics + go.opentelemetry.io/otel/trace/embedded from go.opentelemetry.io/otel/trace + đŸ’Ŗ go.opentelemetry.io/otel/trace/internal/telemetry from go.opentelemetry.io/otel/trace go.uber.org/multierr from go.uber.org/zap+ go.uber.org/zap from github.com/go-logr/zapr+ go.uber.org/zap/buffer from go.uber.org/zap/internal/bufferpool+ @@ -227,19 +146,20 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ go.uber.org/zap/internal/pool from go.uber.org/zap+ go.uber.org/zap/internal/stacktrace from go.uber.org/zap go.uber.org/zap/zapcore from github.com/go-logr/zapr+ + go.yaml.in/yaml/v2 from k8s.io/kube-openapi/pkg/util/proto+ + go.yaml.in/yaml/v3 from github.com/google/gnostic-models/compiler+ đŸ’Ŗ go4.org/mem from tailscale.com/client/local+ go4.org/netipx from tailscale.com/ipn/ipnlocal+ W đŸ’Ŗ golang.zx2c4.com/wintun from github.com/tailscale/wireguard-go/tun W đŸ’Ŗ golang.zx2c4.com/wireguard/windows/tunnel/winipcfg from tailscale.com/net/dns+ gomodules.xyz/jsonpatch/v2 from sigs.k8s.io/controller-runtime/pkg/webhook+ google.golang.org/protobuf/encoding/protodelim from github.com/prometheus/common/expfmt - google.golang.org/protobuf/encoding/prototext from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/encoding/protowire from github.com/golang/protobuf/proto+ + google.golang.org/protobuf/encoding/prototext from github.com/prometheus/common/expfmt+ + google.golang.org/protobuf/encoding/protowire from google.golang.org/protobuf/encoding/protodelim+ google.golang.org/protobuf/internal/descfmt from google.golang.org/protobuf/internal/filedesc google.golang.org/protobuf/internal/descopts from google.golang.org/protobuf/internal/filedesc+ google.golang.org/protobuf/internal/detrand from google.golang.org/protobuf/internal/descfmt+ - google.golang.org/protobuf/internal/editiondefaults from google.golang.org/protobuf/internal/filedesc+ - google.golang.org/protobuf/internal/editionssupport from google.golang.org/protobuf/reflect/protodesc + google.golang.org/protobuf/internal/editiondefaults from google.golang.org/protobuf/internal/filedesc google.golang.org/protobuf/internal/encoding/defval from google.golang.org/protobuf/internal/encoding/tag+ google.golang.org/protobuf/internal/encoding/messageset from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/encoding/tag from google.golang.org/protobuf/internal/impl @@ -252,22 +172,21 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ đŸ’Ŗ google.golang.org/protobuf/internal/impl from google.golang.org/protobuf/internal/filetype+ google.golang.org/protobuf/internal/order from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/pragma from google.golang.org/protobuf/encoding/prototext+ + đŸ’Ŗ google.golang.org/protobuf/internal/protolazy from google.golang.org/protobuf/internal/impl+ google.golang.org/protobuf/internal/set from google.golang.org/protobuf/encoding/prototext đŸ’Ŗ google.golang.org/protobuf/internal/strs from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/version from google.golang.org/protobuf/runtime/protoimpl - google.golang.org/protobuf/proto from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/reflect/protodesc from github.com/golang/protobuf/proto - đŸ’Ŗ google.golang.org/protobuf/reflect/protoreflect from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/reflect/protoregistry from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/runtime/protoiface from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/runtime/protoimpl from github.com/golang/protobuf/proto+ - google.golang.org/protobuf/types/descriptorpb from github.com/google/gnostic-models/openapiv3+ - google.golang.org/protobuf/types/gofeaturespb from google.golang.org/protobuf/reflect/protodesc - google.golang.org/protobuf/types/known/anypb from github.com/google/gnostic-models/compiler+ - google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ + google.golang.org/protobuf/proto from github.com/google/gnostic-models/compiler+ + đŸ’Ŗ google.golang.org/protobuf/reflect/protoreflect from github.com/google/gnostic-models/extensions+ + google.golang.org/protobuf/reflect/protoregistry from google.golang.org/protobuf/encoding/prototext+ + google.golang.org/protobuf/runtime/protoiface from google.golang.org/protobuf/internal/impl+ + google.golang.org/protobuf/runtime/protoimpl from github.com/google/gnostic-models/extensions+ + đŸ’Ŗ google.golang.org/protobuf/types/descriptorpb from github.com/google/gnostic-models/openapiv3 + đŸ’Ŗ google.golang.org/protobuf/types/known/anypb from github.com/google/gnostic-models/compiler+ + đŸ’Ŗ google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ gopkg.in/evanphx/json-patch.v4 from k8s.io/client-go/testing gopkg.in/inf.v0 from k8s.io/apimachinery/pkg/api/resource - gopkg.in/yaml.v3 from github.com/go-openapi/swag+ + gopkg.in/yaml.v3 from github.com/go-openapi/swag gvisor.dev/gvisor/pkg/atomicbitops from gvisor.dev/gvisor/pkg/buffer+ gvisor.dev/gvisor/pkg/bits from gvisor.dev/gvisor/pkg/buffer đŸ’Ŗ gvisor.dev/gvisor/pkg/buffer from gvisor.dev/gvisor/pkg/tcpip+ @@ -281,7 +200,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ đŸ’Ŗ gvisor.dev/gvisor/pkg/state from gvisor.dev/gvisor/pkg/atomicbitops+ gvisor.dev/gvisor/pkg/state/wire from gvisor.dev/gvisor/pkg/state đŸ’Ŗ gvisor.dev/gvisor/pkg/sync from gvisor.dev/gvisor/pkg/atomicbitops+ - đŸ’Ŗ gvisor.dev/gvisor/pkg/sync/locking from gvisor.dev/gvisor/pkg/tcpip/stack + đŸ’Ŗ gvisor.dev/gvisor/pkg/sync/locking from gvisor.dev/gvisor/pkg/tcpip/stack+ gvisor.dev/gvisor/pkg/tcpip from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ gvisor.dev/gvisor/pkg/tcpip/adapters/gonet from tailscale.com/wgengine/netstack đŸ’Ŗ gvisor.dev/gvisor/pkg/tcpip/checksum from gvisor.dev/gvisor/pkg/buffer+ @@ -348,7 +267,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/api/flowcontrol/v1beta2 from k8s.io/client-go/applyconfigurations/flowcontrol/v1beta2+ k8s.io/api/flowcontrol/v1beta3 from k8s.io/client-go/applyconfigurations/flowcontrol/v1beta3+ k8s.io/api/networking/v1 from k8s.io/client-go/applyconfigurations/networking/v1+ - k8s.io/api/networking/v1alpha1 from k8s.io/client-go/applyconfigurations/networking/v1alpha1+ k8s.io/api/networking/v1beta1 from k8s.io/client-go/applyconfigurations/networking/v1beta1+ k8s.io/api/node/v1 from k8s.io/client-go/applyconfigurations/node/v1+ k8s.io/api/node/v1alpha1 from k8s.io/client-go/applyconfigurations/node/v1alpha1+ @@ -358,8 +276,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/api/rbac/v1 from k8s.io/client-go/applyconfigurations/rbac/v1+ k8s.io/api/rbac/v1alpha1 from k8s.io/client-go/applyconfigurations/rbac/v1alpha1+ k8s.io/api/rbac/v1beta1 from k8s.io/client-go/applyconfigurations/rbac/v1beta1+ + k8s.io/api/resource/v1 from k8s.io/client-go/applyconfigurations/resource/v1+ k8s.io/api/resource/v1alpha3 from k8s.io/client-go/applyconfigurations/resource/v1alpha3+ k8s.io/api/resource/v1beta1 from k8s.io/client-go/applyconfigurations/resource/v1beta1+ + k8s.io/api/resource/v1beta2 from k8s.io/client-go/applyconfigurations/resource/v1beta2+ k8s.io/api/scheduling/v1 from k8s.io/client-go/applyconfigurations/scheduling/v1+ k8s.io/api/scheduling/v1alpha1 from k8s.io/client-go/applyconfigurations/scheduling/v1alpha1+ k8s.io/api/scheduling/v1beta1 from k8s.io/client-go/applyconfigurations/scheduling/v1beta1+ @@ -373,15 +293,20 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/apimachinery/pkg/api/errors from k8s.io/apimachinery/pkg/util/managedfields/internal+ k8s.io/apimachinery/pkg/api/meta from k8s.io/apimachinery/pkg/api/validation+ k8s.io/apimachinery/pkg/api/meta/testrestmapper from k8s.io/client-go/testing + k8s.io/apimachinery/pkg/api/operation from k8s.io/api/extensions/v1beta1+ k8s.io/apimachinery/pkg/api/resource from k8s.io/api/autoscaling/v1+ + k8s.io/apimachinery/pkg/api/safe from k8s.io/api/extensions/v1beta1 + k8s.io/apimachinery/pkg/api/validate from k8s.io/api/extensions/v1beta1 + k8s.io/apimachinery/pkg/api/validate/constraints from k8s.io/apimachinery/pkg/api/validate+ + k8s.io/apimachinery/pkg/api/validate/content from k8s.io/apimachinery/pkg/api/validate k8s.io/apimachinery/pkg/api/validation from k8s.io/apimachinery/pkg/util/managedfields/internal+ + k8s.io/apimachinery/pkg/api/validation/path from k8s.io/apiserver/pkg/endpoints/request đŸ’Ŗ k8s.io/apimachinery/pkg/apis/meta/internalversion from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+ - k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme from k8s.io/client-go/metadata - k8s.io/apimachinery/pkg/apis/meta/internalversion/validation from k8s.io/client-go/util/watchlist + k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme from k8s.io/client-go/metadata+ đŸ’Ŗ k8s.io/apimachinery/pkg/apis/meta/v1 from k8s.io/api/admission/v1+ k8s.io/apimachinery/pkg/apis/meta/v1/unstructured from k8s.io/apimachinery/pkg/runtime/serializer/versioning+ k8s.io/apimachinery/pkg/apis/meta/v1/validation from k8s.io/apimachinery/pkg/api/validation+ - đŸ’Ŗ k8s.io/apimachinery/pkg/apis/meta/v1beta1 from k8s.io/apimachinery/pkg/apis/meta/internalversion + đŸ’Ŗ k8s.io/apimachinery/pkg/apis/meta/v1beta1 from k8s.io/apimachinery/pkg/apis/meta/internalversion+ k8s.io/apimachinery/pkg/conversion from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+ k8s.io/apimachinery/pkg/conversion/queryparams from k8s.io/apimachinery/pkg/runtime+ k8s.io/apimachinery/pkg/fields from k8s.io/apimachinery/pkg/api/equality+ @@ -400,7 +325,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/apimachinery/pkg/selection from k8s.io/apimachinery/pkg/apis/meta/v1+ k8s.io/apimachinery/pkg/types from k8s.io/api/admission/v1+ k8s.io/apimachinery/pkg/util/cache from k8s.io/client-go/tools/cache - k8s.io/apimachinery/pkg/util/diff from k8s.io/client-go/tools/cache + k8s.io/apimachinery/pkg/util/diff from k8s.io/client-go/tools/cache+ k8s.io/apimachinery/pkg/util/dump from k8s.io/apimachinery/pkg/util/diff+ k8s.io/apimachinery/pkg/util/errors from k8s.io/apimachinery/pkg/api/meta+ k8s.io/apimachinery/pkg/util/framer from k8s.io/apimachinery/pkg/runtime/serializer/json+ @@ -419,13 +344,18 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/apimachinery/pkg/util/uuid from sigs.k8s.io/controller-runtime/pkg/internal/controller+ k8s.io/apimachinery/pkg/util/validation from k8s.io/apimachinery/pkg/api/validation+ k8s.io/apimachinery/pkg/util/validation/field from k8s.io/apimachinery/pkg/api/errors+ + k8s.io/apimachinery/pkg/util/version from k8s.io/apiserver/pkg/features+ k8s.io/apimachinery/pkg/util/wait from k8s.io/client-go/tools/cache+ k8s.io/apimachinery/pkg/util/yaml from k8s.io/apimachinery/pkg/runtime/serializer/json k8s.io/apimachinery/pkg/version from k8s.io/client-go/discovery+ k8s.io/apimachinery/pkg/watch from k8s.io/apimachinery/pkg/apis/meta/v1+ k8s.io/apimachinery/third_party/forked/golang/json from k8s.io/apimachinery/pkg/util/strategicpatch k8s.io/apimachinery/third_party/forked/golang/reflect from k8s.io/apimachinery/pkg/conversion + k8s.io/apiserver/pkg/authentication/user from k8s.io/apiserver/pkg/endpoints/request + k8s.io/apiserver/pkg/endpoints/request from tailscale.com/k8s-operator/api-proxy + k8s.io/apiserver/pkg/features from k8s.io/apiserver/pkg/endpoints/request k8s.io/apiserver/pkg/storage/names from tailscale.com/cmd/k8s-operator + k8s.io/apiserver/pkg/util/feature from k8s.io/apiserver/pkg/endpoints/request+ k8s.io/client-go/applyconfigurations/admissionregistration/v1 from k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1+ k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1 k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1 from k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 @@ -458,7 +388,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/applyconfigurations/internal from k8s.io/client-go/applyconfigurations/admissionregistration/v1+ k8s.io/client-go/applyconfigurations/meta/v1 from k8s.io/client-go/applyconfigurations/admissionregistration/v1+ k8s.io/client-go/applyconfigurations/networking/v1 from k8s.io/client-go/kubernetes/typed/networking/v1 - k8s.io/client-go/applyconfigurations/networking/v1alpha1 from k8s.io/client-go/kubernetes/typed/networking/v1alpha1 k8s.io/client-go/applyconfigurations/networking/v1beta1 from k8s.io/client-go/kubernetes/typed/networking/v1beta1 k8s.io/client-go/applyconfigurations/node/v1 from k8s.io/client-go/kubernetes/typed/node/v1 k8s.io/client-go/applyconfigurations/node/v1alpha1 from k8s.io/client-go/kubernetes/typed/node/v1alpha1 @@ -468,8 +397,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/applyconfigurations/rbac/v1 from k8s.io/client-go/kubernetes/typed/rbac/v1 k8s.io/client-go/applyconfigurations/rbac/v1alpha1 from k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 k8s.io/client-go/applyconfigurations/rbac/v1beta1 from k8s.io/client-go/kubernetes/typed/rbac/v1beta1 + k8s.io/client-go/applyconfigurations/resource/v1 from k8s.io/client-go/kubernetes/typed/resource/v1 k8s.io/client-go/applyconfigurations/resource/v1alpha3 from k8s.io/client-go/kubernetes/typed/resource/v1alpha3 k8s.io/client-go/applyconfigurations/resource/v1beta1 from k8s.io/client-go/kubernetes/typed/resource/v1beta1 + k8s.io/client-go/applyconfigurations/resource/v1beta2 from k8s.io/client-go/kubernetes/typed/resource/v1beta2 k8s.io/client-go/applyconfigurations/scheduling/v1 from k8s.io/client-go/kubernetes/typed/scheduling/v1 k8s.io/client-go/applyconfigurations/scheduling/v1alpha1 from k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 k8s.io/client-go/applyconfigurations/scheduling/v1beta1 from k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 @@ -526,7 +457,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/informers/internalinterfaces from k8s.io/client-go/informers+ k8s.io/client-go/informers/networking from k8s.io/client-go/informers k8s.io/client-go/informers/networking/v1 from k8s.io/client-go/informers/networking - k8s.io/client-go/informers/networking/v1alpha1 from k8s.io/client-go/informers/networking k8s.io/client-go/informers/networking/v1beta1 from k8s.io/client-go/informers/networking k8s.io/client-go/informers/node from k8s.io/client-go/informers k8s.io/client-go/informers/node/v1 from k8s.io/client-go/informers/node @@ -540,8 +470,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/informers/rbac/v1alpha1 from k8s.io/client-go/informers/rbac k8s.io/client-go/informers/rbac/v1beta1 from k8s.io/client-go/informers/rbac k8s.io/client-go/informers/resource from k8s.io/client-go/informers + k8s.io/client-go/informers/resource/v1 from k8s.io/client-go/informers/resource k8s.io/client-go/informers/resource/v1alpha3 from k8s.io/client-go/informers/resource k8s.io/client-go/informers/resource/v1beta1 from k8s.io/client-go/informers/resource + k8s.io/client-go/informers/resource/v1beta2 from k8s.io/client-go/informers/resource k8s.io/client-go/informers/scheduling from k8s.io/client-go/informers k8s.io/client-go/informers/scheduling/v1 from k8s.io/client-go/informers/scheduling k8s.io/client-go/informers/scheduling/v1alpha1 from k8s.io/client-go/informers/scheduling @@ -576,8 +508,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/kubernetes/typed/certificates/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/certificates/v1beta1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/coordination/v1 from k8s.io/client-go/kubernetes+ - k8s.io/client-go/kubernetes/typed/coordination/v1alpha2 from k8s.io/client-go/kubernetes+ - k8s.io/client-go/kubernetes/typed/coordination/v1beta1 from k8s.io/client-go/kubernetes + k8s.io/client-go/kubernetes/typed/coordination/v1alpha2 from k8s.io/client-go/kubernetes + k8s.io/client-go/kubernetes/typed/coordination/v1beta1 from k8s.io/client-go/kubernetes+ k8s.io/client-go/kubernetes/typed/core/v1 from k8s.io/client-go/kubernetes+ k8s.io/client-go/kubernetes/typed/discovery/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/discovery/v1beta1 from k8s.io/client-go/kubernetes @@ -589,7 +521,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/networking/v1 from k8s.io/client-go/kubernetes - k8s.io/client-go/kubernetes/typed/networking/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/networking/v1beta1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/node/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/node/v1alpha1 from k8s.io/client-go/kubernetes @@ -599,8 +530,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/kubernetes/typed/rbac/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/rbac/v1beta1 from k8s.io/client-go/kubernetes + k8s.io/client-go/kubernetes/typed/resource/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/resource/v1alpha3 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/resource/v1beta1 from k8s.io/client-go/kubernetes + k8s.io/client-go/kubernetes/typed/resource/v1beta2 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/scheduling/v1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 from k8s.io/client-go/kubernetes k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 from k8s.io/client-go/kubernetes @@ -639,7 +572,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/listers/flowcontrol/v1beta2 from k8s.io/client-go/informers/flowcontrol/v1beta2 k8s.io/client-go/listers/flowcontrol/v1beta3 from k8s.io/client-go/informers/flowcontrol/v1beta3 k8s.io/client-go/listers/networking/v1 from k8s.io/client-go/informers/networking/v1 - k8s.io/client-go/listers/networking/v1alpha1 from k8s.io/client-go/informers/networking/v1alpha1 k8s.io/client-go/listers/networking/v1beta1 from k8s.io/client-go/informers/networking/v1beta1 k8s.io/client-go/listers/node/v1 from k8s.io/client-go/informers/node/v1 k8s.io/client-go/listers/node/v1alpha1 from k8s.io/client-go/informers/node/v1alpha1 @@ -649,8 +581,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/listers/rbac/v1 from k8s.io/client-go/informers/rbac/v1 k8s.io/client-go/listers/rbac/v1alpha1 from k8s.io/client-go/informers/rbac/v1alpha1 k8s.io/client-go/listers/rbac/v1beta1 from k8s.io/client-go/informers/rbac/v1beta1 + k8s.io/client-go/listers/resource/v1 from k8s.io/client-go/informers/resource/v1 k8s.io/client-go/listers/resource/v1alpha3 from k8s.io/client-go/informers/resource/v1alpha3 k8s.io/client-go/listers/resource/v1beta1 from k8s.io/client-go/informers/resource/v1beta1 + k8s.io/client-go/listers/resource/v1beta2 from k8s.io/client-go/informers/resource/v1beta2 k8s.io/client-go/listers/scheduling/v1 from k8s.io/client-go/informers/scheduling/v1 k8s.io/client-go/listers/scheduling/v1alpha1 from k8s.io/client-go/informers/scheduling/v1alpha1 k8s.io/client-go/listers/scheduling/v1beta1 from k8s.io/client-go/informers/scheduling/v1beta1 @@ -689,12 +623,18 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/client-go/util/apply from k8s.io/client-go/dynamic+ k8s.io/client-go/util/cert from k8s.io/client-go/rest+ k8s.io/client-go/util/connrotation from k8s.io/client-go/plugin/pkg/client/auth/exec+ - k8s.io/client-go/util/consistencydetector from k8s.io/client-go/dynamic+ + k8s.io/client-go/util/consistencydetector from k8s.io/client-go/tools/cache k8s.io/client-go/util/flowcontrol from k8s.io/client-go/kubernetes+ k8s.io/client-go/util/homedir from k8s.io/client-go/tools/clientcmd k8s.io/client-go/util/keyutil from k8s.io/client-go/util/cert - k8s.io/client-go/util/watchlist from k8s.io/client-go/dynamic+ k8s.io/client-go/util/workqueue from k8s.io/client-go/transport+ + k8s.io/component-base/featuregate from k8s.io/apiserver/pkg/features+ + k8s.io/component-base/metrics from k8s.io/component-base/metrics/legacyregistry+ + k8s.io/component-base/metrics/legacyregistry from k8s.io/component-base/metrics/prometheus/feature + k8s.io/component-base/metrics/prometheus/feature from k8s.io/component-base/featuregate + k8s.io/component-base/metrics/prometheusextension from k8s.io/component-base/metrics + k8s.io/component-base/version from k8s.io/component-base/featuregate+ + k8s.io/component-base/zpages/features from k8s.io/apiserver/pkg/features k8s.io/klog/v2 from k8s.io/apimachinery/pkg/api/meta+ k8s.io/klog/v2/internal/buffer from k8s.io/klog/v2 k8s.io/klog/v2/internal/clock from k8s.io/klog/v2 @@ -713,15 +653,13 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/kube-openapi/pkg/validation/spec from k8s.io/apimachinery/pkg/util/managedfields+ k8s.io/utils/buffer from k8s.io/client-go/tools/cache k8s.io/utils/clock from k8s.io/apimachinery/pkg/util/cache+ - k8s.io/utils/clock/testing from k8s.io/client-go/util/flowcontrol k8s.io/utils/internal/third_party/forked/golang/golang-lru from k8s.io/utils/lru k8s.io/utils/internal/third_party/forked/golang/net from k8s.io/utils/net k8s.io/utils/lru from k8s.io/client-go/tools/record k8s.io/utils/net from k8s.io/apimachinery/pkg/util/net+ - k8s.io/utils/pointer from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+ k8s.io/utils/ptr from k8s.io/client-go/tools/cache+ k8s.io/utils/trace from k8s.io/client-go/tools/cache - sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator + sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator+ sigs.k8s.io/controller-runtime/pkg/cache from sigs.k8s.io/controller-runtime/pkg/cluster+ sigs.k8s.io/controller-runtime/pkg/cache/internal from sigs.k8s.io/controller-runtime/pkg/cache sigs.k8s.io/controller-runtime/pkg/certwatcher from sigs.k8s.io/controller-runtime/pkg/metrics/server+ @@ -762,66 +700,75 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics from sigs.k8s.io/controller-runtime/pkg/webhook+ sigs.k8s.io/json from k8s.io/apimachinery/pkg/runtime/serializer/json+ sigs.k8s.io/json/internal/golang/encoding/json from sigs.k8s.io/json - đŸ’Ŗ sigs.k8s.io/structured-merge-diff/v4/fieldpath from k8s.io/apimachinery/pkg/util/managedfields+ - sigs.k8s.io/structured-merge-diff/v4/merge from k8s.io/apimachinery/pkg/util/managedfields/internal - sigs.k8s.io/structured-merge-diff/v4/schema from k8s.io/apimachinery/pkg/util/managedfields+ - sigs.k8s.io/structured-merge-diff/v4/typed from k8s.io/apimachinery/pkg/util/managedfields+ - sigs.k8s.io/structured-merge-diff/v4/value from k8s.io/apimachinery/pkg/runtime+ + đŸ’Ŗ sigs.k8s.io/randfill from k8s.io/apimachinery/pkg/apis/meta/v1+ + sigs.k8s.io/randfill/bytesource from sigs.k8s.io/randfill + đŸ’Ŗ sigs.k8s.io/structured-merge-diff/v6/fieldpath from k8s.io/apimachinery/pkg/util/managedfields+ + sigs.k8s.io/structured-merge-diff/v6/merge from k8s.io/apimachinery/pkg/util/managedfields/internal + sigs.k8s.io/structured-merge-diff/v6/schema from k8s.io/apimachinery/pkg/util/managedfields+ + sigs.k8s.io/structured-merge-diff/v6/typed from k8s.io/apimachinery/pkg/util/managedfields+ + sigs.k8s.io/structured-merge-diff/v6/value from k8s.io/apimachinery/pkg/runtime+ sigs.k8s.io/yaml from k8s.io/apimachinery/pkg/runtime/serializer/json+ - sigs.k8s.io/yaml/goyaml.v2 from sigs.k8s.io/yaml+ tailscale.com from tailscale.com/version tailscale.com/appc from tailscale.com/ipn/ipnlocal đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/ipn+ tailscale.com/client/local from tailscale.com/client/tailscale+ - tailscale.com/client/tailscale from tailscale.com/cmd/k8s-operator+ + tailscale.com/client/tailscale from tailscale.com/internal/client/tailscale tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ + tailscale.com/client/tailscale/v2 from tailscale.com/cmd/k8s-operator+ tailscale.com/client/web from tailscale.com/ipn/ipnlocal - tailscale.com/clientupdate from tailscale.com/client/web+ - LW tailscale.com/clientupdate/distsign from tailscale.com/clientupdate tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ tailscale.com/control/controlclient from tailscale.com/ipn/ipnlocal+ - tailscale.com/control/controlhttp from tailscale.com/control/controlclient + tailscale.com/control/controlhttp from tailscale.com/control/ts2021 tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp tailscale.com/control/controlknobs from tailscale.com/control/controlclient+ + tailscale.com/control/ts2021 from tailscale.com/control/controlclient tailscale.com/derp from tailscale.com/derp/derphttp+ - tailscale.com/derp/derpconst from tailscale.com/derp+ + tailscale.com/derp/derpconst from tailscale.com/derp/derphttp+ tailscale.com/derp/derphttp from tailscale.com/ipn/localapi+ - tailscale.com/disco from tailscale.com/derp+ - tailscale.com/doctor from tailscale.com/ipn/ipnlocal - tailscale.com/doctor/ethtool from tailscale.com/ipn/ipnlocal - đŸ’Ŗ tailscale.com/doctor/permissions from tailscale.com/ipn/ipnlocal - tailscale.com/doctor/routetable from tailscale.com/ipn/ipnlocal + tailscale.com/disco from tailscale.com/net/tstun+ tailscale.com/drive from tailscale.com/client/local+ tailscale.com/envknob from tailscale.com/client/local+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ + tailscale.com/feature/buildfeatures from tailscale.com/wgengine/magicsock+ + tailscale.com/feature/c2n from tailscale.com/tsnet + tailscale.com/feature/condlite/expvar from tailscale.com/wgengine/magicsock + tailscale.com/feature/condregister/oauthkey from tailscale.com/tsnet + tailscale.com/feature/condregister/portmapper from tailscale.com/tsnet + tailscale.com/feature/condregister/useproxy from tailscale.com/tsnet + tailscale.com/feature/oauthkey from tailscale.com/feature/condregister/oauthkey + tailscale.com/feature/portmapper from tailscale.com/feature/condregister/portmapper + tailscale.com/feature/useproxy from tailscale.com/feature/condregister/useproxy tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal tailscale.com/hostinfo from tailscale.com/client/web+ - tailscale.com/internal/client/tailscale from tailscale.com/cmd/k8s-operator - tailscale.com/internal/noiseconn from tailscale.com/control/controlclient + tailscale.com/internal/client/tailscale from tailscale.com/feature/oauthkey+ tailscale.com/ipn from tailscale.com/client/local+ tailscale.com/ipn/conffile from tailscale.com/ipn/ipnlocal+ đŸ’Ŗ tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnlocal+ - tailscale.com/ipn/ipnext from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/ipnext from tailscale.com/ipn/ipnlocal+ tailscale.com/ipn/ipnlocal from tailscale.com/ipn/localapi+ + tailscale.com/ipn/ipnlocal/netmapcache from tailscale.com/ipn/ipnlocal tailscale.com/ipn/ipnstate from tailscale.com/client/local+ tailscale.com/ipn/localapi from tailscale.com/tsnet - tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+ - L tailscale.com/ipn/store/awsstore from tailscale.com/ipn/store - tailscale.com/ipn/store/kubestore from tailscale.com/cmd/k8s-operator+ + tailscale.com/ipn/store/kubestore from tailscale.com/cmd/k8s-operator tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+ - tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator+ tailscale.com/k8s-operator/api-proxy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/apis from tailscale.com/k8s-operator/apis/v1alpha1 tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+ + tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet + tailscale.com/k8s-operator/reconciler/proxygrouppolicy from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/sessionrecording from tailscale.com/k8s-operator/api-proxy tailscale.com/k8s-operator/sessionrecording/spdy from tailscale.com/k8s-operator/sessionrecording tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+ tailscale.com/k8s-operator/sessionrecording/ws from tailscale.com/k8s-operator/sessionrecording + tailscale.com/k8s-operator/tsclient from tailscale.com/cmd/k8s-operator+ tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator tailscale.com/kube/ingressservices from tailscale.com/cmd/k8s-operator + tailscale.com/kube/k8s-proxy/conf from tailscale.com/cmd/k8s-operator tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore tailscale.com/kube/kubetypes from tailscale.com/cmd/k8s-operator+ @@ -830,20 +777,18 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/log/sockstatlog from tailscale.com/ipn/ipnlocal tailscale.com/logpolicy from tailscale.com/ipn/ipnlocal+ tailscale.com/logtail from tailscale.com/control/controlclient+ - tailscale.com/logtail/backoff from tailscale.com/control/controlclient+ tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ - tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/metrics from tailscale.com/tsweb+ tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+ + đŸ’Ŗ tailscale.com/net/batching from tailscale.com/wgengine/magicsock tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ - tailscale.com/net/connstats from tailscale.com/net/tstun+ tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+ tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ - tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback tailscale.com/net/dns/resolvconffile from tailscale.com/cmd/k8s-operator+ tailscale.com/net/dns/resolver from tailscale.com/net/dns+ tailscale.com/net/dnscache from tailscale.com/control/controlclient+ tailscale.com/net/dnsfallback from tailscale.com/control/controlclient+ - tailscale.com/net/flowtrack from tailscale.com/net/packet+ + tailscale.com/net/flowtrack from tailscale.com/wgengine+ tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ tailscale.com/net/memnet from tailscale.com/tsnet tailscale.com/net/netaddr from tailscale.com/ipn+ @@ -853,30 +798,30 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/net/netknob from tailscale.com/logpolicy+ đŸ’Ŗ tailscale.com/net/netmon from tailscale.com/control/controlclient+ đŸ’Ŗ tailscale.com/net/netns from tailscale.com/derp/derphttp+ - W đŸ’Ŗ tailscale.com/net/netstat from tailscale.com/portlist tailscale.com/net/netutil from tailscale.com/client/local+ tailscale.com/net/netx from tailscale.com/control/controlclient+ - tailscale.com/net/packet from tailscale.com/net/connstats+ + tailscale.com/net/packet from tailscale.com/ipn/ipnlocal+ tailscale.com/net/packet/checksum from tailscale.com/net/tstun tailscale.com/net/ping from tailscale.com/net/netcheck+ - tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ + tailscale.com/net/portmapper from tailscale.com/feature/portmapper + tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ tailscale.com/net/proxymux from tailscale.com/tsnet - tailscale.com/net/routetable from tailscale.com/doctor/routetable + tailscale.com/net/routecheck from tailscale.com/client/local + đŸ’Ŗ tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock tailscale.com/net/socks5 from tailscale.com/tsnet tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/ipn/localapi+ - L tailscale.com/net/tcpinfo from tailscale.com/derp tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal+ tailscale.com/net/tsaddr from tailscale.com/client/web+ tailscale.com/net/tsdial from tailscale.com/control/controlclient+ - đŸ’Ŗ tailscale.com/net/tshttpproxy from tailscale.com/clientupdate/distsign+ + đŸ’Ŗ tailscale.com/net/tshttpproxy from tailscale.com/feature/useproxy tailscale.com/net/tstun from tailscale.com/tsd+ tailscale.com/net/udprelay/endpoint from tailscale.com/wgengine/magicsock + tailscale.com/net/udprelay/status from tailscale.com/client/local tailscale.com/omit from tailscale.com/ipn/conffile tailscale.com/paths from tailscale.com/client/local+ - đŸ’Ŗ tailscale.com/portlist from tailscale.com/ipn/ipnlocal - tailscale.com/posture from tailscale.com/ipn/ipnlocal tailscale.com/proxymap from tailscale.com/tsd+ đŸ’Ŗ tailscale.com/safesocket from tailscale.com/client/local+ tailscale.com/sessionrecording from tailscale.com/k8s-operator/sessionrecording+ @@ -884,44 +829,49 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/tailcfg from tailscale.com/client/local+ tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock - tailscale.com/tempfork/httprec from tailscale.com/control/controlclient + tailscale.com/tempfork/httprec from tailscale.com/feature/c2n tailscale.com/tka from tailscale.com/client/local+ tailscale.com/tsconst from tailscale.com/net/netmon+ tailscale.com/tsd from tailscale.com/ipn/ipnlocal+ tailscale.com/tsnet from tailscale.com/cmd/k8s-operator+ tailscale.com/tstime from tailscale.com/cmd/k8s-operator+ tailscale.com/tstime/mono from tailscale.com/net/tstun+ - tailscale.com/tstime/rate from tailscale.com/derp+ - tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tstime/rate from tailscale.com/wgengine/filter + tailscale.com/tsweb from tailscale.com/util/eventbus+ tailscale.com/tsweb/varz from tailscale.com/util/usermetric+ - tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal - tailscale.com/types/bools from tailscale.com/tsnet + tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+ + tailscale.com/types/bools from tailscale.com/tsnet+ tailscale.com/types/dnstype from tailscale.com/ipn/ipnlocal+ tailscale.com/types/empty from tailscale.com/ipn+ + tailscale.com/types/events from tailscale.com/control/controlclient+ tailscale.com/types/ipproto from tailscale.com/net/flowtrack+ tailscale.com/types/key from tailscale.com/client/local+ tailscale.com/types/lazy from tailscale.com/ipn/ipnlocal+ tailscale.com/types/logger from tailscale.com/appc+ tailscale.com/types/logid from tailscale.com/ipn/ipnlocal+ tailscale.com/types/mapx from tailscale.com/ipn/ipnext - tailscale.com/types/netlogtype from tailscale.com/net/connstats+ + tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ + tailscale.com/types/netlogtype from tailscale.com/wgengine/netlog tailscale.com/types/netmap from tailscale.com/control/controlclient+ tailscale.com/types/nettype from tailscale.com/ipn/localapi+ tailscale.com/types/opt from tailscale.com/client/tailscale+ tailscale.com/types/persist from tailscale.com/control/controlclient+ tailscale.com/types/preftype from tailscale.com/ipn+ - tailscale.com/types/ptr from tailscale.com/cmd/k8s-operator+ tailscale.com/types/result from tailscale.com/util/lineiter tailscale.com/types/structs from tailscale.com/control/controlclient+ tailscale.com/types/tkatype from tailscale.com/client/local+ tailscale.com/types/views from tailscale.com/appc+ - tailscale.com/util/cibuild from tailscale.com/health + tailscale.com/util/backoff from tailscale.com/cmd/k8s-operator+ + tailscale.com/util/bufiox from tailscale.com/types/key + tailscale.com/util/checkchange from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/cibuild from tailscale.com/health+ tailscale.com/util/clientmetric from tailscale.com/cmd/k8s-operator+ tailscale.com/util/cloudenv from tailscale.com/hostinfo+ - tailscale.com/util/cmpver from tailscale.com/clientupdate+ + tailscale.com/util/cloudinfo from tailscale.com/wgengine/magicsock + LW tailscale.com/util/cmpver from tailscale.com/net/dns+ tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ - đŸ’Ŗ tailscale.com/util/deephash from tailscale.com/ipn/ipnlocal+ - L đŸ’Ŗ tailscale.com/util/dirwalk from tailscale.com/metrics+ + đŸ’Ŗ tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting + L đŸ’Ŗ tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/appc+ tailscale.com/util/eventbus from tailscale.com/tsd+ tailscale.com/util/execqueue from tailscale.com/appc+ @@ -930,10 +880,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ đŸ’Ŗ tailscale.com/util/hashx from tailscale.com/util/deephash tailscale.com/util/httpm from tailscale.com/client/tailscale+ tailscale.com/util/lineiter from tailscale.com/hostinfo+ - L tailscale.com/util/linuxfw from tailscale.com/net/netns+ tailscale.com/util/mak from tailscale.com/appc+ - tailscale.com/util/multierr from tailscale.com/control/controlclient+ - tailscale.com/util/must from tailscale.com/clientupdate/distsign+ + tailscale.com/util/must from tailscale.com/logpolicy+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto đŸ’Ŗ tailscale.com/util/osdiag from tailscale.com/ipn/localapi W đŸ’Ŗ tailscale.com/util/osdiag/internal/wsc from tailscale.com/util/osdiag @@ -941,25 +889,25 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/util/race from tailscale.com/net/dns/resolver tailscale.com/util/racebuild from tailscale.com/logpolicy tailscale.com/util/rands from tailscale.com/ipn/ipnlocal+ - tailscale.com/util/ringbuffer from tailscale.com/wgengine/magicsock + tailscale.com/util/ringlog from tailscale.com/wgengine/magicsock tailscale.com/util/set from tailscale.com/cmd/k8s-operator+ tailscale.com/util/singleflight from tailscale.com/control/controlclient+ tailscale.com/util/slicesx from tailscale.com/appc+ - tailscale.com/util/syspolicy from tailscale.com/control/controlclient+ tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy/setting+ tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy/internal/metrics+ tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source - tailscale.com/util/syspolicy/rsop from tailscale.com/util/syspolicy+ - tailscale.com/util/syspolicy/setting from tailscale.com/util/syspolicy+ - tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy+ - tailscale.com/util/sysresources from tailscale.com/wgengine/magicsock - tailscale.com/util/systemd from tailscale.com/control/controlclient+ + tailscale.com/util/syspolicy/pkey from tailscale.com/control/controlclient+ + tailscale.com/util/syspolicy/policyclient from tailscale.com/control/controlclient+ + tailscale.com/util/syspolicy/ptype from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/syspolicy/rsop from tailscale.com/ipn/localapi + tailscale.com/util/syspolicy/setting from tailscale.com/client/local+ + tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy/rsop tailscale.com/util/testenv from tailscale.com/control/controlclient+ tailscale.com/util/truncate from tailscale.com/logtail tailscale.com/util/usermetric from tailscale.com/health+ tailscale.com/util/vizerror from tailscale.com/tailcfg+ - đŸ’Ŗ tailscale.com/util/winutil from tailscale.com/clientupdate+ - W đŸ’Ŗ tailscale.com/util/winutil/authenticode from tailscale.com/clientupdate+ + đŸ’Ŗ tailscale.com/util/winutil from tailscale.com/hostinfo+ + W đŸ’Ŗ tailscale.com/util/winutil/authenticode from tailscale.com/util/osdiag W đŸ’Ŗ tailscale.com/util/winutil/gp from tailscale.com/net/dns+ W tailscale.com/util/winutil/policy from tailscale.com/ipn/ipnlocal W đŸ’Ŗ tailscale.com/util/winutil/winenv from tailscale.com/hostinfo+ @@ -978,16 +926,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal đŸ’Ŗ tailscale.com/wgengine/wgint from tailscale.com/wgengine+ tailscale.com/wgengine/wglog from tailscale.com/wgengine - W đŸ’Ŗ tailscale.com/wgengine/winnet from tailscale.com/wgengine/router golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ - LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf - golang.org/x/crypto/chacha20 from golang.org/x/crypto/ssh+ - golang.org/x/crypto/chacha20poly1305 from crypto/tls+ - golang.org/x/crypto/cryptobyte from crypto/ecdsa+ - golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ - golang.org/x/crypto/curve25519 from golang.org/x/crypto/ssh+ + golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305 + golang.org/x/crypto/chacha20poly1305 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/curve25519 from github.com/tailscale/wireguard-go/device+ golang.org/x/crypto/hkdf from tailscale.com/control/controlbase golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ @@ -995,31 +939,29 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ - LD golang.org/x/crypto/ssh from tailscale.com/ipn/ipnlocal - LD golang.org/x/crypto/ssh/internal/bcrypt_pbkdf from golang.org/x/crypto/ssh - golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ + golang.org/x/exp/constraints from tailscale.com/tsweb/varz+ golang.org/x/exp/maps from sigs.k8s.io/controller-runtime/pkg/cache+ golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator+ - golang.org/x/net/bpf from github.com/mdlayher/genetlink+ - golang.org/x/net/dns/dnsmessage from net+ + golang.org/x/net/bpf from github.com/mdlayher/netlink+ + golang.org/x/net/dns/dnsmessage from tailscale.com/appc+ golang.org/x/net/http/httpguts from golang.org/x/net/http2+ - golang.org/x/net/http/httpproxy from net/http+ - golang.org/x/net/http2 from golang.org/x/net/http2/h2c+ - golang.org/x/net/http2/h2c from tailscale.com/ipn/ipnlocal + golang.org/x/net/http/httpproxy from tailscale.com/net/tshttpproxy + golang.org/x/net/http2 from k8s.io/apimachinery/pkg/util/net+ golang.org/x/net/http2/hpack from golang.org/x/net/http2+ - golang.org/x/net/icmp from github.com/prometheus-community/pro-bing+ + golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpguts+ golang.org/x/net/internal/httpcommon from golang.org/x/net/http2 + golang.org/x/net/internal/httpsfv from golang.org/x/net/http2 golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/internal/socks from golang.org/x/net/proxy - golang.org/x/net/ipv4 from github.com/miekg/dns+ - golang.org/x/net/ipv6 from github.com/miekg/dns+ + golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ golang.org/x/net/proxy from tailscale.com/net/netns - D golang.org/x/net/route from net+ + D golang.org/x/net/route from tailscale.com/net/netmon+ golang.org/x/net/websocket from tailscale.com/k8s-operator/sessionrecording/ws golang.org/x/oauth2 from golang.org/x/oauth2/clientcredentials+ - golang.org/x/oauth2/clientcredentials from tailscale.com/cmd/k8s-operator + golang.org/x/oauth2/clientcredentials from tailscale.com/client/tailscale/v2+ golang.org/x/oauth2/internal from golang.org/x/oauth2+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ golang.org/x/sys/cpu from github.com/tailscale/certstore+ @@ -1034,30 +976,48 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ golang.org/x/text/unicode/norm from golang.org/x/net/idna golang.org/x/time/rate from gvisor.dev/gvisor/pkg/log+ - archive/tar from tailscale.com/clientupdate + vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ + vendor/golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + vendor/golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + vendor/golang.org/x/crypto/internal/alias from vendor/golang.org/x/crypto/chacha20+ + vendor/golang.org/x/crypto/internal/poly1305 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/net/dns/dnsmessage from net + vendor/golang.org/x/net/http/httpguts from net/http+ + vendor/golang.org/x/net/http/httpproxy from net/http + vendor/golang.org/x/net/http2/hpack from net/http+ + vendor/golang.org/x/net/idna from net/http+ + vendor/golang.org/x/sys/cpu from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/text/secure/bidirule from vendor/golang.org/x/net/idna + vendor/golang.org/x/text/transform from vendor/golang.org/x/text/secure/bidirule+ + vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ + vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna bufio from compress/flate+ - bytes from archive/tar+ - cmp from github.com/gaissmai/bart+ + bytes from bufio+ + cmp from encoding/json+ compress/flate from compress/gzip+ - compress/gzip from github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding+ - compress/zlib from debug/pe+ + compress/gzip from github.com/emicklei/go-restful/v3+ + compress/zlib from github.com/emicklei/go-restful/v3+ container/heap from gvisor.dev/gvisor/pkg/tcpip/transport/tcp+ container/list from crypto/tls+ context from crypto/tls+ crypto from crypto/ecdh+ - crypto/aes from crypto/internal/hpke+ + crypto/aes from crypto/tls+ crypto/cipher from crypto/aes+ crypto/des from crypto/tls+ - crypto/dsa from crypto/x509+ + crypto/dsa from crypto/x509 crypto/ecdh from crypto/ecdsa+ crypto/ecdsa from crypto/tls+ crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ + crypto/fips140 from crypto/tls/internal/fips140tls+ + crypto/hkdf from crypto/hpke+ crypto/hmac from crypto/tls+ + crypto/hpke from crypto/tls crypto/internal/boring from crypto/aes+ crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring - crypto/internal/entropy from crypto/internal/fips140/drbg + crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ crypto/internal/fips140 from crypto/internal/fips140/aes+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ @@ -1072,7 +1032,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls+ + crypto/internal/fips140/mlkem from crypto/mlkem crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -1082,26 +1042,28 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ crypto/internal/fips140/tls12 from crypto/tls crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140cache from crypto/ecdsa+ crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140deps/time from crypto/internal/entropy/v1.0.0 crypto/internal/fips140hash from crypto/ecdsa+ crypto/internal/fips140only from crypto/cipher+ - crypto/internal/hpke from crypto/tls crypto/internal/impl from crypto/internal/fips140/aes+ - crypto/internal/randutil from crypto/dsa+ - crypto/internal/sysrand from crypto/internal/entropy+ + crypto/internal/rand from crypto/dsa+ + crypto/internal/randutil from crypto/internal/rand + crypto/internal/sysrand from crypto/internal/fips140/drbg crypto/md5 from crypto/tls+ - LD crypto/mlkem from golang.org/x/crypto/ssh + crypto/mlkem from crypto/hpke+ crypto/rand from crypto/ed25519+ - crypto/rc4 from crypto/tls+ + crypto/rc4 from crypto/tls crypto/rsa from crypto/tls+ crypto/sha1 from crypto/tls+ crypto/sha256 from crypto/tls+ - crypto/sha3 from crypto/internal/fips140hash + crypto/sha3 from crypto/internal/fips140hash+ crypto/sha512 from crypto/ecdsa+ crypto/subtle from crypto/cipher+ - crypto/tls from github.com/aws/aws-sdk-go-v2/aws/transport/http+ + crypto/tls from github.com/prometheus/client_golang/prometheus/promhttp+ crypto/tls/internal/fips140tls from crypto/tls crypto/x509 from crypto/tls+ D crypto/x509/internal/macos from crypto/x509 @@ -1120,22 +1082,23 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ - encoding/xml from github.com/aws/aws-sdk-go-v2/aws/protocol/xml+ - errors from archive/tar+ + encoding/xml from github.com/emicklei/go-restful/v3+ + errors from bufio+ expvar from github.com/prometheus/client_golang/prometheus+ flag from github.com/spf13/pflag+ - fmt from archive/tar+ + fmt from compress/flate+ go/ast from go/doc+ go/build/constraint from go/parser go/doc from k8s.io/apimachinery/pkg/runtime go/doc/comment from go/doc + go/internal/scannerhooks from go/parser+ go/parser from k8s.io/apimachinery/pkg/runtime go/scanner from go/ast+ go/token from go/ast+ hash from compress/zlib+ hash/adler32 from compress/zlib hash/crc32 from compress/gzip+ - hash/fnv from google.golang.org/protobuf/internal/detrand + hash/fnv from google.golang.org/protobuf/internal/detrand+ hash/maphash from go4.org/mem html from html/template+ html/template from tailscale.com/util/eventbus @@ -1150,11 +1113,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/filepathlite from os+ internal/fmtsort from fmt+ internal/goarch from crypto/internal/fips140deps/cpu+ - internal/godebug from archive/tar+ + internal/godebug from crypto/internal/fips140deps/godebug+ internal/godebugs from internal/godebug+ - internal/goexperiment from hash/maphash+ + internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ - internal/itoa from internal/poll+ internal/lazyregexp from go/doc internal/msan from internal/runtime/maps+ internal/nettrace from net+ @@ -1164,26 +1126,35 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/profilerecord from runtime+ internal/race from internal/poll+ internal/reflectlite from context+ + D internal/routebsd from net internal/runtime/atomic from internal/runtime/exithook+ + L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime + internal/runtime/gc from runtime+ + internal/runtime/gc/scan from runtime internal/runtime/maps from reflect+ internal/runtime/math from internal/runtime/maps+ + internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ - L internal/runtime/syscall from runtime+ - W internal/saferio from debug/pe + L internal/runtime/syscall/linux from internal/runtime/cgroup+ + W internal/runtime/syscall/windows from internal/syscall/windows+ + internal/saferio from debug/pe+ internal/singleflight from net + internal/strconv from internal/poll+ internal/stringslite from embed+ internal/sync from sync+ + internal/synctest from sync internal/syscall/execenv from os+ LD internal/syscall/unix from crypto/internal/sysrand+ W internal/syscall/windows from crypto/internal/sysrand+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os + internal/trace/tracev2 from runtime+ internal/unsafeheader from internal/reflectlite+ - io from archive/tar+ - io/fs from archive/tar+ - io/ioutil from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + io from bufio+ + io/fs from crypto/x509+ + io/ioutil from github.com/google/gnostic-models/compiler+ iter from go/ast+ log from expvar+ log/internal from log+ @@ -1191,52 +1162,54 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ log/slog/internal from log/slog log/slog/internal/buffer from log/slog maps from sigs.k8s.io/controller-runtime/pkg/predicate+ - math from archive/tar+ + math from compress/flate+ math/big from crypto/dsa+ math/bits from compress/flate+ - math/rand from github.com/google/go-cmp/cmp+ - math/rand/v2 from tailscale.com/derp+ + math/rand from github.com/fxamacker/cbor/v2+ + math/rand/v2 from crypto/ecdsa+ mime from github.com/prometheus/common/expfmt+ mime/multipart from github.com/go-openapi/swag+ mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ - net/http/httptrace from github.com/prometheus-community/pro-bing+ - net/http/httputil from github.com/aws/smithy-go/transport/http+ + net/http/httptrace from github.com/prometheus/client_golang/prometheus/promhttp+ + net/http/httputil from tailscale.com/client/web+ net/http/internal from net/http+ net/http/internal/ascii from net/http+ + net/http/internal/httpcommon from net/http net/http/pprof from sigs.k8s.io/controller-runtime/pkg/manager+ net/netip from github.com/gaissmai/bart+ - net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ + net/textproto from github.com/coder/websocket+ net/url from crypto/x509+ os from crypto/internal/sysrand+ - os/exec from github.com/aws/aws-sdk-go-v2/credentials/processcreds+ + os/exec from github.com/godbus/dbus/v5+ os/signal from sigs.k8s.io/controller-runtime/pkg/manager/signals - os/user from archive/tar+ - path from archive/tar+ - path/filepath from archive/tar+ - reflect from archive/tar+ - regexp from github.com/aws/aws-sdk-go-v2/internal/endpoints+ + os/user from github.com/godbus/dbus/v5+ + path from debug/dwarf+ + path/filepath from crypto/x509+ + reflect from database/sql+ + regexp from github.com/davecgh/go-spew/spew+ regexp/syntax from regexp - runtime from archive/tar+ - runtime/debug from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+ + runtime from crypto/internal/fips140+ + runtime/debug from github.com/klauspost/compress/zstd+ runtime/metrics from github.com/prometheus/client_golang/prometheus+ runtime/pprof from net/http/pprof+ runtime/trace from net/http/pprof slices from encoding/base32+ sort from compress/flate+ - strconv from archive/tar+ - strings from archive/tar+ - sync from archive/tar+ + strconv from compress/flate+ + strings from bufio+ + W structs from internal/syscall/windows + sync from compress/flate+ sync/atomic from context+ - syscall from archive/tar+ + syscall from crypto/internal/sysrand+ text/tabwriter from k8s.io/apimachinery/pkg/util/diff+ text/template from html/template text/template/parse from html/template+ - time from archive/tar+ + time from compress/gzip+ unicode from bytes+ unicode/utf16 from crypto/x509+ unicode/utf8 from bufio+ unique from net/netip unsafe from bytes+ - weak from unique + weak from unique+ diff --git a/cmd/k8s-operator/deploy/chart/Chart.yaml b/cmd/k8s-operator/deploy/chart/Chart.yaml index 363d87d15954a..b16fc4c37fb8a 100644 --- a/cmd/k8s-operator/deploy/chart/Chart.yaml +++ b/cmd/k8s-operator/deploy/chart/Chart.yaml @@ -1,4 +1,4 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause apiVersion: v2 @@ -26,4 +26,4 @@ maintainers: version: 0.1.0 # appVersion will be set to Tailscale repo tag at release time. -appVersion: "unstable" +appVersion: "stable" diff --git a/cmd/k8s-operator/deploy/chart/templates/.gitignore b/cmd/k8s-operator/deploy/chart/templates/.gitignore new file mode 100644 index 0000000000000..185ea9e2be316 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/.gitignore @@ -0,0 +1,12 @@ +# Don't add helm chart CRDs to git. Canonical CRD files live in +# cmd/k8s-operator/deploy/crds. +# +# Generate for local usage with: +# go run tailscale.com/cmd/k8s-operator/generate helmcrd +/connector.yaml +/dnsconfig.yaml +/proxyclass.yaml +/proxygroup.yaml +/recorder.yaml +/tailnet.yaml +/proxygrouppolicy.yaml diff --git a/cmd/k8s-operator/deploy/chart/templates/NOTES.txt b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt new file mode 100644 index 0000000000000..a1a351c5e526a --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt @@ -0,0 +1,37 @@ +{{/* +Fail on presence of removed TS_EXPERIMENTAL_KUBE_API_EVENTS extraEnv var. +*/}} +{{- $removed := "TS_EXPERIMENTAL_KUBE_API_EVENTS" -}} +{{- range .Values.operatorConfig.extraEnv }} + {{- if and .name (eq .name $removed) (eq .value "true") -}} + {{- fail (printf "ERROR: operatorConfig.extraEnv.%s has been removed. Use ACLs instead." $removed) -}} + {{- end -}} +{{- end -}} + +You have successfully installed the Tailscale Kubernetes Operator! + +Once connected, the operator should appear as a device within the Tailscale admin console: +https://login.tailscale.com/admin/machines + +If you have not used the Tailscale operator before, here are some examples to try out: + +* Private Kubernetes API access and authorization using the API server proxy + https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy + +* Private access to cluster Services using an ingress proxy + https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress + +* Private access to the cluster's available subnets using a subnet router + https://tailscale.com/kb/1441/kubernetes-operator-connector + +You can also explore the CRDs, operator, and associated resources within the {{ .Release.Namespace }} namespace: + +$ kubectl explain connector +$ kubectl explain proxygroup +$ kubectl explain proxyclass +$ kubectl explain recorder +$ kubectl explain dnsconfig + +If you're interested to explore what resources were created: + +$ kubectl --namespace={{ .Release.Namespace }} get all -l app.kubernetes.io/managed-by=Helm diff --git a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml index 072ecf6d22e2f..2ca4f398ad2da 100644 --- a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml @@ -1,7 +1,16 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause -{{ if eq .Values.apiServerProxyConfig.mode "true" }} +# If old setting used, enable both old (operator) and new (ProxyGroup) workflows. +# If new setting used, enable only new workflow. +{{ if or (eq (toString .Values.apiServerProxyConfig.mode) "true") + (eq (toString .Values.apiServerProxyConfig.allowImpersonation) "true") }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -16,9 +25,14 @@ kind: ClusterRoleBinding metadata: name: tailscale-auth-proxy subjects: +{{- if eq (toString .Values.apiServerProxyConfig.mode) "true" }} - kind: ServiceAccount name: operator namespace: {{ .Release.Namespace }} +{{- end }} +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml index 1b9b97186b6ca..8f835be39ef7b 100644 --- a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -1,4 +1,4 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause apiVersion: apps/v1 @@ -6,6 +6,9 @@ kind: Deployment metadata: name: operator namespace: {{ .Release.Namespace }} + {{- if .Values.annotations }} + annotations: {{- toYaml .Values.annotations | nindent 4 }} + {{- end }} spec: replicas: 1 strategy: @@ -35,13 +38,23 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if .Values.oauthSecretVolume }} + - name: oauth + {{- toYaml .Values.oauthSecretVolume | nindent 10 }} + {{- else if .Values.oauth.audience }} + - name: oidc-jwt + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: {{ .Values.oauth.audience }} + expirationSeconds: 3600 + path: token + {{- else }} - name: oauth - {{- with .Values.oauthSecretVolume }} - {{- toYaml . | nindent 10 }} - {{- else }} secret: secretName: operator-oauth - {{- end }} + {{- end }} containers: - name: operator {{- with .Values.operatorConfig.securityContext }} @@ -68,10 +81,28 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_SERVICE_ACCOUNT_NAME + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_LOGIN_SERVER + value: {{ .Values.loginServer }} + - name: OPERATOR_INGRESS_CLASS_NAME + value: {{ .Values.ingressClass.name }} + {{- if .Values.oauthSecretVolume }} + - name: CLIENT_ID_FILE + value: /oauth/client_id + - name: CLIENT_SECRET_FILE + value: /oauth/client_secret + {{- else if .Values.oauth.audience }} + - name: CLIENT_ID + value: {{ .Values.oauth.clientId }} + {{- else }} - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE value: /oauth/client_secret + {{- end }} {{- $proxyTag := printf ":%s" ( .Values.proxyConfig.image.tag | default .Chart.AppVersion )}} - name: PROXY_IMAGE value: {{ coalesce .Values.proxyConfig.image.repo .Values.proxyConfig.image.repository }}{{- if .Values.proxyConfig.image.digest -}}{{ printf "@%s" .Values.proxyConfig.image.digest}}{{- else -}}{{ printf "%s" $proxyTag }}{{- end }} @@ -97,9 +128,19 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: - - name: oauth - mountPath: /oauth - readOnly: true + {{- if .Values.oauthSecretVolume }} + - name: oauth + mountPath: /oauth + readOnly: true + {{- else if .Values.oauth.audience }} + - name: oidc-jwt + mountPath: /var/run/secrets/tailscale/serviceaccount + readOnly: true + {{- else }} + - name: oauth + mountPath: /oauth + readOnly: true + {{- end }} {{- with .Values.operatorConfig.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -112,3 +153,6 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.operatorConfig.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} diff --git a/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml index 208d58ee10f08..54851955db67a 100644 --- a/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: - name: tailscale # class name currently can not be changed + name: {{ .Values.ingressClass.name }} annotations: {} # we do not support default IngressClass annotation https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class spec: controller: tailscale.com/ts-ingress # controller name currently can not be changed diff --git a/cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml b/cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml index b44fde0a17b49..34928d6dcd6c8 100644 --- a/cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/oauth-secret.yaml @@ -1,7 +1,7 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause -{{ if and .Values.oauth .Values.oauth.clientId -}} +{{ if and .Values.oauth .Values.oauth.clientId (not .Values.oauth.audience) -}} apiVersion: v1 kind: Secret metadata: diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 00d8318acdce4..08dea80a5f13a 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -1,4 +1,4 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause apiVersion: v1 @@ -16,6 +16,9 @@ kind: ClusterRole metadata: name: tailscale-operator rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events", "services", "services/status"] verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] @@ -34,6 +37,12 @@ rules: - apiGroups: ["tailscale.com"] resources: ["dnsconfigs", "dnsconfigs/status"] verbs: ["get", "list", "watch", "update"] +- apiGroups: ["tailscale.com"] + resources: ["tailnets", "tailnets/status"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["tailscale.com"] + resources: ["proxygrouppolicies", "proxygrouppolicies/status"] + verbs: ["get", "list", "watch", "update"] - apiGroups: ["tailscale.com"] resources: ["recorders", "recorders/status"] verbs: ["get", "list", "watch", "update"] @@ -41,6 +50,9 @@ rules: resources: ["customresourcedefinitions"] verbs: ["get", "list", "watch"] resourceNames: ["servicemonitors.monitoring.coreos.com"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingadmissionpolicies", "validatingadmissionpolicybindings"] + verbs: ["list", "create", "delete", "update", "get", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -64,6 +76,10 @@ rules: - apiGroups: [""] resources: ["secrets", "serviceaccounts", "configmaps"] verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] +- apiGroups: [""] + resources: ["serviceaccounts/token"] + resourceNames: ["operator"] + verbs: ["create"] - apiGroups: [""] resources: ["pods"] verbs: ["get","list","watch", "update"] diff --git a/cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml index fa552a7c7e39a..89d6736b790a1 100644 --- a/cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/proxy-rbac.yaml @@ -1,4 +1,4 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause apiVersion: v1 diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 2d1effc255dc5..f5591e856ba9c 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -1,13 +1,23 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause -# Operator oauth credentials. If set a Kubernetes Secret with the provided -# values will be created in the operator namespace. If unset a Secret named -# operator-oauth must be precreated or oauthSecretVolume needs to be adjusted. -# This block will be overridden by oauthSecretVolume, if set. -oauth: {} - # clientId: "" - # clientSecret: "" +# Operator oauth credentials. If unset a Secret named operator-oauth must be +# precreated or oauthSecretVolume needs to be adjusted. This block will be +# overridden by oauthSecretVolume, if set. +oauth: + # The Client ID the operator will authenticate with. + clientId: "" + # If set a Kubernetes Secret with the provided value will be created in + # the operator namespace, and mounted into the operator Pod. Takes precedence + # over oauth.audience. + clientSecret: "" + # The audience for oauth.clientId if using a workload identity federation + # OAuth client. Mutually exclusive with oauth.clientSecret. + # See https://tailscale.com/kb/1581/workload-identity-federation. + audience: "" + +# URL of the control plane to be used by all resources managed by the operator. +loginServer: "" # Secret volume. # If set it defines the volume the oauth secrets will be mounted from. @@ -52,6 +62,9 @@ operatorConfig: resources: {} + # Specifies annotations for deployment + annotations: {} + podAnnotations: {} podLabels: {} @@ -62,6 +75,8 @@ operatorConfig: affinity: {} + priorityClassName: "" + podSecurityContext: {} securityContext: {} @@ -74,6 +89,10 @@ operatorConfig: # In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here ingressClass: + # Allows for customization of the ingress class name used by the operator to identify ingresses to reconcile. This does + # not allow multiple operator instances to manage different ingresses, but provides an onboarding route for users that + # may have previously set up ingress classes named "tailscale" prior to using the operator. + name: "tailscale" enabled: true # proxyConfig contains configuraton that will be applied to any ingress/egress @@ -85,6 +104,13 @@ ingressClass: # If you need more configuration options, take a look at ProxyClass: # https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource proxyConfig: + # Configure the proxy image to use instead of the default tailscale/tailscale:latest. + # Applying a ProxyClass with `spec.statefulSet.pod.tailscaleContainer.image` + # set will override any defaults here. + # + # Note that ProxyGroups of type "kube-apiserver" use a different default image, + # tailscale/k8s-proxy:latest, and it is currently only possible to override + # that image via the same ProxyClass field. image: # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale. repository: tailscale/tailscale @@ -108,6 +134,15 @@ proxyConfig: # Kubernetes API server. # https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy apiServerProxyConfig: + # Set to "true" to create the ClusterRole permissions required for the API + # server proxy's auth mode. In auth mode, the API server proxy impersonates + # groups and users based on tailnet ACL grants. Required for ProxyGroups of + # type "kube-apiserver" running in auth mode. + allowImpersonation: "false" # "true", "false" + + # If true or noauth, the operator will run an in-process API server proxy. + # You can deploy a ProxyGroup of type "kube-apiserver" to run a high + # availability set of API server proxies instead. mode: "false" # "true", "false", "noauth" imagePullSecrets: [] diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml index d645e39228062..03c51c7553bf9 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml @@ -115,9 +115,19 @@ spec: Connector node. If unset, hostname defaults to -connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 - and 63 characters long. + and 63 characters long. This field should only be used when creating a connector + with an unspecified number of replicas, or a single replica. type: string pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$ + hostnamePrefix: + description: |- + HostnamePrefix specifies the hostname prefix for each + replica. Each device will have the integer number + from its StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + type: string + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that @@ -125,6 +135,14 @@ spec: resources created for this Connector. If unset, the operator will create resources with the default configuration. type: string + replicas: + description: |- + Replicas specifies how many devices to create. Set this to enable + high availability for app connectors, subnet routers, or exit nodes. + https://tailscale.com/kb/1115/high-availability. Defaults to 1. + type: integer + format: int32 + minimum: 0 subnetRouter: description: |- SubnetRouter defines subnet routes that the Connector device should @@ -163,11 +181,23 @@ spec: items: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: Connector tailnet is immutable x-kubernetes-validations: - rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector) message: A Connector needs to have at least one of exit node, subnet router or app connector configured. - rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))' message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields. + - rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)' + message: The hostname field cannot be specified when replicas is greater than 1. + - rule: '!(has(self.hostname) && has(self.hostnamePrefix))' + message: The hostname and hostnamePrefix fields are mutually exclusive. status: description: |- ConnectorStatus describes the status of the Connector. This is set @@ -235,11 +265,32 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + devices: + description: Devices contains information on each device managed by the Connector resource. + type: array + items: + type: object + properties: + hostname: + description: |- + Hostname is the fully qualified domain name of the Connector replica. + If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the + node. + type: string + tailnetIPs: + description: |- + TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) + assigned to the Connector replica. + type: array + items: + type: string hostname: description: |- Hostname is the fully qualified domain name of the Connector node. If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the - node. + node. When using multiple replicas, this field will be populated with the + first replica's hostname. Use the Hostnames field for the full list + of hostnames. type: string isAppConnector: description: IsAppConnector is set to true if the Connector acts as an app connector. diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml index 268d978c15f37..4d6422ede46ec 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml @@ -52,7 +52,6 @@ spec: using its MagicDNS name, you must also annotate the Ingress resource with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation to ensure that the proxy created for the Ingress listens on its Pod IP address. - NB: Clusters where Pods get assigned IPv6 addresses only are currently not supported. type: object required: - spec @@ -101,6 +100,939 @@ spec: tag: description: Tag defaults to unstable. type: string + pod: + description: Pod configuration. + type: object + properties: + affinity: + description: If specified, applies affinity rules to the pods deployed by the DNSConfig resource. + type: object + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + type: array + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + type: object + required: + - preference + - weight + properties: + preference: + description: A node selector term, associated with the corresponding weight. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + type: integer + format: int32 + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + type: object + required: + - nodeSelectorTerms + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + type: array + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + x-kubernetes-map-type: atomic + x-kubernetes-list-type: atomic + x-kubernetes-map-type: atomic + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + type: integer + format: int32 + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + x-kubernetes-list-type: atomic + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + type: integer + format: int32 + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + type: array + items: + type: string + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + type: array + items: + type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic + matchLabels: + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + x-kubernetes-list-type: atomic + nodeSelector: + description: If specified, applies node selector rules to the pods deployed by the DNSConfig resource. + type: object + additionalProperties: + type: string + tolerations: + description: If specified, applies tolerations to the pods deployed by the DNSConfig resource. + type: array + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + type: object + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + type: integer + format: int64 + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + replicas: + description: Replicas specifies how many Pods to create. Defaults to 1. + type: integer + format: int32 + minimum: 0 + service: + description: Service configuration. + type: object + properties: + clusterIP: + description: ClusterIP sets the static IP of the service used by the nameserver. + type: string status: description: |- Status describes the status of the DNSConfig. This is set @@ -172,7 +1104,7 @@ spec: ip: description: |- IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - Currently you must manually update your cluster DNS config to add + Currently, you must manually update your cluster DNS config to add this address as a stub nameserver for ts.net for cluster workloads to be able to resolve MagicDNS names associated with egress or Ingress proxies. diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 1541234755029..d25915e987760 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -431,7 +431,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -446,7 +445,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -603,7 +601,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -618,7 +615,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -703,8 +699,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: @@ -776,7 +772,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -791,7 +786,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -948,7 +942,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -963,7 +956,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -1046,6 +1038,62 @@ spec: type: object additionalProperties: type: string + dnsConfig: + description: |- + DNSConfig defines DNS parameters for the proxy Pod in addition to those generated from DNSPolicy. + When DNSPolicy is set to "None", DNSConfig must be specified. + https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config + type: object + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + type: array + items: + type: string + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + type: array + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + type: object + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's value. + type: string + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + type: array + items: + type: string + x-kubernetes-list-type: atomic + dnsPolicy: + description: |- + DNSPolicy defines how DNS will be configured for the proxy Pod. + By default the Tailscale Kubernetes Operator does not set a DNS policy (uses cluster default). + https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + type: string + enum: + - ClusterFirstWithHostNet + - ClusterFirst + - Default + - None imagePullSecrets: description: |- Proxy Pod's image pull Secrets. @@ -1093,6 +1141,12 @@ spec: type: object additionalProperties: type: string + priorityClassName: + description: |- + PriorityClassName for the proxy Pod. + By default Tailscale Kubernetes operator does not apply any priority class. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: string securityContext: description: |- Proxy Pod's security context. @@ -1379,12 +1433,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -1411,7 +1474,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -1655,7 +1718,9 @@ spec: PodSecurityContext, the value specified in SecurityContext takes precedence. type: string tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". type: object properties: debug: @@ -1709,12 +1774,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -1741,7 +1815,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2156,7 +2230,6 @@ spec: - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string nodeTaintsPolicy: description: |- @@ -2167,7 +2240,6 @@ spec: - Ignore: node taints are ignored. All nodes are included. If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string topologyKey: description: |- @@ -2203,6 +2275,51 @@ spec: won't make it *more* imbalanced. It's a required field. type: string + staticEndpoints: + description: |- + Configuration for 'static endpoints' on proxies in order to facilitate + direct connections from other devices on the tailnet. + See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + type: object + required: + - nodePort + properties: + nodePort: + description: The configuration for static endpoints using NodePort Services. + type: object + required: + - ports + properties: + ports: + description: |- + The port ranges from which the operator will select NodePorts for the Services. + You must ensure that firewall rules allow UDP ingress traffic for these ports + to the node's external IPs. + The ports must be in the range of service node ports for the cluster (default `30000-32767`). + See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + type: array + minItems: 1 + items: + type: object + required: + - port + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be used. This field cannot be defined if the port field is not defined. + The endPort must be either unset, or equal or greater than port. + type: integer + port: + description: port represents a port selected to be used. This is a required field. + type: integer + selector: + description: |- + A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + by the ProxyGroup as Static Endpoints. + type: object + additionalProperties: + type: string tailscale: description: |- TailscaleConfig contains options to configure the tailscale-specific diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygrouppolicies.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygrouppolicies.yaml new file mode 100644 index 0000000000000..d1425fba80165 --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygrouppolicies.yaml @@ -0,0 +1,135 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: proxygrouppolicies.tailscale.com +spec: + group: tailscale.com + names: + kind: ProxyGroupPolicy + listKind: ProxyGroupPolicyList + plural: proxygrouppolicies + shortNames: + - pgp + singular: proxygrouppolicy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - metadata + - spec + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the ProxyGroupPolicy. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + type: object + properties: + egress: + description: |- + Names of ProxyGroup resources that can be used by Service resources within this namespace. An empty list + denotes that no egress via ProxyGroups is allowed within this namespace. + type: array + items: + type: string + ingress: + description: |- + Names of ProxyGroup resources that can be used by Ingress resources within this namespace. An empty list + denotes that no ingress via ProxyGroups is allowed within this namespace. + type: array + items: + type: string + status: + description: |- + Status describes the status of the ProxyGroupPolicy. This is set + and managed by the Tailscale operator. + type: object + properties: + conditions: + type: array + items: + description: Condition contains details for one aspect of the current state of this API Resource. + type: object + required: + - lastTransitionTime + - message + - reason + - status + - type + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + type: string + format: date-time + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + type: string + maxLength: 32768 + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + type: integer + format: int64 + minimum: 0 + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + type: string + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + description: status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 4b9149e23e55b..0254f01b8f0bf 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -20,6 +20,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver. + jsonPath: .status.url + name: URL + type: string - description: ProxyGroup type. jsonPath: .spec.type name: Type @@ -32,15 +36,22 @@ spec: openAPIV3Schema: description: |- ProxyGroup defines a set of Tailscale devices that will act as proxies. - Currently only egress ProxyGroups are supported. + Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver + proxies. In addition to running a highly available set of proxies, ingress + and egress ProxyGroups also allow for serving many annotated Services from a + single set of proxies to minimise resource consumption. + + For ingress and egress, use the tailscale.com/proxy-group annotation on a + Service to specify that the proxy should be implemented by a ProxyGroup + instead of a single dedicated proxy. - Use the tailscale.com/proxy-group annotation on a Service to specify that - the egress proxy should be implemented by a ProxyGroup instead of a single - dedicated proxy. In addition to running a highly available set of proxies, - ProxyGroup also allows for serving many annotated Services from a single - set of proxies to minimise resource consumption. + More info: + * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress - More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + For kube-apiserver, the ProxyGroup is a standalone resource. Use the + spec.kubeAPIServer field to configure options specific to the kube-apiserver + ProxyGroup type. type: object required: - spec @@ -77,6 +88,30 @@ spec: must not start with a dash and must be between 1 and 62 characters long. type: string pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + type: object + properties: + hostname: + description: |- + Hostname is the hostname with which to expose the Kubernetes API server + proxies. Must be a valid DNS label no longer than 63 characters. If not + specified, the name of the ProxyGroup is used as the hostname. Must be + unique across the whole tailnet. + type: string + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + type: string + enum: + - auth + - noauth proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that contains @@ -104,14 +139,23 @@ spec: items: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: ProxyGroup tailnet is immutable type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. Type is immutable once a ProxyGroup is created. type: string enum: - egress - ingress + - kube-apiserver x-kubernetes-validations: - rule: self == oldSelf message: ProxyGroup type is immutable @@ -124,7 +168,20 @@ spec: conditions: description: |- List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`. + resources. Known condition types include `ProxyGroupReady` and + `ProxyGroupAvailable`. + + * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + all expected conditions are true. + * `ProxyGroupAvailable` indicates that at least one proxy is ready to + serve traffic. + + For ProxyGroups of type kube-apiserver, there are two additional conditions: + + * `KubeAPIServerProxyConfigured` indicates that at least one API server + proxy is configured and ready to serve traffic. + * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + valid. type: array items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -196,6 +253,11 @@ spec: If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the node. type: string + staticEndpoints: + description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + type: array + items: + type: string tailnetIPs: description: |- TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) @@ -206,6 +268,11 @@ spec: x-kubernetes-list-map-keys: - hostname x-kubernetes-list-type: map + url: + description: |- + URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + any. Only applies to ProxyGroups of type kube-apiserver. + type: string served: true storage: true subresources: diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml index 0f3dcfcca52c8..ca43a72a557f2 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml @@ -68,6 +68,12 @@ spec: Corresponds to --ui tsrecorder flag https://tailscale.com/kb/1246/tailscale-ssh-session-recording#deploy-a-recorder-node. Required if S3 storage is not set up, to ensure that recordings are accessible. type: boolean + replicas: + description: Replicas specifies how many instances of tsrecorder to run. Defaults to 1. + type: integer + format: int32 + default: 1 + minimum: 0 statefulSet: description: |- Configuration parameters for the Recorder's StatefulSet. The operator @@ -375,7 +381,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -390,7 +395,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -547,7 +551,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -562,7 +565,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -647,8 +649,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: @@ -720,7 +722,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -735,7 +736,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -892,7 +892,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -907,7 +906,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). type: array items: type: string @@ -1052,7 +1050,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -1683,6 +1681,17 @@ spec: items: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: Recorder tailnet is immutable + x-kubernetes-validations: + - rule: '!(self.replicas > 1 && (!has(self.storage) || !has(self.storage.s3)))' + message: S3 storage must be used when deploying multiple Recorder replicas status: description: |- RecorderStatus describes the status of the recorder. This is set diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml new file mode 100644 index 0000000000000..5e6dd5880f770 --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml @@ -0,0 +1,144 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: tailnets.tailscale.com +spec: + group: tailscale.com + names: + kind: Tailnet + listKind: TailnetList + plural: tailnets + shortNames: + - tn + singular: tailnet + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Status of the deployed Tailnet resources. + jsonPath: .status.conditions[?(@.type == "TailnetReady")].reason + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - metadata + - spec + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the Tailnet. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + type: object + required: + - credentials + properties: + credentials: + description: Denotes the location of the credentials to use for authenticating with this Tailnet. + type: object + required: + - secretName + properties: + secretName: + description: |- + The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always + contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate + via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth + client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API + token. "client_secret" and "audience" are mutually exclusive. + type: string + loginUrl: + description: URL of the control plane to be used by all resources managed by the operator using this Tailnet. + type: string + status: + description: |- + Status describes the status of the Tailnet. This is set + and managed by the Tailscale operator. + type: object + properties: + conditions: + type: array + items: + description: Condition contains details for one aspect of the current state of this API Resource. + type: object + required: + - lastTransitionTime + - message + - reason + - status + - type + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + type: string + format: date-time + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + type: string + maxLength: 32768 + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + type: integer + format: int64 + minimum: 0 + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + type: string + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + description: status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/examples/connector.yaml b/cmd/k8s-operator/deploy/examples/connector.yaml index d29f27cf51c98..a025eef98cd26 100644 --- a/cmd/k8s-operator/deploy/examples/connector.yaml +++ b/cmd/k8s-operator/deploy/examples/connector.yaml @@ -1,9 +1,10 @@ # Before applying ensure that the operator owns tag:prod. # https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. -# To set up autoapproval set tag:prod as approver for 10.40.0.0/14 route and exit node. +# To set up autoapproval set tag:prod as approver for 10.40.0.0/14 route. # Otherwise approve it manually in Machines panel once the # ts-prod Tailscale node has been created. # See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes +# For an exit node example, see exitnode.yaml apiVersion: tailscale.com/v1alpha1 kind: Connector metadata: @@ -11,9 +12,9 @@ metadata: spec: tags: - "tag:prod" - hostname: ts-prod + hostnamePrefix: ts-prod + replicas: 2 subnetRouter: advertiseRoutes: - "10.40.0.0/14" - "192.168.0.0/14" - exitNode: true diff --git a/cmd/k8s-operator/deploy/examples/exitnode.yaml b/cmd/k8s-operator/deploy/examples/exitnode.yaml new file mode 100644 index 0000000000000..b2ce516cd98bf --- /dev/null +++ b/cmd/k8s-operator/deploy/examples/exitnode.yaml @@ -0,0 +1,26 @@ +# Before applying ensure that the operator owns tag:k8s-operator +# To use both subnet routing and exit node on the same cluster, deploy a separate +# Connector resource for each. +# See connector.yaml for a subnet router example. +# See: https://tailscale.com/kb/1441/kubernetes-operator-connector +--- +apiVersion: tailscale.com/v1alpha1 +kind: Connector +metadata: + name: exit-node +spec: + # Exit node configuration - allows Tailscale clients to route all internet traffic through this Connector + exitNode: true + + # High availability: 2 replicas for redundancy + # Note: Must use hostnamePrefix (not hostname) when replicas > 1 + replicas: 2 + + # Hostname prefix for the exit node devices + # Devices will be named: exit-node-0, exit-node-1 + hostnamePrefix: exit-node + + # Tailscale tags for ACL policy management + tags: + - tag:k8s-operator + diff --git a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml index ddbdda32e476e..2dc9cad228ffa 100644 --- a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml @@ -1,6 +1,12 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: tailscale +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -18,6 +24,9 @@ subjects: - kind: ServiceAccount name: operator namespace: tailscale +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: tailscale roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 1d910cf92c6c6..8068923d2f4e5 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -1,4 +1,4 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause apiVersion: v1 @@ -140,9 +140,19 @@ spec: Connector node. If unset, hostname defaults to -connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 - and 63 characters long. + and 63 characters long. This field should only be used when creating a connector + with an unspecified number of replicas, or a single replica. pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$ type: string + hostnamePrefix: + description: |- + HostnamePrefix specifies the hostname prefix for each + replica. Each device will have the integer number + from its StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + type: string proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that @@ -150,6 +160,14 @@ spec: resources created for this Connector. If unset, the operator will create resources with the default configuration. type: string + replicas: + description: |- + Replicas specifies how many devices to create. Set this to enable + high availability for app connectors, subnet routers, or exit nodes. + https://tailscale.com/kb/1115/high-availability. Defaults to 1. + format: int32 + minimum: 0 + type: integer subnetRouter: description: |- SubnetRouter defines subnet routes that the Connector device should @@ -188,12 +206,24 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: string type: array + tailnet: + description: |- + Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - message: Connector tailnet is immutable + rule: self == oldSelf type: object x-kubernetes-validations: - message: A Connector needs to have at least one of exit node, subnet router or app connector configured. rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector) - message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields. rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))' + - message: The hostname field cannot be specified when replicas is greater than 1. + rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)' + - message: The hostname and hostnamePrefix fields are mutually exclusive. + rule: '!(has(self.hostname) && has(self.hostnamePrefix))' status: description: |- ConnectorStatus describes the status of the Connector. This is set @@ -260,11 +290,32 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + devices: + description: Devices contains information on each device managed by the Connector resource. + items: + properties: + hostname: + description: |- + Hostname is the fully qualified domain name of the Connector replica. + If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the + node. + type: string + tailnetIPs: + description: |- + TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) + assigned to the Connector replica. + items: + type: string + type: array + type: object + type: array hostname: description: |- Hostname is the fully qualified domain name of the Connector node. If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the - node. + node. When using multiple replicas, this field will be populated with the + first replica's hostname. Use the Hostnames field for the full list + of hostnames. type: string isAppConnector: description: IsAppConnector is set to true if the Connector acts as an app connector. @@ -347,7 +398,6 @@ spec: using its MagicDNS name, you must also annotate the Ingress resource with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation to ensure that the proxy created for the Ingress listens on its Pod IP address. - NB: Clusters where Pods get assigned IPv6 addresses only are currently not supported. properties: apiVersion: description: |- @@ -382,11 +432,944 @@ spec: image: description: Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. properties: - repo: - description: Repo defaults to tailscale/k8s-nameserver. - type: string - tag: - description: Tag defaults to unstable. + repo: + description: Repo defaults to tailscale/k8s-nameserver. + type: string + tag: + description: Tag defaults to unstable. + type: string + type: object + pod: + description: Pod configuration. + properties: + affinity: + description: If specified, applies affinity rules to the pods deployed by the DNSConfig resource. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + description: If specified, applies node selector rules to the pods deployed by the DNSConfig resource. + type: object + tolerations: + description: If specified, applies tolerations to the pods deployed by the DNSConfig resource. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + replicas: + description: Replicas specifies how many Pods to create. Defaults to 1. + format: int32 + minimum: 0 + type: integer + service: + description: Service configuration. + properties: + clusterIP: + description: ClusterIP sets the static IP of the service used by the nameserver. type: string type: object type: object @@ -462,7 +1445,7 @@ spec: ip: description: |- IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - Currently you must manually update your cluster DNS config to add + Currently, you must manually update your cluster DNS config to add this address as a stub nameserver for ts.net for cluster workloads to be able to resolve MagicDNS names associated with egress or Ingress proxies. @@ -895,7 +1878,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -910,7 +1892,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1071,7 +2052,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1086,7 +2066,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1175,8 +2154,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -1240,7 +2219,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1255,7 +2233,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1416,7 +2393,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1431,7 +2407,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1510,16 +2485,72 @@ spec: x-kubernetes-list-type: atomic type: object type: object - annotations: - additionalProperties: - type: string + annotations: + additionalProperties: + type: string + description: |- + Annotations that will be added to the proxy Pod. + Any annotations specified here will be merged with the default + annotations applied to the Pod by the Tailscale Kubernetes operator. + Annotations must be valid Kubernetes annotations. + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + dnsConfig: + description: |- + DNSConfig defines DNS parameters for the proxy Pod in addition to those generated from DNSPolicy. + When DNSPolicy is set to "None", DNSConfig must be specified. + https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: description: |- - Annotations that will be added to the proxy Pod. - Any annotations specified here will be merged with the default - annotations applied to the Pod by the Tailscale Kubernetes operator. - Annotations must be valid Kubernetes annotations. - https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set - type: object + DNSPolicy defines how DNS will be configured for the proxy Pod. + By default the Tailscale Kubernetes Operator does not set a DNS policy (uses cluster default). + https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + enum: + - ClusterFirstWithHostNet + - ClusterFirst + - Default + - None + type: string imagePullSecrets: description: |- Proxy Pod's image pull Secrets. @@ -1567,6 +2598,12 @@ spec: selector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling type: object + priorityClassName: + description: |- + PriorityClassName for the proxy Pod. + By default Tailscale Kubernetes operator does not apply any priority class. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: string securityContext: description: |- Proxy Pod's security context. @@ -1852,12 +2889,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -1883,7 +2929,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2129,7 +3175,9 @@ spec: type: object type: object tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". properties: debug: description: |- @@ -2182,12 +3230,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2213,7 +3270,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -2624,7 +3681,6 @@ spec: - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string nodeTaintsPolicy: description: |- @@ -2635,7 +3691,6 @@ spec: - Ignore: node taints are ignored. All nodes are included. If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string topologyKey: description: |- @@ -2679,6 +3734,51 @@ spec: type: array type: object type: object + staticEndpoints: + description: |- + Configuration for 'static endpoints' on proxies in order to facilitate + direct connections from other devices on the tailnet. + See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + properties: + nodePort: + description: The configuration for static endpoints using NodePort Services. + properties: + ports: + description: |- + The port ranges from which the operator will select NodePorts for the Services. + You must ensure that firewall rules allow UDP ingress traffic for these ports + to the node's external IPs. + The ports must be in the range of service node ports for the cluster (default `30000-32767`). + See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + items: + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be used. This field cannot be defined if the port field is not defined. + The endPort must be either unset, or equal or greater than port. + type: integer + port: + description: port represents a port selected to be used. This is a required field. + type: integer + required: + - port + type: object + minItems: 1 + type: array + selector: + additionalProperties: + type: string + description: |- + A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + by the ProxyGroup as Static Endpoints. + type: object + required: + - ports + type: object + required: + - nodePort + type: object tailscale: description: |- TailscaleConfig contains options to configure the tailscale-specific @@ -2808,6 +3908,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver. + jsonPath: .status.url + name: URL + type: string - description: ProxyGroup type. jsonPath: .spec.type name: Type @@ -2818,17 +3922,274 @@ spec: name: v1alpha1 schema: openAPIV3Schema: - description: |- - ProxyGroup defines a set of Tailscale devices that will act as proxies. - Currently only egress ProxyGroups are supported. - - Use the tailscale.com/proxy-group annotation on a Service to specify that - the egress proxy should be implemented by a ProxyGroup instead of a single - dedicated proxy. In addition to running a highly available set of proxies, - ProxyGroup also allows for serving many annotated Services from a single - set of proxies to minimise resource consumption. - - More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + description: |- + ProxyGroup defines a set of Tailscale devices that will act as proxies. + Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver + proxies. In addition to running a highly available set of proxies, ingress + and egress ProxyGroups also allow for serving many annotated Services from a + single set of proxies to minimise resource consumption. + + For ingress and egress, use the tailscale.com/proxy-group annotation on a + Service to specify that the proxy should be implemented by a ProxyGroup + instead of a single dedicated proxy. + + More info: + * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress + + For kube-apiserver, the ProxyGroup is a standalone resource. Use the + spec.kubeAPIServer field to configure options specific to the kube-apiserver + ProxyGroup type. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec describes the desired ProxyGroup instances. + properties: + hostnamePrefix: + description: |- + HostnamePrefix is the hostname prefix to use for tailnet devices created + by the ProxyGroup. Each device will have the integer number from its + StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + type: string + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + properties: + hostname: + description: |- + Hostname is the hostname with which to expose the Kubernetes API server + proxies. Must be a valid DNS label no longer than 63 characters. If not + specified, the name of the ProxyGroup is used as the hostname. Must be + unique across the whole tailnet. + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ + type: string + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + enum: + - auth + - noauth + type: string + type: object + proxyClass: + description: |- + ProxyClass is the name of the ProxyClass custom resource that contains + configuration options that should be applied to the resources created + for this ProxyGroup. If unset, and there is no default ProxyClass + configured, the operator will create resources with the default + configuration. + type: string + replicas: + description: |- + Replicas specifies how many replicas to create the StatefulSet with. + Defaults to 2. + format: int32 + minimum: 0 + type: integer + tags: + description: |- + Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s]. + If you specify custom tags here, make sure you also make the operator + an owner of these tags. + See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + Tags cannot be changed once a ProxyGroup device has been created. + Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + items: + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + type: string + type: array + tailnet: + description: |- + Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - message: ProxyGroup tailnet is immutable + rule: self == oldSelf + type: + description: |- + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. + Type is immutable once a ProxyGroup is created. + enum: + - egress + - ingress + - kube-apiserver + type: string + x-kubernetes-validations: + - message: ProxyGroup type is immutable + rule: self == oldSelf + required: + - type + type: object + status: + description: |- + ProxyGroupStatus describes the status of the ProxyGroup resources. This is + set and managed by the Tailscale operator. + properties: + conditions: + description: |- + List of status conditions to indicate the status of the ProxyGroup + resources. Known condition types include `ProxyGroupReady` and + `ProxyGroupAvailable`. + + * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + all expected conditions are true. + * `ProxyGroupAvailable` indicates that at least one proxy is ready to + serve traffic. + + For ProxyGroups of type kube-apiserver, there are two additional conditions: + + * `KubeAPIServerProxyConfigured` indicates that at least one API server + proxy is configured and ready to serve traffic. + * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + valid. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + devices: + description: List of tailnet devices associated with the ProxyGroup StatefulSet. + items: + properties: + hostname: + description: |- + Hostname is the fully qualified domain name of the device. + If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the + node. + type: string + staticEndpoints: + description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + items: + type: string + type: array + tailnetIPs: + description: |- + TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) + assigned to the device. + items: + type: string + type: array + required: + - hostname + type: object + type: array + x-kubernetes-list-map-keys: + - hostname + x-kubernetes-list-type: map + url: + description: |- + URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + any. Only applies to ProxyGroups of type kube-apiserver. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: proxygrouppolicies.tailscale.com +spec: + group: tailscale.com + names: + kind: ProxyGroupPolicy + listKind: ProxyGroupPolicyList + plural: proxygrouppolicies + shortNames: + - pgp + singular: proxygrouppolicy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: properties: apiVersion: description: |- @@ -2848,67 +4209,32 @@ spec: metadata: type: object spec: - description: Spec describes the desired ProxyGroup instances. + description: |- + Spec describes the desired state of the ProxyGroupPolicy. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: - hostnamePrefix: - description: |- - HostnamePrefix is the hostname prefix to use for tailnet devices created - by the ProxyGroup. Each device will have the integer number from its - StatefulSet pod appended to this prefix to form the full hostname. - HostnamePrefix can contain lower case letters, numbers and dashes, it - must not start with a dash and must be between 1 and 62 characters long. - pattern: ^[a-z0-9][a-z0-9-]{0,61}$ - type: string - proxyClass: - description: |- - ProxyClass is the name of the ProxyClass custom resource that contains - configuration options that should be applied to the resources created - for this ProxyGroup. If unset, and there is no default ProxyClass - configured, the operator will create resources with the default - configuration. - type: string - replicas: - description: |- - Replicas specifies how many replicas to create the StatefulSet with. - Defaults to 2. - format: int32 - minimum: 0 - type: integer - tags: + egress: description: |- - Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s]. - If you specify custom tags here, make sure you also make the operator - an owner of these tags. - See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. - Tags cannot be changed once a ProxyGroup device has been created. - Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + Names of ProxyGroup resources that can be used by Service resources within this namespace. An empty list + denotes that no egress via ProxyGroups is allowed within this namespace. items: - pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: string type: array - type: + ingress: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. - Type is immutable once a ProxyGroup is created. - enum: - - egress - - ingress - type: string - x-kubernetes-validations: - - message: ProxyGroup type is immutable - rule: self == oldSelf - required: - - type + Names of ProxyGroup resources that can be used by Ingress resources within this namespace. An empty list + denotes that no ingress via ProxyGroups is allowed within this namespace. + items: + type: string + type: array type: object status: description: |- - ProxyGroupStatus describes the status of the ProxyGroup resources. This is - set and managed by the Tailscale operator. + Status describes the status of the ProxyGroupPolicy. This is set + and managed by the Tailscale operator. properties: conditions: - description: |- - List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -2966,32 +4292,9 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map - devices: - description: List of tailnet devices associated with the ProxyGroup StatefulSet. - items: - properties: - hostname: - description: |- - Hostname is the fully qualified domain name of the device. - If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the - node. - type: string - tailnetIPs: - description: |- - TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) - assigned to the device. - items: - type: string - type: array - required: - - hostname - type: object - type: array - x-kubernetes-list-map-keys: - - hostname - x-kubernetes-list-type: map type: object required: + - metadata - spec type: object served: true @@ -3065,6 +4368,12 @@ spec: Corresponds to --ui tsrecorder flag https://tailscale.com/kb/1246/tailscale-ssh-session-recording#deploy-a-recorder-node. Required if S3 storage is not set up, to ensure that recordings are accessible. type: boolean + replicas: + default: 1 + description: Replicas specifies how many instances of tsrecorder to run. Defaults to 1. + format: int32 + minimum: 0 + type: integer statefulSet: description: |- Configuration parameters for the Recorder's StatefulSet. The operator @@ -3360,7 +4669,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3375,7 +4683,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3536,7 +4843,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3551,7 +4857,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3640,8 +4945,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -3705,7 +5010,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3720,7 +5024,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3881,7 +5184,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -3896,7 +5198,6 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -4045,7 +5346,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This is an alpha field and requires enabling the + This field depends on the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -4680,7 +5981,18 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: string type: array + tailnet: + description: |- + Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - message: Recorder tailnet is immutable + rule: self == oldSelf type: object + x-kubernetes-validations: + - message: S3 storage must be used when deploying multiple Recorder replicas + rule: '!(self.replicas > 1 && (!has(self.storage) || !has(self.storage.s3)))' status: description: |- RecorderStatus describes the status of the recorder. This is set @@ -4786,11 +6098,164 @@ spec: subresources: status: {} --- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: tailnets.tailscale.com +spec: + group: tailscale.com + names: + kind: Tailnet + listKind: TailnetList + plural: tailnets + shortNames: + - tn + singular: tailnet + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Status of the deployed Tailnet resources. + jsonPath: .status.conditions[?(@.type == "TailnetReady")].reason + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the Tailnet. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + credentials: + description: Denotes the location of the credentials to use for authenticating with this Tailnet. + properties: + secretName: + description: |- + The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always + contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate + via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth + client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API + token. "client_secret" and "audience" are mutually exclusive. + type: string + required: + - secretName + type: object + loginUrl: + description: URL of the control plane to be used by all resources managed by the operator using this Tailnet. + type: string + required: + - credentials + type: object + status: + description: |- + Status describes the status of the Tailnet. This is set + and managed by the Tailscale operator. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: tailscale-operator rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -4860,6 +6325,26 @@ rules: - list - watch - update + - apiGroups: + - tailscale.com + resources: + - tailnets + - tailnets/status + verbs: + - get + - list + - watch + - update + - apiGroups: + - tailscale.com + resources: + - proxygrouppolicies + - proxygrouppolicies/status + verbs: + - get + - list + - watch + - update - apiGroups: - tailscale.com resources: @@ -4880,6 +6365,18 @@ rules: - get - list - watch + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingadmissionpolicies + - validatingadmissionpolicybindings + verbs: + - list + - create + - delete + - update + - get + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -4915,6 +6412,14 @@ rules: - patch - update - watch + - apiGroups: + - "" + resourceNames: + - operator + resources: + - serviceaccounts/token + verbs: + - create - apiGroups: - "" resources: @@ -5066,12 +6571,20 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_SERVICE_ACCOUNT_NAME + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_LOGIN_SERVER + value: null + - name: OPERATOR_INGRESS_CLASS_NAME + value: tailscale - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE value: /oauth/client_secret - name: PROXY_IMAGE - value: tailscale/tailscale:unstable + value: tailscale/tailscale:stable - name: PROXY_TAGS value: tag:k8s - name: APISERVER_PROXY @@ -5086,7 +6599,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid - image: tailscale/k8s-operator:unstable + image: tailscale/k8s-operator:stable imagePullPolicy: Always name: operator volumeMounts: diff --git a/cmd/k8s-operator/deploy/manifests/proxy.yaml b/cmd/k8s-operator/deploy/manifests/proxy.yaml index 3c9a3eaa36c56..74e36cf788c0f 100644 --- a/cmd/k8s-operator/deploy/manifests/proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/proxy.yaml @@ -16,12 +16,12 @@ spec: privileged: true command: ["/bin/sh", "-c"] args: [sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi] - resources: - requests: - cpu: 1m - memory: 1Mi containers: - name: tailscale + resources: + requests: + cpu: 1m + memory: 1Mi imagePullPolicy: Always env: - name: TS_USERSPACE diff --git a/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml b/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml index a96d4c37ee421..800025e90003b 100644 --- a/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml +++ b/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml @@ -1,3 +1,3 @@ -# Copyright (c) Tailscale Inc & AUTHORS +# Copyright (c) Tailscale Inc & contributors # SPDX-License-Identifier: BSD-3-Clause diff --git a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml index 6617f6d4b52fe..f93ab5855e7b2 100644 --- a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml @@ -10,12 +10,12 @@ spec: deletionGracePeriodSeconds: 10 spec: serviceAccountName: proxies - resources: - requests: - cpu: 1m - memory: 1Mi containers: - name: tailscale + resources: + requests: + cpu: 1m + memory: 1Mi imagePullPolicy: Always env: - name: TS_USERSPACE diff --git a/cmd/k8s-operator/dnsrecords.go b/cmd/k8s-operator/dnsrecords.go index f91dd49ec255e..f926dd684ca8b 100644 --- a/cmd/k8s-operator/dnsrecords.go +++ b/cmd/k8s-operator/dnsrecords.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -22,6 +22,7 @@ import ( "k8s.io/utils/net" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/util/mak" @@ -31,15 +32,19 @@ import ( const ( dnsRecordsRecocilerFinalizer = "tailscale.com/dns-records-reconciler" annotationTSMagicDNSName = "tailscale.com/magic-dnsname" + + // Service types for consistent string usage + serviceTypeIngress = "ingress" + serviceTypeSvc = "svc" ) // dnsRecordsReconciler knows how to update dnsrecords ConfigMap with DNS // records. // The records that it creates are: -// - For tailscale Ingress, a mapping of the Ingress's MagicDNSName to the IP address of -// the ingress proxy Pod. +// - For tailscale Ingress, a mapping of the Ingress's MagicDNSName to the IP addresses +// (both IPv4 and IPv6) of the ingress proxy Pod. // - For egress proxies configured via tailscale.com/tailnet-fqdn annotation, a -// mapping of the tailnet FQDN to the IP address of the egress proxy Pod. +// mapping of the tailnet FQDN to the IP addresses (both IPv4 and IPv6) of the egress proxy Pod. // // Records will only be created if there is exactly one ready // tailscale.com/v1alpha1.DNSConfig instance in the cluster (so that we know @@ -51,7 +56,7 @@ type dnsRecordsReconciler struct { isDefaultLoadBalancer bool // true if operator is the default ingress controller in this cluster } -// Reconcile takes a reconcile.Request for a headless Service fronting a +// Reconcile takes a reconcile.Request for a Service fronting a // tailscale proxy and updates DNS Records in dnsrecords ConfigMap for the // in-cluster ts.net nameserver if required. func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { @@ -59,8 +64,8 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. logger.Debugf("starting reconcile") defer logger.Debugf("reconcile finished") - headlessSvc := new(corev1.Service) - err = dnsRR.Client.Get(ctx, req.NamespacedName, headlessSvc) + proxySvc := new(corev1.Service) + err = dnsRR.Client.Get(ctx, req.NamespacedName, proxySvc) if apierrors.IsNotFound(err) { logger.Debugf("Service not found") return reconcile.Result{}, nil @@ -68,14 +73,14 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get Service: %w", err) } - if !(isManagedByType(headlessSvc, "svc") || isManagedByType(headlessSvc, "ingress")) { - logger.Debugf("Service is not a headless Service for a tailscale ingress or egress proxy; do nothing") + if !(isManagedByType(proxySvc, serviceTypeSvc) || isManagedByType(proxySvc, serviceTypeIngress)) { + logger.Debugf("Service is not a proxy Service for a tailscale ingress or egress proxy; do nothing") return reconcile.Result{}, nil } - if !headlessSvc.DeletionTimestamp.IsZero() { + if !proxySvc.DeletionTimestamp.IsZero() { logger.Debug("Service is being deleted, clean up resources") - return reconcile.Result{}, dnsRR.maybeCleanup(ctx, headlessSvc, logger) + return reconcile.Result{}, dnsRR.maybeCleanup(ctx, proxySvc, logger) } // Check that there is a ts.net nameserver deployed to the cluster by @@ -99,7 +104,7 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. return reconcile.Result{}, nil } - if err := dnsRR.maybeProvision(ctx, headlessSvc, logger); err != nil { + if err := dnsRR.maybeProvision(ctx, proxySvc, logger); err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { logger.Infof("optimistic lock error, retrying: %s", err) } else { @@ -111,37 +116,33 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile. } // maybeProvision ensures that dnsrecords ConfigMap contains a record for the -// proxy associated with the headless Service. +// proxy associated with the Service. // The record is only provisioned if the proxy is for a tailscale Ingress or // egress configured via tailscale.com/tailnet-fqdn annotation. // // For Ingress, the record is a mapping between the MagicDNSName of the Ingress, retrieved from // ingress.status.loadBalancer.ingress.hostname field and the proxy Pod IP addresses -// retrieved from the EndpoinSlice associated with this headless Service, i.e -// Records{IP4: : <[IPs of the ingress proxy Pods]>} +// retrieved from the EndpointSlice associated with this Service, i.e +// Records{IP4: {: <[IPv4 addresses]>}, IP6: {: <[IPv6 addresses]>}} // // For egress, the record is a mapping between tailscale.com/tailnet-fqdn // annotation and the proxy Pod IP addresses, retrieved from the EndpointSlice -// associated with this headless Service, i.e -// Records{IP4: {: <[IPs of the egress proxy Pods]>} +// associated with this Service, i.e +// Records{IP4: {: <[IPv4 addresses]>}, IP6: {: <[IPv6 addresses]>}} +// +// For ProxyGroup egress, the record is a mapping between tailscale.com/magic-dnsname +// annotation and the ClusterIP Service IPs (which provides portmapping), i.e +// Records{IP4: {: <[IPv4 ClusterIPs]>}, IP6: {: <[IPv6 ClusterIPs]>}} // // If records need to be created for this proxy, maybeProvision will also: -// - update the headless Service with a tailscale.com/magic-dnsname annotation -// - update the headless Service with a finalizer -func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error { - if headlessSvc == nil { - logger.Info("[unexpected] maybeProvision called with a nil Service") - return nil - } - isEgressFQDNSvc, err := dnsRR.isSvcForFQDNEgressProxy(ctx, headlessSvc) - if err != nil { - return fmt.Errorf("error checking whether the Service is for an egress proxy: %w", err) - } - if !(isEgressFQDNSvc || isManagedByType(headlessSvc, "ingress")) { +// - update the Service with a tailscale.com/magic-dnsname annotation +// - update the Service with a finalizer +func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) error { + if !dnsRR.isInterestingService(ctx, proxySvc) { logger.Debug("Service is not fronting a proxy that we create DNS records for; do nothing") return nil } - fqdn, err := dnsRR.fqdnForDNSRecord(ctx, headlessSvc, logger) + fqdn, err := dnsRR.fqdnForDNSRecord(ctx, proxySvc, logger) if err != nil { return fmt.Errorf("error determining DNS name for record: %w", err) } @@ -150,18 +151,18 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS return nil // a new reconcile will be triggered once it's added } - oldHeadlessSvc := headlessSvc.DeepCopy() - // Ensure that headless Service is annotated with a finalizer to help + oldProxySvc := proxySvc.DeepCopy() + // Ensure that proxy Service is annotated with a finalizer to help // with records cleanup when proxy resources are deleted. - if !slices.Contains(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer) { - headlessSvc.Finalizers = append(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer) + if !slices.Contains(proxySvc.Finalizers, dnsRecordsRecocilerFinalizer) { + proxySvc.Finalizers = append(proxySvc.Finalizers, dnsRecordsRecocilerFinalizer) } - // Ensure that headless Service is annotated with the current MagicDNS + // Ensure that proxy Service is annotated with the current MagicDNS // name to help with records cleanup when proxy resources are deleted or // MagicDNS name changes. - oldFqdn := headlessSvc.Annotations[annotationTSMagicDNSName] + oldFqdn := proxySvc.Annotations[annotationTSMagicDNSName] if oldFqdn != "" && oldFqdn != fqdn { // i.e user has changed the value of tailscale.com/tailnet-fqdn annotation - logger.Debugf("MagicDNS name has changed, remvoving record for %s", oldFqdn) + logger.Debugf("MagicDNS name has changed, removing record for %s", oldFqdn) updateFunc := func(rec *operatorutils.Records) { delete(rec.IP4, oldFqdn) } @@ -169,58 +170,32 @@ func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessS return fmt.Errorf("error removing record for %s: %w", oldFqdn, err) } } - mak.Set(&headlessSvc.Annotations, annotationTSMagicDNSName, fqdn) - if !apiequality.Semantic.DeepEqual(oldHeadlessSvc, headlessSvc) { + mak.Set(&proxySvc.Annotations, annotationTSMagicDNSName, fqdn) + if !apiequality.Semantic.DeepEqual(oldProxySvc, proxySvc) { logger.Infof("provisioning DNS record for MagicDNS name: %s", fqdn) // this will be printed exactly once - if err := dnsRR.Update(ctx, headlessSvc); err != nil { - return fmt.Errorf("error updating proxy headless Service metadata: %w", err) + if err := dnsRR.Update(ctx, proxySvc); err != nil { + return fmt.Errorf("error updating proxy Service metadata: %w", err) } } - // Get the Pod IP addresses for the proxy from the EndpointSlices for - // the headless Service. The Service can have multiple EndpointSlices - // associated with it, for example in dual-stack clusters. - labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership - var eps = new(discoveryv1.EndpointSliceList) - if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil { - return fmt.Errorf("error listing EndpointSlices for the proxy's headless Service: %w", err) + // Get the IP addresses for the DNS record + ip4s, ip6s, err := dnsRR.getTargetIPs(ctx, proxySvc, logger) + if err != nil { + return fmt.Errorf("error getting target IPs: %w", err) } - if len(eps.Items) == 0 { - logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created") + if len(ip4s) == 0 && len(ip6s) == 0 { + logger.Debugf("No target IP addresses available yet. We will reconcile again once they are available.") return nil } - // Each EndpointSlice for a Service can have a list of endpoints that each - // can have multiple addresses - these are the IP addresses of any Pods - // selected by that Service. Pick all the IPv4 addresses. - // It is also possible that multiple EndpointSlices have overlapping addresses. - // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#duplicate-endpoints - ips := make(set.Set[string], 0) - for _, slice := range eps.Items { - if slice.AddressType != discoveryv1.AddressTypeIPv4 { - logger.Infof("EndpointSlice is for AddressType %s, currently only IPv4 address type is supported", slice.AddressType) - continue + + updateFunc := func(rec *operatorutils.Records) { + if len(ip4s) > 0 { + mak.Set(&rec.IP4, fqdn, ip4s) } - for _, ep := range slice.Endpoints { - if !epIsReady(&ep) { - logger.Debugf("Endpoint with addresses %v appears not ready to receive traffic %v", ep.Addresses, ep.Conditions.String()) - continue - } - for _, ip := range ep.Addresses { - if !net.IsIPv4String(ip) { - logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip) - } else { - ips.Add(ip) - } - } + if len(ip6s) > 0 { + mak.Set(&rec.IP6, fqdn, ip6s) } } - if ips.Len() == 0 { - logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.") - return nil - } - updateFunc := func(rec *operatorutils.Records) { - mak.Set(&rec.IP4, fqdn, ips.Slice()) - } if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil { return fmt.Errorf("error updating DNS records: %w", err) } @@ -243,79 +218,90 @@ func epIsReady(ep *discoveryv1.Endpoint) bool { // has been removed from the Service. If the record is not found in the // ConfigMap, the ConfigMap does not exist, or the Service does not have // tailscale.com/magic-dnsname annotation, just remove the finalizer. -func (h *dnsRecordsReconciler) maybeCleanup(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error { - ix := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer) +func (dnsRR *dnsRecordsReconciler) maybeCleanup(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) error { + ix := slices.Index(proxySvc.Finalizers, dnsRecordsRecocilerFinalizer) if ix == -1 { logger.Debugf("no finalizer, nothing to do") return nil } cm := &corev1.ConfigMap{} - err := h.Client.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: h.tsNamespace}, cm) + err := dnsRR.Client.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm) if apierrors.IsNotFound(err) { - logger.Debug("'dsnrecords' ConfigMap not found") - return h.removeHeadlessSvcFinalizer(ctx, headlessSvc) + logger.Debug("'dnsrecords' ConfigMap not found") + return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) } if err != nil { return fmt.Errorf("error retrieving 'dnsrecords' ConfigMap: %w", err) } if cm.Data == nil { logger.Debug("'dnsrecords' ConfigMap contains no records") - return h.removeHeadlessSvcFinalizer(ctx, headlessSvc) + return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) } _, ok := cm.Data[operatorutils.DNSRecordsCMKey] if !ok { logger.Debug("'dnsrecords' ConfigMap contains no records") - return h.removeHeadlessSvcFinalizer(ctx, headlessSvc) + return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) } - fqdn, _ := headlessSvc.GetAnnotations()[annotationTSMagicDNSName] + fqdn := proxySvc.GetAnnotations()[annotationTSMagicDNSName] if fqdn == "" { - return h.removeHeadlessSvcFinalizer(ctx, headlessSvc) + return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) } logger.Infof("removing DNS record for MagicDNS name %s", fqdn) updateFunc := func(rec *operatorutils.Records) { delete(rec.IP4, fqdn) + if rec.IP6 != nil { + delete(rec.IP6, fqdn) + } } - if err = h.updateDNSConfig(ctx, updateFunc); err != nil { + if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil { return fmt.Errorf("error updating DNS config: %w", err) } - return h.removeHeadlessSvcFinalizer(ctx, headlessSvc) + return dnsRR.removeProxySvcFinalizer(ctx, proxySvc) } -func (dnsRR *dnsRecordsReconciler) removeHeadlessSvcFinalizer(ctx context.Context, headlessSvc *corev1.Service) error { - idx := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer) +func (dnsRR *dnsRecordsReconciler) removeProxySvcFinalizer(ctx context.Context, proxySvc *corev1.Service) error { + idx := slices.Index(proxySvc.Finalizers, dnsRecordsRecocilerFinalizer) if idx == -1 { return nil } - headlessSvc.Finalizers = append(headlessSvc.Finalizers[:idx], headlessSvc.Finalizers[idx+1:]...) - return dnsRR.Update(ctx, headlessSvc) + proxySvc.Finalizers = slices.Delete(proxySvc.Finalizers, idx, idx+1) + return dnsRR.Update(ctx, proxySvc) } -// fqdnForDNSRecord returns MagicDNS name associated with a given headless Service. -// If the headless Service is for a tailscale Ingress proxy, returns ingress.status.loadBalancer.ingress.hostname. -// If the headless Service is for an tailscale egress proxy configured via tailscale.com/tailnet-fqdn annotation, returns the annotation value. -// This function is not expected to be called with headless Services for other +// fqdnForDNSRecord returns MagicDNS name associated with a given proxy Service. +// If the proxy Service is for a tailscale Ingress proxy, returns ingress.status.loadBalancer.ingress.hostname. +// If the proxy Service is for an tailscale egress proxy configured via tailscale.com/tailnet-fqdn annotation, returns the annotation value. +// For ProxyGroup egress Services, returns the tailnet-fqdn annotation from the parent Service. +// This function is not expected to be called with proxy Services for other // proxy types, or any other Services, but it just returns an empty string if // that happens. -func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) (string, error) { - parentName := parentFromObjectLabels(headlessSvc) - if isManagedByType(headlessSvc, "ingress") { +func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) (string, error) { + parentName := parentFromObjectLabels(proxySvc) + if isManagedByType(proxySvc, serviceTypeIngress) { ing := new(networkingv1.Ingress) if err := dnsRR.Get(ctx, parentName, ing); err != nil { return "", err } + if len(ing.Status.LoadBalancer.Ingress) == 0 { return "", nil } + return ing.Status.LoadBalancer.Ingress[0].Hostname, nil } - if isManagedByType(headlessSvc, "svc") { - svc := new(corev1.Service) - if err := dnsRR.Get(ctx, parentName, svc); apierrors.IsNotFound(err) { - logger.Info("[unexpected] parent Service for egress proxy %s not found", headlessSvc.Name) + + if isManagedByType(proxySvc, serviceTypeSvc) { + var svc corev1.Service + + err := dnsRR.Get(ctx, parentName, &svc) + switch { + case apierrors.IsNotFound(err): + logger.Warnf("parent Service for egress proxy %q not found", proxySvc.Name) return "", nil - } else if err != nil { + case err != nil: return "", err } + return svc.Annotations[AnnotationTailnetTargetFQDN], nil } return "", nil @@ -325,28 +311,31 @@ func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, headles // ConfigMap. At this point the in-cluster ts.net nameserver is expected to be // successfully created together with the ConfigMap. func (dnsRR *dnsRecordsReconciler) updateDNSConfig(ctx context.Context, update func(*operatorutils.Records)) error { - cm := &corev1.ConfigMap{} - err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm) - if apierrors.IsNotFound(err) { - dnsRR.logger.Info("[unexpected] dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an isue and attach operator logs.") + var cm corev1.ConfigMap + err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, &cm) + switch { + case apierrors.IsNotFound(err): + dnsRR.logger.Warn("dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an issue and attach operator logs.") return nil + case err != nil: + return fmt.Errorf("failed to retrieve dnsrecords ConfigMap: %w", err) } - if err != nil { - return fmt.Errorf("error retrieving dnsrecords ConfigMap: %w", err) - } + dnsRecords := operatorutils.Records{Version: operatorutils.Alpha1Version, IP4: map[string][]string{}} if cm.Data != nil && cm.Data[operatorutils.DNSRecordsCMKey] != "" { - if err := json.Unmarshal([]byte(cm.Data[operatorutils.DNSRecordsCMKey]), &dnsRecords); err != nil { + if err = json.Unmarshal([]byte(cm.Data[operatorutils.DNSRecordsCMKey]), &dnsRecords); err != nil { return err } } + update(&dnsRecords) dnsRecordsBs, err := json.Marshal(dnsRecords) if err != nil { return fmt.Errorf("error marshalling DNS records: %w", err) } + mak.Set(&cm.Data, operatorutils.DNSRecordsCMKey, string(dnsRecordsBs)) - return dnsRR.Update(ctx, cm) + return dnsRR.Update(ctx, &cm) } // isSvcForFQDNEgressProxy returns true if the Service is a headless Service @@ -366,3 +355,153 @@ func (dnsRR *dnsRecordsReconciler) isSvcForFQDNEgressProxy(ctx context.Context, annots := parentSvc.Annotations return annots != nil && annots[AnnotationTailnetTargetFQDN] != "", nil } + +// isProxyGroupEgressService reports whether the Service is a ClusterIP Service +// created for ProxyGroup egress. For ProxyGroup egress, there are no headless +// services. Instead, the DNS reconciler processes the ClusterIP Service +// directly, which has portmapping and should use its own IP for DNS records. +func (dnsRR *dnsRecordsReconciler) isProxyGroupEgressService(svc *corev1.Service) bool { + return svc.GetLabels()[labelProxyGroup] != "" && + svc.GetLabels()[labelSvcType] == typeEgress && + svc.Spec.Type == corev1.ServiceTypeClusterIP && + isManagedByType(svc, serviceTypeSvc) +} + +// isInterestingService reports whether the Service is one that we should create +// DNS records for. +func (dnsRR *dnsRecordsReconciler) isInterestingService(ctx context.Context, svc *corev1.Service) bool { + if isManagedByType(svc, serviceTypeIngress) { + return true + } + + isEgressFQDNSvc, err := dnsRR.isSvcForFQDNEgressProxy(ctx, svc) + if err != nil { + return false + } + if isEgressFQDNSvc { + return true + } + + if dnsRR.isProxyGroupEgressService(svc) { + return dnsRR.parentSvcTargetsFQDN(ctx, svc) + } + + return false +} + +// parentSvcTargetsFQDN reports whether the parent Service of a ProxyGroup +// egress Service has an FQDN target (not an IP target). +func (dnsRR *dnsRecordsReconciler) parentSvcTargetsFQDN(ctx context.Context, svc *corev1.Service) bool { + + parentName := parentFromObjectLabels(svc) + parentSvc := new(corev1.Service) + if err := dnsRR.Get(ctx, parentName, parentSvc); err != nil { + return false + } + + return parentSvc.Annotations[AnnotationTailnetTargetFQDN] != "" +} + +// getTargetIPs returns the IPv4 and IPv6 addresses that should be used for DNS records +// for the given proxy Service. +func (dnsRR *dnsRecordsReconciler) getTargetIPs(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { + if dnsRR.isProxyGroupEgressService(proxySvc) { + return dnsRR.getClusterIPServiceIPs(proxySvc, logger) + } + return dnsRR.getPodIPs(ctx, proxySvc, logger) +} + +// getClusterIPServiceIPs returns the ClusterIPs of a ProxyGroup egress Service. +// It separates IPv4 and IPv6 addresses for dual-stack services. +func (dnsRR *dnsRecordsReconciler) getClusterIPServiceIPs(proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { + // Handle services with no ClusterIP + if proxySvc.Spec.ClusterIP == "" || proxySvc.Spec.ClusterIP == "None" { + logger.Debugf("ProxyGroup egress ClusterIP Service does not have a ClusterIP yet.") + return nil, nil, nil + } + + var ip4s, ip6s []string + + // Check all ClusterIPs for dual-stack support + clusterIPs := proxySvc.Spec.ClusterIPs + if len(clusterIPs) == 0 && proxySvc.Spec.ClusterIP != "" { + // Fallback to single ClusterIP for backward compatibility + clusterIPs = []string{proxySvc.Spec.ClusterIP} + } + + for _, ip := range clusterIPs { + if net.IsIPv4String(ip) { + ip4s = append(ip4s, ip) + logger.Debugf("Using IPv4 ClusterIP %s for ProxyGroup egress DNS record", ip) + } else if net.IsIPv6String(ip) { + ip6s = append(ip6s, ip) + logger.Debugf("Using IPv6 ClusterIP %s for ProxyGroup egress DNS record", ip) + } else { + logger.Debugf("ClusterIP %s is not a valid IP address", ip) + } + } + + if len(ip4s) == 0 && len(ip6s) == 0 { + return nil, nil, fmt.Errorf("no valid ClusterIPs found") + } + + return ip4s, ip6s, nil +} + +// getPodIPs returns Pod IPv4 and IPv6 addresses from EndpointSlices for non-ProxyGroup Services. +func (dnsRR *dnsRecordsReconciler) getPodIPs(ctx context.Context, proxySvc *corev1.Service, logger *zap.SugaredLogger) ([]string, []string, error) { + // Get the Pod IP addresses for the proxy from the EndpointSlices for + // the headless Service. The Service can have multiple EndpointSlices + // associated with it, for example in dual-stack clusters. + labels := map[string]string{discoveryv1.LabelServiceName: proxySvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership + var eps = new(discoveryv1.EndpointSliceList) + if err := dnsRR.List(ctx, eps, client.InNamespace(dnsRR.tsNamespace), client.MatchingLabels(labels)); err != nil { + return nil, nil, fmt.Errorf("error listing EndpointSlices for the proxy's Service: %w", err) + } + if len(eps.Items) == 0 { + logger.Debugf("proxy's Service EndpointSlice does not yet exist.") + return nil, nil, nil + } + // Each EndpointSlice for a Service can have a list of endpoints that each + // can have multiple addresses - these are the IP addresses of any Pods + // selected by that Service. Separate IPv4 and IPv6 addresses. + // It is also possible that multiple EndpointSlices have overlapping addresses. + // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#duplicate-endpoints + ip4s := make(set.Set[string], 0) + ip6s := make(set.Set[string], 0) + for _, slice := range eps.Items { + for _, ep := range slice.Endpoints { + if !epIsReady(&ep) { + logger.Debugf("Endpoint with addresses %v appears not ready to receive traffic %v", ep.Addresses, ep.Conditions.String()) + continue + } + for _, ip := range ep.Addresses { + switch slice.AddressType { + case discoveryv1.AddressTypeIPv4: + if net.IsIPv4String(ip) { + ip4s.Add(ip) + } else { + logger.Debugf("EndpointSlice with AddressType IPv4 contains non-IPv4 address %q, ignoring", ip) + } + case discoveryv1.AddressTypeIPv6: + if net.IsIPv6String(ip) { + // Strip zone ID if present (e.g., fe80::1%eth0 -> fe80::1) + if idx := strings.IndexByte(ip, '%'); idx != -1 { + ip = ip[:idx] + } + ip6s.Add(ip) + } else { + logger.Debugf("EndpointSlice with AddressType IPv6 contains non-IPv6 address %q, ignoring", ip) + } + default: + logger.Debugf("EndpointSlice is for unsupported AddressType %s, skipping", slice.AddressType) + } + } + } + } + if ip4s.Len() == 0 && ip6s.Len() == 0 { + logger.Debugf("EndpointSlice for the Service contains no IP addresses.") + return nil, nil, nil + } + return ip4s.Slice(), ip6s.Slice(), nil +} diff --git a/cmd/k8s-operator/dnsrecords_test.go b/cmd/k8s-operator/dnsrecords_test.go index 4e73e6c9e33ba..c6c5ee0296ca3 100644 --- a/cmd/k8s-operator/dnsrecords_test.go +++ b/cmd/k8s-operator/dnsrecords_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -18,13 +18,13 @@ import ( networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" - "tailscale.com/types/ptr" ) func TestDNSRecordsReconciler(t *testing.T) { @@ -43,7 +43,7 @@ func TestDNSRecordsReconciler(t *testing.T) { Namespace: "test", }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), }, Status: networkingv1.IngressStatus{ LoadBalancer: networkingv1.IngressLoadBalancerStatus{ @@ -66,7 +66,7 @@ func TestDNSRecordsReconciler(t *testing.T) { } cl := tstest.NewClock(tstest.ClockOpts{}) // Set the ready condition of the DNSConfig - mustUpdateStatus[tsapi.DNSConfig](t, fc, "", "test", func(c *tsapi.DNSConfig) { + mustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) { operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated, 0, cl, zl.Sugar()) }) dnsRR := &dnsRecordsReconciler{ @@ -98,8 +98,9 @@ func TestDNSRecordsReconciler(t *testing.T) { mustCreate(t, fc, epv6) expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service // ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7 - wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} // IPv6 endpoint is currently ignored - expectHostsRecords(t, fc, wantHosts) + wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}} + wantHostsIPv6 := map[string][]string{"foo.bar.ts.net": {"2600:1900:4011:161:0:d:0:d"}} + expectHostsRecordsWithIPv6(t, fc, wantHosts, wantHostsIPv6) // 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's // value changes @@ -148,7 +149,7 @@ func TestDNSRecordsReconciler(t *testing.T) { // 7. A not-ready Endpoint is removed from DNS config. mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) { - ep.Endpoints[0].Conditions.Ready = ptr.To(false) + ep.Endpoints[0].Conditions.Ready = new(false) ep.Endpoints = append(ep.Endpoints, discoveryv1.Endpoint{ Addresses: []string{"1.2.3.4"}, }) @@ -156,6 +157,262 @@ func TestDNSRecordsReconciler(t *testing.T) { expectReconciled(t, dnsRR, "tailscale", "ts-ingress") wantHosts["another.ingress.ts.net"] = []string{"1.2.3.4"} expectHostsRecords(t, fc, wantHosts) + + // 8. DNS record is created for ProxyGroup egress using ClusterIP Service IP instead of Pod IPs + t.Log("test case 8: ProxyGroup egress") + + // Create the parent ExternalName service with tailnet-fqdn annotation + parentEgressSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external-service", + Namespace: "default", + Annotations: map[string]string{ + AnnotationTailnetTargetFQDN: "external-service.example.ts.net", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "unused", + }, + } + mustCreate(t, fc, parentEgressSvc) + + proxyGroupEgressSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ts-proxygroup-egress-abcd1", + Namespace: "tailscale", + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentName: "external-service", + LabelParentNamespace: "default", + LabelParentType: "svc", + labelProxyGroup: "test-proxy-group", + labelSvcType: typeEgress, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.0.100.50", // This IP should be used in DNS, not Pod IPs + Ports: []corev1.ServicePort{{ + Port: 443, + TargetPort: intstr.FromInt(10443), // Port mapping + }}, + }, + } + + // Create EndpointSlice with Pod IPs (these should NOT be used in DNS records) + proxyGroupEps := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ts-proxygroup-egress-abcd1-ipv4", + Namespace: "tailscale", + Labels: map[string]string{ + discoveryv1.LabelServiceName: "ts-proxygroup-egress-abcd1", + kubetypes.LabelManaged: "true", + LabelParentName: "external-service", + LabelParentNamespace: "default", + LabelParentType: "svc", + labelProxyGroup: "test-proxy-group", + labelSvcType: typeEgress, + }, + }, + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: []discoveryv1.Endpoint{{ + Addresses: []string{"10.1.0.100", "10.1.0.101", "10.1.0.102"}, // Pod IPs that should NOT be used + Conditions: discoveryv1.EndpointConditions{ + Ready: new(true), + Serving: new(true), + Terminating: new(false), + }, + }}, + Ports: []discoveryv1.EndpointPort{{ + Port: new(int32(10443)), + }}, + } + + mustCreate(t, fc, proxyGroupEgressSvc) + mustCreate(t, fc, proxyGroupEps) + expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + + // Verify DNS record uses ClusterIP Service IP, not Pod IPs + wantHosts["external-service.example.ts.net"] = []string{"10.0.100.50"} + expectHostsRecords(t, fc, wantHosts) + + // 9. ProxyGroup egress DNS record updates when ClusterIP changes + t.Log("test case 9: ProxyGroup egress ClusterIP change") + mustUpdate(t, fc, "tailscale", "ts-proxygroup-egress-abcd1", func(svc *corev1.Service) { + svc.Spec.ClusterIP = "10.0.100.51" + }) + expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + wantHosts["external-service.example.ts.net"] = []string{"10.0.100.51"} + expectHostsRecords(t, fc, wantHosts) + + // 10. Test ProxyGroup service deletion and DNS cleanup + t.Log("test case 10: ProxyGroup egress service deletion") + mustDeleteAll(t, fc, proxyGroupEgressSvc) + expectReconciled(t, dnsRR, "tailscale", "ts-proxygroup-egress-abcd1") + delete(wantHosts, "external-service.example.ts.net") + expectHostsRecords(t, fc, wantHosts) +} + +func TestDNSRecordsReconcilerErrorCases(t *testing.T) { + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + dnsRR := &dnsRecordsReconciler{ + logger: zl.Sugar(), + } + + testSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP}, + } + + // Test invalid IP format + testSvc.Spec.ClusterIP = "invalid-ip" + _, _, err = dnsRR.getClusterIPServiceIPs(testSvc, zl.Sugar()) + if err == nil { + t.Error("expected error for invalid IP format") + } + + // Test valid IP + testSvc.Spec.ClusterIP = "10.0.100.50" + ip4s, ip6s, err := dnsRR.getClusterIPServiceIPs(testSvc, zl.Sugar()) + if err != nil { + t.Errorf("unexpected error for valid IP: %v", err) + } + if len(ip4s) != 1 || ip4s[0] != "10.0.100.50" { + t.Errorf("expected IPv4 address 10.0.100.50, got %v", ip4s) + } + if len(ip6s) != 0 { + t.Errorf("expected no IPv6 addresses, got %v", ip6s) + } +} + +func TestDNSRecordsReconcilerDualStack(t *testing.T) { + // Test dual-stack (IPv4 and IPv6) scenarios + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + // Preconfigure cluster with DNSConfig + dnsCfg := &tsapi.DNSConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + TypeMeta: metav1.TypeMeta{Kind: "DNSConfig"}, + Spec: tsapi.DNSConfigSpec{Nameserver: &tsapi.Nameserver{}}, + } + dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{ + Type: string(tsapi.NameserverReady), + Status: metav1.ConditionTrue, + }) + + // Create dual-stack ingress + ing := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dual-stack-ingress", + Namespace: "test", + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + }, + Status: networkingv1.IngressStatus{ + LoadBalancer: networkingv1.IngressLoadBalancerStatus{ + Ingress: []networkingv1.IngressLoadBalancerIngress{ + {Hostname: "dual-stack.example.ts.net"}, + }, + }, + }, + } + + headlessSvc := headlessSvcForParent(ing, "ingress") + headlessSvc.Name = "ts-dual-stack-ingress" + headlessSvc.SetLabels(map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentName: "dual-stack-ingress", + LabelParentNamespace: "test", + LabelParentType: "ingress", + }) + + // Create both IPv4 and IPv6 endpoints + epv4 := endpointSliceForService(headlessSvc, "10.1.2.3", discoveryv1.AddressTypeIPv4) + epv6 := endpointSliceForService(headlessSvc, "2001:db8::1", discoveryv1.AddressTypeIPv6) + + dnsRRDualStack := &dnsRecordsReconciler{ + tsNamespace: "tailscale", + logger: zl.Sugar(), + } + + // Create the dnsrecords ConfigMap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorutils.DNSRecordsCMName, + Namespace: "tailscale", + }, + } + + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(dnsCfg, ing, headlessSvc, epv4, epv6, cm). + WithStatusSubresource(dnsCfg). + Build() + + dnsRRDualStack.Client = fc + + // Test dual-stack service records + expectReconciled(t, dnsRRDualStack, "tailscale", "ts-dual-stack-ingress") + + wantIPv4 := map[string][]string{"dual-stack.example.ts.net": {"10.1.2.3"}} + wantIPv6 := map[string][]string{"dual-stack.example.ts.net": {"2001:db8::1"}} + expectHostsRecordsWithIPv6(t, fc, wantIPv4, wantIPv6) + + // Test ProxyGroup with dual-stack ClusterIPs + // First create parent service + parentEgressSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg-service", + Namespace: "tailscale", + Annotations: map[string]string{ + AnnotationTailnetTargetFQDN: "pg-service.example.ts.net", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "unused", + }, + } + + proxyGroupSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ts-proxygroup-dualstack", + Namespace: "tailscale", + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + labelProxyGroup: "test-pg", + labelSvcType: typeEgress, + LabelParentName: "pg-service", + LabelParentNamespace: "tailscale", + LabelParentType: "svc", + }, + Annotations: map[string]string{ + annotationTSMagicDNSName: "pg-service.example.ts.net", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.96.0.100", + ClusterIPs: []string{"10.96.0.100", "2001:db8::100"}, + }, + } + + mustCreate(t, fc, parentEgressSvc) + mustCreate(t, fc, proxyGroupSvc) + expectReconciled(t, dnsRRDualStack, "tailscale", "ts-proxygroup-dualstack") + + wantIPv4["pg-service.example.ts.net"] = []string{"10.96.0.100"} + wantIPv6["pg-service.example.ts.net"] = []string{"2001:db8::100"} + expectHostsRecordsWithIPv6(t, fc, wantIPv4, wantIPv6) } func headlessSvcForParent(o client.Object, typ string) *corev1.Service { @@ -189,9 +446,9 @@ func endpointSliceForService(svc *corev1.Service, ip string, fam discoveryv1.Add Endpoints: []discoveryv1.Endpoint{{ Addresses: []string{ip}, Conditions: discoveryv1.EndpointConditions{ - Ready: ptr.To(true), - Serving: ptr.To(true), - Terminating: ptr.To(false), + Ready: new(true), + Serving: new(true), + Terminating: new(false), }, }}, } @@ -218,3 +475,28 @@ func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][] t.Fatalf("unexpected dns config (-got +want):\n%s", diff) } } + +func expectHostsRecordsWithIPv6(t *testing.T, cl client.Client, wantsHostsIPv4, wantsHostsIPv6 map[string][]string) { + t.Helper() + cm := new(corev1.ConfigMap) + if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil { + t.Fatalf("getting dnsconfig ConfigMap: %v", err) + } + if cm.Data == nil { + t.Fatal("dnsconfig ConfigMap has no data") + } + dnsConfigString, ok := cm.Data[operatorutils.DNSRecordsCMKey] + if !ok { + t.Fatal("dnsconfig ConfigMap does not contain dnsconfig") + } + dnsConfig := &operatorutils.Records{} + if err := json.Unmarshal([]byte(dnsConfigString), dnsConfig); err != nil { + t.Fatalf("unmarshaling dnsconfig: %v", err) + } + if diff := cmp.Diff(dnsConfig.IP4, wantsHostsIPv4); diff != "" { + t.Fatalf("unexpected IPv4 dns config (-got +want):\n%s", diff) + } + if diff := cmp.Diff(dnsConfig.IP6, wantsHostsIPv6); diff != "" { + t.Fatalf("unexpected IPv6 dns config (-got +want):\n%s", diff) + } +} diff --git a/cmd/k8s-operator/e2e/acl.hujson b/cmd/k8s-operator/e2e/acl.hujson new file mode 100644 index 0000000000000..1a7b61767c92b --- /dev/null +++ b/cmd/k8s-operator/e2e/acl.hujson @@ -0,0 +1,33 @@ +// To run the e2e tests against a tailnet, ensure its access controls are a +// superset of the following: +{ + "tagOwners": { + "tag:k8s-operator": [], + "tag:k8s": ["tag:k8s-operator"], + "tag:k8s-recorder": ["tag:k8s-operator"], + }, + "autoApprovers": { + // Could be relaxed if we coordinated with the cluster config, but this + // wide subnet maximises compatibility for most clusters. + "routes": { + "10.0.0.0/8": ["tag:k8s"], + }, + "services": { + "tag:k8s": ["tag:k8s"], + }, + }, + "grants": [ + { + "src": ["tag:k8s"], + "dst": ["tag:k8s", "tag:k8s-operator"], + "ip": ["tcp:80", "tcp:443"], + "app": { + "tailscale.com/cap/kubernetes": [{ + "impersonate": { + "groups": ["ts:e2e-test-proxy"], + }, + }], + }, + }, + ], +} \ No newline at end of file diff --git a/cmd/k8s-operator/e2e/certs/pebble.minica.crt b/cmd/k8s-operator/e2e/certs/pebble.minica.crt new file mode 100644 index 0000000000000..35388ee56db91 --- /dev/null +++ b/cmd/k8s-operator/e2e/certs/pebble.minica.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCTCCAfGgAwIBAgIIJOLbes8sTr4wDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE +AxMVbWluaWNhIHJvb3QgY2EgMjRlMmRiMCAXDTE3MTIwNjE5NDIxMFoYDzIxMTcx +MjA2MTk0MjEwWjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSAyNGUyZGIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5WgZNoVJandj43kkLyU50vzCZ +alozvdRo3OFiKoDtmqKPNWRNO2hC9AUNxTDJco51Yc42u/WV3fPbbhSznTiOOVtn +Ajm6iq4I5nZYltGGZetGDOQWr78y2gWY+SG078MuOO2hyDIiKtVc3xiXYA+8Hluu +9F8KbqSS1h55yxZ9b87eKR+B0zu2ahzBCIHKmKWgc6N13l7aDxxY3D6uq8gtJRU0 +toumyLbdzGcupVvjbjDP11nl07RESDWBLG1/g3ktJvqIa4BWgU2HMh4rND6y8OD3 +Hy3H8MY6CElL+MOCbFJjWqhtOxeFyZZV9q3kYnk9CAuQJKMEGuN4GU6tzhW1AgMB +AAGjRTBDMA4GA1UdDwEB/wQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwEgYDVR0TAQH/BAgwBgEB/wIBADANBgkqhkiG9w0BAQsFAAOCAQEAF85v +d40HK1ouDAtWeO1PbnWfGEmC5Xa478s9ddOd9Clvp2McYzNlAFfM7kdcj6xeiNhF +WPIfaGAi/QdURSL/6C1KsVDqlFBlTs9zYfh2g0UXGvJtj1maeih7zxFLvet+fqll +xseM4P9EVJaQxwuK/F78YBt0tCNfivC6JNZMgxKF59h0FBpH70ytUSHXdz7FKwix +Mfn3qEb9BXSk0Q3prNV5sOV3vgjEtB4THfDxSz9z3+DepVnW3vbbqwEbkXdk3j82 +2muVldgOUgTwK8eT+XdofVdntzU/kzygSAtAQwLJfn51fS1GvEcYGBc1bDryIqmF +p9BI7gVKtWSZYegicA== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/cmd/k8s-operator/e2e/doc.go b/cmd/k8s-operator/e2e/doc.go new file mode 100644 index 0000000000000..27d10e637c8c2 --- /dev/null +++ b/cmd/k8s-operator/e2e/doc.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package e2e runs end-to-end tests for the Tailscale Kubernetes operator. +// +// To run without arguments, it requires: +// +// * Kubernetes cluster with local kubeconfig for it (direct connection, no API server proxy) +// * Tailscale operator installed with --set apiServerProxyConfig.mode="true" +// * ACLs from acl.hujson +// * OAuth client secret in TS_API_CLIENT_SECRET env, with at least auth_keys write scope and tag:k8s tag +// * Default ProxyClass and operator env vars as appropriate to set the desired default proxy images. +// +// It also supports running against devcontrol, using the --devcontrol flag, +// which it expects to reach at http://localhost:31544. Use --cluster to create +// a dedicated kind cluster for the tests, and --build to build and test the +// operator and proxy images for the current checkout. +// +// To run with minimal dependencies, use: +// +// go test -count=1 -v ./cmd/k8s-operator/e2e/ --build --cluster --devcontrol --skip-cleanup +// +// Running like this, it requires: +// +// * go +// * container runtime with the docker daemon API available +// * devcontrol: ./tool/go run --tags=tailscale_saas ./cmd/devcontrol --generate-test-devices=k8s-operator-e2e --scenario-output-dir=/tmp/k8s-operator-e2e --test-dns=http://localhost:8055 +package e2e diff --git a/cmd/k8s-operator/e2e/helpers.go b/cmd/k8s-operator/e2e/helpers.go new file mode 100644 index 0000000000000..e01821c2367e3 --- /dev/null +++ b/cmd/k8s-operator/e2e/helpers.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "crypto/rand" + "crypto/tls" + "fmt" + "net/http" + "strings" + "time" + + "tailscale.com/tsnet" +) + +func generateName(prefix string) string { + return fmt.Sprintf("%s-%s", prefix, strings.ToLower(rand.Text())) +} + +// newHTTPClient returns a HTTP client for the given tailnet client. +// When running against devcontrol, trusts Pebble testCAs. +func newHTTPClient(cl *tsnet.Server) *http.Client { + return &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: testCAs}, + DialContext: cl.Dial, + }, + } +} diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go index 373dd2c7dc88f..bef24ca5a0a3b 100644 --- a/cmd/k8s-operator/e2e/ingress_test.go +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package e2e @@ -7,60 +7,138 @@ import ( "context" "fmt" "net/http" + "strings" "testing" "time" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" + + "tailscale.com/client/tailscale/v2" kube "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" + "tailscale.com/tsnet" "tailscale.com/tstest" + "tailscale.com/util/httpm" ) // See [TestMain] for test requirements. -func TestIngress(t *testing.T) { - if tsClient == nil { - t.Skip("TestIngress requires credentials for a tailscale client") +func TestL3Ingress(t *testing.T) { + if tnClient == nil { + t.Skip("TestL3Ingress requires a working tailnet client") } - ctx := context.Background() - cfg := config.GetConfigOrDie() - cl, err := client.New(cfg, client.Options{}) - if err != nil { - t.Fatal(err) - } // Apply nginx - createAndCleanup(t, ctx, cl, &corev1.Pod{ + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) + // Apply service to expose it as ingress + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "nginx", - Namespace: "default", - Labels: map[string]string{ - "app.kubernetes.io/name": "nginx", + Name: generateName("test-ingress"), + Namespace: ns, + Annotations: map[string]string{ + "tailscale.com/expose": "true", }, }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": nginx.Name, + }, + Ports: []corev1.ServicePort{ { - Name: "nginx", - Image: "nginx", + Name: "http", + Protocol: "TCP", + Port: 80, }, }, }, - }) - // Apply service to expose it as ingress + } + createAndCleanup(t, kubeClient, svc) + + if err := tstest.WaitFor(time.Minute, func() error { + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, svc.Name)} + if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { + return err + } + isReady := kube.SvcIsReady(maybeReadySvc) + if isReady { + t.Log("Service is ready") + return nil + } + return fmt.Errorf("Service is not ready yet") + }); err != nil { + t.Fatalf("error waiting for the Service to become Ready: %v", err) + } + + // Get the DNS name for the Service from the associated Secret. + var fqdn string + if err := tstest.WaitFor(time.Minute, func() error { + var secrets corev1.SecretList + if err := kubeClient.List(t.Context(), &secrets, + client.InNamespace("tailscale"), + client.MatchingLabels{ + "tailscale.com/parent-resource": svc.Name, + "tailscale.com/parent-resource-ns": ns, + }, + ); err != nil { + return err + } + if len(secrets.Items) == 0 { + return fmt.Errorf("Service not ready yet") + } + fqdn = strings.TrimSuffix(string(secrets.Items[0].Data[kubetypes.KeyDeviceFQDN]), ".") + if fqdn != "" { + t.Log("Got DNS name for Service") + return nil + } + return fmt.Errorf("device FQDN not set yet") + }); err != nil { + t.Fatalf("error waiting for DNS Name for Service: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("http://%s:80", fqdn)); err != nil { + t.Fatal(err) + } +} + +func TestL3HAIngress(t *testing.T) { + if tnClient == nil { + t.Skip("TestL3HAIngress requires a working tailnet client") + } + + // Apply nginx. + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) + + // Create an ingress ProxyGroup. + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName("ingress"), + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + } + createAndCleanup(t, kubeClient, pg) + + // Apply a Service to expose nginx via the ProxyGroup. svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "default", + Name: generateName("test-ingress"), + Namespace: ns, Annotations: map[string]string{ - "tailscale.com/expose": "true", + "tailscale.com/proxy-group": pg.Name, }, }, Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), Selector: map[string]string{ - "app.kubernetes.io/name": "nginx", + "app.kubernetes.io/name": nginx.Name, }, Ports: []corev1.ServicePort{ { @@ -71,38 +149,423 @@ func TestIngress(t *testing.T) { }, }, } - createAndCleanup(t, ctx, cl, svc) + createAndCleanup(t, kubeClient, svc) + + var svcIPv4 string + forceReconcile := triggerReconcile(t, + client.ObjectKey{Namespace: ns, Name: svc.Name}, + &corev1.Service{}, 30*time.Second) - // TODO: instead of timing out only when test times out, cancel context after 60s or so. - if err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) { - maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")} - if err := get(ctx, cl, maybeReadySvc); err != nil { - return false, err + // Wait for Service to be ready + if err := tstest.WaitFor(5*time.Minute, func() error { + maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta(ns, svc.Name)} + forceReconcile() + if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { + return err } - isReady := kube.SvcIsReady(maybeReadySvc) - if isReady { - t.Log("Service is ready") + for _, cond := range maybeReadySvc.Status.Conditions { + if cond.Type == string(tsapi.IngressSvcConfigured) && cond.Status == metav1.ConditionTrue { + if len(maybeReadySvc.Status.LoadBalancer.Ingress) == 0 { + return fmt.Errorf("Service does not have an IP assigned yet") + } + svcIPv4 = maybeReadySvc.Status.LoadBalancer.Ingress[0].IP + t.Log("Service is ready") + return nil + } } - return isReady, nil + return fmt.Errorf("Service is not ready yet") }); err != nil { - t.Fatalf("error waiting for the Service to become Ready: %v", err) + t.Fatalf("error waiting for the Service to become ready: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("http://%s:80", svcIPv4)); err != nil { + t.Fatal(err) + } +} + +func TestL7Ingress(t *testing.T) { + if tnClient == nil { + t.Skip("TestL7Ingress requires a working tailnet client") + } + + // Apply nginx Deployment and Service. + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: nginx.Name, + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": nginx.Name, + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Apply Ingress to expose nginx. + ingress := l7Ingress(ns, nginx.Name, map[string]string{}) + createAndCleanup(t, kubeClient, ingress) + + t.Log("Waiting for the Ingress to be ready...") + + hostname, err := waitForIngressHostname(t, ns, ingress.Name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { + t.Fatal(err) + } +} + +func TestL7HAIngress(t *testing.T) { + if tnClient == nil { + t.Skip("TestL7HAIngress requires a working tailnet client") + } + + // Apply nginx Deployment and Service. + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: nginx.Name, + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": nginx.Name, + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Create ProxyGroup that the Ingress will reference. + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName("ingress"), + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, } + createAndCleanup(t, kubeClient, pg) + + // Apply Ingress to expose nginx. + ingress := l7Ingress(ns, nginx.Name, map[string]string{"tailscale.com/proxy-group": pg.Name}) + createAndCleanup(t, kubeClient, ingress) + + t.Log("Waiting for the Ingress to be ready...") + hostname, err := waitForIngressHostname(t, ns, ingress.Name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + + if err := testIngressIsReachable(t, newHTTPClient(tnClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { + t.Fatal(err) + } +} + +func TestL7HAIngressMultiTailnet(t *testing.T) { + if tnClient == nil || secondTNClient == nil { + t.Skip("TestL7HAIngressMultiTailnet requires a working tailnet client for a first and second tailnet") + } + + // Apply nginx Deployment and Service. + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: nginx.Name, + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": nginx.Name, + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Create Ingress ProxyGroup for each Tailnet. + firstTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName("first-tailnet"), + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + } + createAndCleanup(t, kubeClient, firstTailnetPG) + secondTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName("second-tailnet"), + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + Tailnet: "second-tailnet", + }, + } + createAndCleanup(t, kubeClient, secondTailnetPG) + + if err := verifyProxyGroupTailnet(t, firstTailnetPG, tnClient); err != nil { + t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", firstTailnetPG.Name, err) + } + if err := verifyProxyGroupTailnet(t, secondTailnetPG, secondTNClient); err != nil { + t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", secondTailnetPG.Name, err) + } + + // Apply Ingress to expose nginx. + ingress := l7Ingress(ns, nginx.Name, map[string]string{ + "tailscale.com/proxy-group": secondTailnetPG.Name, + }) + createAndCleanup(t, kubeClient, ingress) + + // Check that the tailscale (VIP) Service has been created in the expected Tailnet. + svcName := "svc:" + ingress.Name + if err := tstest.WaitFor(3*time.Minute, func() error { + _, err := secondTSClient.VIPServices().Get(t.Context(), svcName) + if tailscale.IsNotFound(err) { + return fmt.Errorf("Tailscale service %q not yet in expected tailnet", svcName) + } + return err + }); err != nil { + t.Fatalf("Tailscale service %q never appeared in expected tailnet: %v", svcName, err) + } + hostname, err := waitForIngressHostname(t, ns, ingress.Name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + if err := testIngressIsReachable(t, newHTTPClient(secondTNClient), fmt.Sprintf("https://%s:443", hostname)); err != nil { + t.Fatal(err) + } +} + +func l7Ingress(namespace, svc string, annotations map[string]string) *networkingv1.Ingress { + name := generateName("test-ingress") + ingress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: annotations, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{name}}, + }, + Rules: []networkingv1.IngressRule{ + { + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: new(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: svc, + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + return ingress +} + +func nginxDeployment(namespace string) *appsv1.Deployment { + name := generateName("nginx") + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": name, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: new(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/name": name, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } +} + +// triggerReconcile triggers an expected reconcile for the given object if +// none occurs. This is needed when running some tests against devcontrol, +// where the final change that should trigger a reconcile does not always do so. +// This has not been reproducible in a real tailnet environment, so a +// workaround that runs only when using devcontrol is acceptable. +func triggerReconcile(t testing.TB, key client.ObjectKey, obj client.Object, after time.Duration) func() { + if !*fDevcontrol { + return func() {} + } + triggerAt := time.Now().Add(after) + var triggered bool + return func() { + if triggered || !time.Now().After(triggerAt) { + return + } + if err := kubeClient.Get(t.Context(), key, obj); err != nil { + t.Logf("failed to get %s: %v", key, err) + return + } + ann := obj.GetAnnotations() + if ann == nil { + ann = map[string]string{} + } + ann["tailscale.com/trigger-reconcile"] = "true" + obj.SetAnnotations(ann) + if err := kubeClient.Update(t.Context(), obj); err != nil { + t.Logf("failed to update %s: %v", key, err) + return + } + triggered = true + } +} + +func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) error { + t.Helper() var resp *http.Response - if err := tstest.WaitFor(time.Second*60, func() error { - // TODO(tomhjp): Get the tailnet DNS name from the associated secret instead. - // If we are not the first tailnet node with the requested name, we'll get - // a -N suffix. - resp, err = tsClient.HTTPClient.Get(fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name)) + if err := tstest.WaitFor(time.Minute, func() error { + req, err := http.NewRequest(httpm.GET, url, nil) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + resp, err = httpClient.Do(req.WithContext(ctx)) if err != nil { return err } + resp.Body.Close() return nil }); err != nil { - t.Fatalf("error trying to reach service: %v", err) + return fmt.Errorf("error trying to reach %s: %w", url, err) } - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %v; response body s", resp.StatusCode) + return fmt.Errorf("unexpected status from %s: %d", url, resp.StatusCode) + } + return nil +} + +// verifyProxyGroupTailnet verifies that a ProxyGroup is registered to the correct tailnet. +// This is done by getting the expected tailnet domain for the tailnet client, +// and comparing this with the actual device fqdn in the ProxyGroup state secret. +func verifyProxyGroupTailnet(t *testing.T, pg *tsapi.ProxyGroup, cl *tsnet.Server) error { + t.Helper() + // Determine the expected tailnet Magic DNS Name. + lc, err := cl.LocalClient() + if err != nil { + return err + } + status, err := lc.Status(t.Context()) + if err != nil { + return err + } + _, expectedTailnet, ok := strings.Cut(strings.TrimSuffix(status.Self.DNSName, "."), ".") + if !ok { + return fmt.Errorf("unexpected DNSName format %q", status.Self.DNSName) + } + // Read the device FQDN from the first state secret for the ProxyGroup, + // and verify that this matches the expected tailnet. + if err := tstest.WaitFor(3*time.Minute, func() error { + var secrets corev1.SecretList + if err := kubeClient.List(t.Context(), &secrets, + client.InNamespace("tailscale"), + client.MatchingLabels{ + kubetypes.LabelSecretType: kubetypes.LabelSecretTypeState, + "tailscale.com/parent-resource-type": "proxygroup", + "tailscale.com/parent-resource": pg.Name, + }, + ); err != nil { + return err + } + if len(secrets.Items) == 0 { + return fmt.Errorf("no state secrets found for ProxyGroup %q yet", pg.Name) + } + fqdn := strings.TrimSuffix(string(secrets.Items[0].Data[kubetypes.KeyDeviceFQDN]), ".") + _, tailnet, ok := strings.Cut(fqdn, ".") + if !ok { + return fmt.Errorf("ProxyGroup %q: device FQDN %q has no domain yet", pg.Name, fqdn) + } + if tailnet != expectedTailnet { + return fmt.Errorf("ProxyGroup %q on wrong tailnet: got domain %q, want %q", pg.Name, tailnet, expectedTailnet) + } + return nil + }); err != nil { + return fmt.Errorf("ProxyGroup %q not on expected tailnet: %v", pg.Name, err) + } + return nil +} + +func waitForIngressHostname(t *testing.T, namespace, name string) (string, error) { + t.Helper() + var hostname string + forceReconcile := triggerReconcile(t, + client.ObjectKey{Namespace: namespace, Name: name}, + &networkingv1.Ingress{}, 30*time.Second) + + if err := tstest.WaitFor(5*time.Minute, func() error { + forceReconcile() + ing := &networkingv1.Ingress{} + if err := kubeClient.Get(t.Context(), client.ObjectKey{ + Namespace: namespace, Name: name, + }, ing); err != nil { + return err + } + if len(ing.Status.LoadBalancer.Ingress) == 0 || + ing.Status.LoadBalancer.Ingress[0].Hostname == "" { + return fmt.Errorf("Ingress not ready yet") + } + hostname = ing.Status.LoadBalancer.Ingress[0].Hostname + t.Log("Ingress is ready") + return nil + }); err != nil { + return "", fmt.Errorf("Ingress %s/%s never got a hostname: %w", namespace, name, err) } + return hostname, nil } diff --git a/cmd/k8s-operator/e2e/main_test.go b/cmd/k8s-operator/e2e/main_test.go index 5a1364e09d0d7..9eab9e30157aa 100644 --- a/cmd/k8s-operator/e2e/main_test.go +++ b/cmd/k8s-operator/e2e/main_test.go @@ -1,191 +1,80 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package e2e import ( "context" - "errors" - "fmt" + "flag" "log" "os" - "slices" - "strings" "testing" - "github.com/go-logr/zapr" - "github.com/tailscale/hujson" - "go.uber.org/zap/zapcore" - "golang.org/x/oauth2/clientcredentials" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" - "tailscale.com/internal/client/tailscale" ) -const ( - e2eManagedComment = "// This is managed by the k8s-operator e2e tests" -) - -var ( - tsClient *tailscale.Client - testGrants = map[string]string{ - "test-proxy": `{ - "src": ["tag:e2e-test-proxy"], - "dst": ["tag:k8s-operator"], - "app": { - "tailscale.com/cap/kubernetes": [{ - "impersonate": { - "groups": ["ts:e2e-test-proxy"], - }, - }], - }, - }`, - } -) - -// This test suite is currently not run in CI. -// It requires some setup not handled by this code: -// - Kubernetes cluster with tailscale operator installed -// - Current kubeconfig context set to connect to that cluster (directly, no operator proxy) -// - Operator installed with --set apiServerProxyConfig.mode="true" -// - ACLs that define tag:e2e-test-proxy tag. TODO(tomhjp): Can maybe replace this prereq onwards with an API key -// - OAuth client ID and secret in TS_API_CLIENT_ID and TS_API_CLIENT_SECRET env -// - OAuth client must have auth_keys and policy_file write for tag:e2e-test-proxy tag func TestMain(m *testing.M) { + flag.Parse() + if !*fDevcontrol && os.Getenv("TS_API_CLIENT_SECRET") == "" { + log.Printf("Skipping setup: devcontrol is false and TS_API_CLIENT_SECRET is not set") + os.Exit(m.Run()) + } code, err := runTests(m) if err != nil { - log.Fatal(err) + log.Printf("Error: %v", err) + os.Exit(1) } os.Exit(code) } -func runTests(m *testing.M) (int, error) { - zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar() - logf.SetLogger(zapr.NewLogger(zlog.Desugar())) - - if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" { - cleanup, err := setupClientAndACLs() - if err != nil { - return 0, err - } - defer func() { - err = errors.Join(err, cleanup()) - }() +func objectMeta(namespace, name string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Namespace: namespace, + Name: name, } - - return m.Run(), nil } -func setupClientAndACLs() (cleanup func() error, _ error) { - ctx := context.Background() - credentials := clientcredentials.Config{ - ClientID: os.Getenv("TS_API_CLIENT_ID"), - ClientSecret: os.Getenv("TS_API_CLIENT_SECRET"), - TokenURL: "https://login.tailscale.com/api/v2/oauth/token", - Scopes: []string{"auth_keys", "policy_file"}, - } - tsClient = tailscale.NewClient("-", nil) - tsClient.HTTPClient = credentials.Client(ctx) +func createAndCleanup(t *testing.T, cl client.Client, obj client.Object) { + t.Helper() - if err := patchACLs(ctx, tsClient, func(acls *hujson.Value) { - for test, grant := range testGrants { - deleteTestGrants(test, acls) - addTestGrant(test, grant, acls) + // Try to create the object first + err := cl.Create(t.Context(), obj) + if err != nil { + if apierrors.IsAlreadyExists(err) { + if updateErr := cl.Update(t.Context(), obj); updateErr != nil { + t.Fatal(updateErr) + } + } else { + t.Fatal(err) } - }); err != nil { - return nil, err } - return func() error { - return patchACLs(ctx, tsClient, func(acls *hujson.Value) { - for test := range testGrants { - deleteTestGrants(test, acls) - } - }) - }, nil + t.Cleanup(func() { + // Use context.Background() for cleanup, as t.Context() is cancelled + // just before cleanup functions are called. + if err := cl.Delete(context.Background(), obj); err != nil { + t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err) + } + }) } -func patchACLs(ctx context.Context, tsClient *tailscale.Client, patchFn func(*hujson.Value)) error { - acls, err := tsClient.ACLHuJSON(ctx) - if err != nil { - return err - } - hj, err := hujson.Parse([]byte(acls.ACL)) - if err != nil { - return err - } - - patchFn(&hj) - - hj.Format() - acls.ACL = hj.String() - if _, err := tsClient.SetACLHuJSON(ctx, *acls, true); err != nil { - return err - } - - return nil -} +func createAndCleanupErr(t *testing.T, cl client.Client, obj client.Object) error { + t.Helper() -func addTestGrant(test, grant string, acls *hujson.Value) error { - v, err := hujson.Parse([]byte(grant)) + err := cl.Create(t.Context(), obj) if err != nil { return err } - // Add the managed comment to the first line of the grant object contents. - v.Value.(*hujson.Object).Members[0].Name.BeforeExtra = hujson.Extra(fmt.Sprintf("%s: %s\n", e2eManagedComment, test)) - - if err := acls.Patch([]byte(fmt.Sprintf(`[{"op": "add", "path": "/grants/-", "value": %s}]`, v.String()))); err != nil { - return err - } - - return nil -} - -func deleteTestGrants(test string, acls *hujson.Value) error { - grants := acls.Find("/grants") - - var patches []string - for i, g := range grants.Value.(*hujson.Array).Elements { - members := g.Value.(*hujson.Object).Members - if len(members) == 0 { - continue - } - comment := strings.TrimSpace(string(members[0].Name.BeforeExtra)) - if name, found := strings.CutPrefix(comment, e2eManagedComment+": "); found && name == test { - patches = append(patches, fmt.Sprintf(`{"op": "remove", "path": "/grants/%d"}`, i)) - } - } - - // Remove in reverse order so we don't affect the found indices as we mutate. - slices.Reverse(patches) - - if err := acls.Patch([]byte(fmt.Sprintf("[%s]", strings.Join(patches, ",")))); err != nil { - return err - } - - return nil -} - -func objectMeta(namespace, name string) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - } -} - -func createAndCleanup(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) { - t.Helper() - if err := cl.Create(ctx, obj); err != nil { - t.Fatal(err) - } t.Cleanup(func() { - if err := cl.Delete(ctx, obj); err != nil { + if err := cl.Delete(context.Background(), obj); err != nil { t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err) } }) + + return nil } func get(ctx context.Context, cl client.Client, obj client.Object) error { diff --git a/cmd/k8s-operator/e2e/pebble.go b/cmd/k8s-operator/e2e/pebble.go new file mode 100644 index 0000000000000..7abe3416ef7dc --- /dev/null +++ b/cmd/k8s-operator/e2e/pebble.go @@ -0,0 +1,173 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func applyPebbleResources(ctx context.Context, cl client.Client) error { + owner := client.FieldOwner("k8s-test") + + if err := cl.Patch(ctx, pebbleDeployment(pebbleTag), client.Apply, owner); err != nil { + return fmt.Errorf("failed to apply pebble Deployment: %w", err) + } + if err := cl.Patch(ctx, pebbleService(), client.Apply, owner); err != nil { + return fmt.Errorf("failed to apply pebble Service: %w", err) + } + if err := cl.Patch(ctx, tailscaleNamespace(), client.Apply, owner); err != nil { + return fmt.Errorf("failed to apply tailscale Namespace: %w", err) + } + if err := cl.Patch(ctx, pebbleExternalNameService(), client.Apply, owner); err != nil { + return fmt.Errorf("failed to apply pebble ExternalName Service: %w", err) + } + + return nil +} + +func pebbleDeployment(tag string) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pebble", + Namespace: ns, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: new(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "pebble", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "pebble", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "pebble", + Image: fmt.Sprintf("ghcr.io/letsencrypt/pebble:%s", tag), + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "-dnsserver=localhost:8053", + "-strict", + }, + Ports: []corev1.ContainerPort{ + { + Name: "acme", + ContainerPort: 14000, + }, + { + Name: "pebble-api", + ContainerPort: 15000, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "PEBBLE_VA_NOSLEEP", + Value: "1", + }, + }, + }, + { + Name: "challtestsrv", + Image: fmt.Sprintf("ghcr.io/letsencrypt/pebble-challtestsrv:%s", tag), + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"-defaultIPv6="}, + Ports: []corev1.ContainerPort{ + { + Name: "mgmt-api", + ContainerPort: 8055, + }, + }, + }, + }, + }, + }, + }, + } +} + +func pebbleService() *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pebble", + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app": "pebble", + }, + Ports: []corev1.ServicePort{ + { + Name: "acme", + Port: 14000, + TargetPort: intstr.FromInt(14000), + }, + { + Name: "pebble-api", + Port: 15000, + TargetPort: intstr.FromInt(15000), + }, + { + Name: "mgmt-api", + Port: 8055, + TargetPort: intstr.FromInt(8055), + }, + }, + }, + } +} + +func tailscaleNamespace() *corev1.Namespace { + return &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tailscale", + }, + } +} + +// pebbleExternalNameService ensures the operator in the tailscale namespace +// can reach pebble on a DNS name (pebble) that matches its TLS cert. +func pebbleExternalNameService() *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pebble", + Namespace: "tailscale", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + Selector: map[string]string{ + "app": "pebble", + }, + ExternalName: "pebble.default.svc.cluster.local", + }, + } +} diff --git a/cmd/k8s-operator/e2e/proxy_test.go b/cmd/k8s-operator/e2e/proxy_test.go index eac983e88d613..3caf1c91d8bc9 100644 --- a/cmd/k8s-operator/e2e/proxy_test.go +++ b/cmd/k8s-operator/e2e/proxy_test.go @@ -1,13 +1,11 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package e2e import ( - "context" "encoding/json" "fmt" - "strings" "testing" "time" @@ -16,27 +14,19 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" - "tailscale.com/client/tailscale" - "tailscale.com/tsnet" + + "tailscale.com/ipn" "tailscale.com/tstest" ) // See [TestMain] for test requirements. func TestProxy(t *testing.T) { - if tsClient == nil { - t.Skip("TestProxy requires credentials for a tailscale client") - } - - ctx := context.Background() - cfg := config.GetConfigOrDie() - cl, err := client.New(cfg, client.Options{}) - if err != nil { - t.Fatal(err) + if tnClient == nil { + t.Skip("TestProxy requires a working tailnet client") } // Create role and role binding to allow a group we'll impersonate to do stuff. - createAndCleanup(t, ctx, cl, &rbacv1.Role{ + createAndCleanup(t, kubeClient, &rbacv1.Role{ ObjectMeta: objectMeta("tailscale", "read-secrets"), Rules: []rbacv1.PolicyRule{{ APIGroups: []string{""}, @@ -44,7 +34,7 @@ func TestProxy(t *testing.T) { Resources: []string{"secrets"}, }}, }) - createAndCleanup(t, ctx, cl, &rbacv1.RoleBinding{ + createAndCleanup(t, kubeClient, &rbacv1.RoleBinding{ ObjectMeta: objectMeta("tailscale", "read-secrets"), Subjects: []rbacv1.Subject{{ Kind: "Group", @@ -60,18 +50,17 @@ func TestProxy(t *testing.T) { operatorSecret := corev1.Secret{ ObjectMeta: objectMeta("tailscale", "operator"), } - if err := get(ctx, cl, &operatorSecret); err != nil { + if err := get(t.Context(), kubeClient, &operatorSecret); err != nil { t.Fatal(err) } - // Connect to tailnet with test-specific tag so we can use the - // [testGrants] ACLs when connecting to the API server proxy - ts := tsnetServerWithTag(t, ctx, "tag:e2e-test-proxy") + // Join tailnet as a client of the API server proxy. proxyCfg := &rest.Config{ Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)), - Dial: ts.Dial, } - proxyCl, err := client.New(proxyCfg, client.Options{}) + proxyCl, err := client.New(proxyCfg, client.Options{ + HTTPClient: newHTTPClient(tnClient), + }) if err != nil { t.Fatal(err) } @@ -82,8 +71,10 @@ func TestProxy(t *testing.T) { } // Wait for up to a minute the first time we use the proxy, to give it time // to provision the TLS certs. - if err := tstest.WaitFor(time.Second*60, func() error { - return get(ctx, proxyCl, &allowedSecret) + if err := tstest.WaitFor(time.Minute, func() error { + err := get(t.Context(), proxyCl, &allowedSecret) + t.Logf("get Secret via proxy: %v", err) + return err }); err != nil { t.Fatal(err) } @@ -92,65 +83,25 @@ func TestProxy(t *testing.T) { forbiddenSecret := corev1.Secret{ ObjectMeta: objectMeta("default", "operator"), } - if err := get(ctx, proxyCl, &forbiddenSecret); err == nil || !apierrors.IsForbidden(err) { + if err := get(t.Context(), proxyCl, &forbiddenSecret); err == nil || !apierrors.IsForbidden(err) { t.Fatalf("expected forbidden error fetching secret from default namespace: %s", err) } } -func tsnetServerWithTag(t *testing.T, ctx context.Context, tag string) *tsnet.Server { - caps := tailscale.KeyCapabilities{ - Devices: tailscale.KeyDeviceCapabilities{ - Create: tailscale.KeyDeviceCreateCapabilities{ - Reusable: false, - Preauthorized: true, - Ephemeral: true, - Tags: []string{tag}, - }, - }, - } - - authKey, authKeyMeta, err := tsClient.CreateKey(ctx, caps) - if err != nil { - t.Fatal(err) +func hostNameFromOperatorSecret(t *testing.T, s corev1.Secret) string { + t.Helper() + prefsBytes, ok := s.Data[string(s.Data["_current-profile"])] + if !ok { + t.Fatalf("no state in operator Secret data: %#v", s.Data) } - t.Cleanup(func() { - if err := tsClient.DeleteKey(ctx, authKeyMeta.ID); err != nil { - t.Errorf("error deleting auth key: %s", err) - } - }) - ts := &tsnet.Server{ - Hostname: "test-proxy", - Ephemeral: true, - Dir: t.TempDir(), - AuthKey: authKey, - } - _, err = ts.Up(ctx) - if err != nil { + prefs := ipn.Prefs{} + if err := json.Unmarshal(prefsBytes, &prefs); err != nil { t.Fatal(err) } - t.Cleanup(func() { - if err := ts.Close(); err != nil { - t.Errorf("error shutting down tsnet.Server: %s", err) - } - }) - return ts -} - -func hostNameFromOperatorSecret(t *testing.T, s corev1.Secret) string { - profiles := map[string]any{} - if err := json.Unmarshal(s.Data["_profiles"], &profiles); err != nil { - t.Fatal(err) - } - key, ok := strings.CutPrefix(string(s.Data["_current-profile"]), "profile-") - if !ok { - t.Fatal(string(s.Data["_current-profile"])) + if prefs.Persist == nil { + t.Fatalf("no hostname in operator Secret data: %#v", s.Data) } - profile, ok := profiles[key] - if !ok { - t.Fatal(profiles) - } - - return ((profile.(map[string]any))["Name"]).(string) + return prefs.Persist.UserProfile.LoginName } diff --git a/cmd/k8s-operator/e2e/proxygrouppolicy_test.go b/cmd/k8s-operator/e2e/proxygrouppolicy_test.go new file mode 100644 index 0000000000000..0e73394d539da --- /dev/null +++ b/cmd/k8s-operator/e2e/proxygrouppolicy_test.go @@ -0,0 +1,160 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" +) + +// See [TestMain] for test requirements. +func TestProxyGroupPolicy(t *testing.T) { + if tnClient == nil { + t.Skip("TestProxyGroupPolicy requires a working tailnet client") + } + + // Apply deny-all policy + denyAllPolicy := &tsapi.ProxyGroupPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deny-all", + Namespace: metav1.NamespaceDefault, + }, + Spec: tsapi.ProxyGroupPolicySpec{ + Ingress: []string{}, + Egress: []string{}, + }, + } + + createAndCleanup(t, kubeClient, denyAllPolicy) + <-time.After(time.Second * 2) + + // Attempt to create an egress Service within the default namespace, the above policy should + // reject it. + egressService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "egress-to-proxy-group", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + "tailscale.com/tailnet-fqdn": "test.something.ts.net", + "tailscale.com/proxy-group": "test", + }, + }, + Spec: corev1.ServiceSpec{ + ExternalName: "placeholder", + Type: corev1.ServiceTypeExternalName, + Ports: []corev1.ServicePort{ + { + Port: 8080, + Protocol: corev1.ProtocolTCP, + Name: "http", + }, + }, + }, + } + + err := createAndCleanupErr(t, kubeClient, egressService) + switch { + case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"): + case err != nil: + t.Fatalf("expected forbidden error, got: %v", err) + default: + t.Fatal("expected error when creating egress service") + } + + // Attempt to create an ingress Service within the default namespace, the above policy should + // reject it. + ingressService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-to-proxy-group", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + Ports: []corev1.ServicePort{ + { + Port: 8080, + Protocol: corev1.ProtocolTCP, + Name: "http", + }, + }, + }, + } + + err = createAndCleanupErr(t, kubeClient, ingressService) + switch { + case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"): + case err != nil: + t.Fatalf("expected forbidden error, got: %v", err) + default: + t.Fatal("expected error when creating ingress service") + } + + // Attempt to create an Ingress within the default namespace, the above policy should reject it + ingress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-to-proxy-group", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "nginx", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + { + Hosts: []string{"nginx"}, + }, + }, + }, + } + + err = createAndCleanupErr(t, kubeClient, ingress) + switch { + case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"): + case err != nil: + t.Fatalf("expected forbidden error, got: %v", err) + default: + t.Fatal("expected error when creating ingress") + } + + // Add policy to allow ingress/egress using the "test" proxy-group. This should be merged with the deny-all + // policy so they do not conflict. + allowTestPolicy := &tsapi.ProxyGroupPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "allow-test", + Namespace: metav1.NamespaceDefault, + }, + Spec: tsapi.ProxyGroupPolicySpec{ + Ingress: []string{"test"}, + Egress: []string{"test"}, + }, + } + + createAndCleanup(t, kubeClient, allowTestPolicy) + <-time.After(time.Second * 2) + + // With this policy in place, the above ingress/egress resources should be allowed to be created. + createAndCleanup(t, kubeClient, egressService) + createAndCleanup(t, kubeClient, ingressService) + createAndCleanup(t, kubeClient, ingress) +} diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go new file mode 100644 index 0000000000000..0d4ca80ad68f9 --- /dev/null +++ b/cmd/k8s-operator/e2e/setup.go @@ -0,0 +1,895 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/tls" + "crypto/x509" + _ "embed" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "os/signal" + "path/filepath" + "slices" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/go-logr/zapr" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/daemon" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "go.uber.org/zap" + "golang.org/x/oauth2/clientcredentials" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/release" + "helm.sh/helm/v3/pkg/storage/driver" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" + "sigs.k8s.io/controller-runtime/pkg/client" + klog "sigs.k8s.io/controller-runtime/pkg/log" + kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/kind/pkg/cluster" + "sigs.k8s.io/kind/pkg/cluster/nodeutils" + "sigs.k8s.io/kind/pkg/cmd" + + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" + "tailscale.com/ipn/store/mem" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tsnet" + "tailscale.com/util/must" +) + +const ( + pebbleTag = "2.8.0" + ns = "default" + tmp = "/tmp/k8s-operator-e2e" + kindClusterName = "k8s-operator-e2e" +) + +var ( + tsClient *tailscale.Client // For API calls to control. + tnClient *tsnet.Server // For testing real tailnet traffic on first tailnet. + secondTSClient *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet). + secondTNClient *tsnet.Server // For testing real tailnet traffic on second tailnet. + restCfg *rest.Config // For constructing a client-go client if necessary. + kubeClient client.WithWatch // For k8s API calls. + clusterLoginServer string + + //go:embed certs/pebble.minica.crt + pebbleMiniCACert []byte + + // Either nil (system) or pebble CAs if pebble is deployed for devcontrol. + // pebble has a static "mini" CA that its ACME directory URL serves a cert + // from, and also dynamically generates a different CA for issuing certs. + testCAs *x509.CertPool + + //go:embed acl.hujson + requiredACLs []byte + + fDevcontrol = flag.Bool("devcontrol", false, "if true, connect to devcontrol at http://localhost:31544. Run devcontrol with "+` + ./tool/go run ./cmd/devcontrol \ + --generate-test-devices=k8s-operator-e2e \ + --dir=/tmp/devcontrol \ + --scenario-output-dir=/tmp/k8s-operator-e2e \ + --test-dns=http://localhost:8055`) + fSkipCleanup = flag.Bool("skip-cleanup", false, "if true, do not delete the kind cluster (if created) or tmp dir on exit") + fCluster = flag.Bool("cluster", false, "if true, create or use a pre-existing kind cluster named k8s-operator-e2e; otherwise assume a usable cluster already exists in kubeconfig") + fBuild = flag.Bool("build", false, "if true, build and deploy the operator and container images from the current checkout; otherwise assume the operator is already set up") +) + +func runTests(m *testing.M) (int, error) { + logger := kzap.NewRaw().Sugar() + klog.SetLogger(zapr.NewLogger(logger.Desugar())) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer cancel() + + ossDir, err := gitRootDir() + if err != nil { + return 0, err + } + + if err = os.MkdirAll(tmp, 0755); err != nil { + return 0, fmt.Errorf("failed to create temp dir: %w", err) + } + + logger.Infof("temp dir: %q", tmp) + logger.Infof("oss dir: %q", ossDir) + + var ( + kubeconfig string + kindProvider *cluster.Provider + ) + if *fCluster { + kubeconfig = filepath.Join(tmp, "kubeconfig") + kindProvider = cluster.NewProvider( + cluster.ProviderWithLogger(cmd.NewLogger()), + ) + + clusters, err := kindProvider.List() + if err != nil { + return 0, fmt.Errorf("failed to list kind clusters: %w", err) + } + + if !slices.Contains(clusters, kindClusterName) { + if err := kindProvider.Create(kindClusterName, + cluster.CreateWithWaitForReady(5*time.Minute), + cluster.CreateWithKubeconfigPath(kubeconfig), + cluster.CreateWithNodeImage("kindest/node:v1.35.0"), + ); err != nil { + return 0, fmt.Errorf("failed to create kind cluster: %w", err) + } + } + + if !*fSkipCleanup { + defer kindProvider.Delete(kindClusterName, kubeconfig) + defer os.Remove(kubeconfig) + } + } + + // Cluster client setup. + restCfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return 0, fmt.Errorf("error loading kubeconfig: %w", err) + } + + kubeClient, err = client.NewWithWatch(restCfg, client.Options{Scheme: tsapi.GlobalScheme}) + if err != nil { + return 0, fmt.Errorf("error creating Kubernetes client: %w", err) + } + + var ( + clientID, clientSecret string // OAuth client for the first tailnet (for the operator to use). + caPaths []string // Extra CA cert file paths to add to images. + + certsDir = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images. + secondClientID, secondClientSecret string // OAuth client for the second tailnet (for the operator to use). + ) + if *fDevcontrol { + // Deploy pebble and get its certs. + if err = applyPebbleResources(ctx, kubeClient); err != nil { + return 0, fmt.Errorf("failed to apply pebble resources: %w", err) + } + + pebblePod, err := waitForPodReady(ctx, logger, kubeClient, ns, client.MatchingLabels{"app": "pebble"}) + if err != nil { + return 0, fmt.Errorf("pebble pod not ready: %w", err) + } + + if err = forwardLocalPortToPod(ctx, logger, restCfg, ns, pebblePod, 15000); err != nil { + return 0, fmt.Errorf("failed to set up port forwarding to pebble: %w", err) + } + + testCAs = x509.NewCertPool() + if ok := testCAs.AppendCertsFromPEM(pebbleMiniCACert); !ok { + return 0, fmt.Errorf("failed to parse pebble minica cert") + } + + var pebbleCAChain []byte + for _, path := range []string{"/intermediates/0", "/roots/0"} { + pem, err := pebbleGet(ctx, 15000, path) + if err != nil { + return 0, err + } + pebbleCAChain = append(pebbleCAChain, pem...) + } + + if ok := testCAs.AppendCertsFromPEM(pebbleCAChain); !ok { + return 0, fmt.Errorf("failed to parse pebble ca chain cert") + } + + if err = os.MkdirAll(certsDir, 0755); err != nil { + return 0, fmt.Errorf("failed to create certs dir: %w", err) + } + + pebbleCAChainPath := filepath.Join(certsDir, "pebble-ca-chain.crt") + if err = os.WriteFile(pebbleCAChainPath, pebbleCAChain, 0644); err != nil { + return 0, fmt.Errorf("failed to write pebble CA chain: %w", err) + } + + pebbleMiniCACertPath := filepath.Join(certsDir, "pebble.minica.crt") + if err = os.WriteFile(pebbleMiniCACertPath, pebbleMiniCACert, 0644); err != nil { + return 0, fmt.Errorf("failed to write pebble minica: %w", err) + } + + caPaths = []string{pebbleCAChainPath, pebbleMiniCACertPath} + if !*fSkipCleanup { + defer os.RemoveAll(certsDir) + } + + // Set up network connectivity between cluster and devcontrol. + // + // For devcontrol -> pebble (DNS mgmt for ACME challenges): + // * Port forward from localhost port 8055 to in-cluster pebble port 8055. + // + // For Pods -> devcontrol (tailscale clients joining the tailnet): + // * Create ssh-server Deployment in cluster. + // * Create reverse ssh tunnel that goes from ssh-server port 31544 to localhost:31544. + if err = forwardLocalPortToPod(ctx, logger, restCfg, ns, pebblePod, 8055); err != nil { + return 0, fmt.Errorf("failed to set up port forwarding to pebble: %w", err) + } + + privateKey, publicKey, err := readOrGenerateSSHKey(tmp) + if err != nil { + return 0, fmt.Errorf("failed to read or generate SSH key: %w", err) + } + + if !*fSkipCleanup { + defer os.Remove(privateKeyPath) + } + + sshServiceIP, err := connectClusterToDevcontrol(ctx, logger, kubeClient, restCfg, privateKey, publicKey) + if err != nil { + return 0, fmt.Errorf("failed to set up cluster->devcontrol connection: %w", err) + } + + if !*fSkipCleanup { + defer func() { + if err := cleanupSSHResources(context.Background(), kubeClient); err != nil { + logger.Infof("failed to clean up ssh-server resources: %v", err) + } + }() + } + + // Address cluster workloads can reach devcontrol at. Must be a private + // IP to make sure tailscale client code recognises it shouldn't try an + // https fallback. See [controlclient.NewNoiseClient] for details. + clusterLoginServer = fmt.Sprintf("http://%s:31544", sshServiceIP) + + b, err := os.ReadFile(filepath.Join(tmp, "api-key.json")) + if err != nil { + return 0, fmt.Errorf("failed to read api-key.json: %w", err) + } + var apiKeyData struct { + APIKey string `json:"apiKey"` + } + if err = json.Unmarshal(b, &apiKeyData); err != nil { + return 0, fmt.Errorf("failed to parse api-key.json: %w", err) + } + if apiKeyData.APIKey == "" { + return 0, fmt.Errorf("api-key.json did not contain an API key") + } + + // Finish setting up tsClient. + tsClient = &tailscale.Client{ + APIKey: apiKeyData.APIKey, + BaseURL: must.Get(url.Parse("http://localhost:31544")), + } + + // Set ACLs and create OAuth client. + if err = tsClient.PolicyFile().Set(ctx, string(requiredACLs), ""); err != nil { + return 0, fmt.Errorf("failed to set policy file: %w", err) + } + + logger.Info("ACLs configured for first tailnet") + + key, err := tsClient.Keys().CreateOAuthClient(ctx, tailscale.CreateOAuthClientRequest{ + Scopes: []string{"auth_keys", "devices:core", "services"}, + Tags: []string{"tag:k8s-operator"}, + Description: "k8s-operator client for e2e tests", + }) + if err != nil { + return 0, fmt.Errorf("failed to create OAuth client for first tailnet: %w", err) + } + clientID = key.ID + clientSecret = key.Key + + logger.Info("OAuth credentials set for first tailnet") + + // Create second tailnet. The bootstrap credentials returned have 'all' permissions- + // they are used for administrative actions and to create a separately scoped + // Oauth client for the k8s operator. + bootstrapClient, err := createTailnet(ctx, tsClient) + if err != nil { + return 0, fmt.Errorf("failed to create second tailnet: %w", err) + } + + // Set HTTPS on second tailnet. + err = bootstrapClient.TailnetSettings().Update(ctx, tailscale.UpdateTailnetSettingsRequest{HTTPSEnabled: new(true)}) + if err != nil { + return 0, fmt.Errorf("failed to configure https for second tailnet: %w", err) + } + logger.Info("HTTPS settings configured for second tailnet") + + // Set ACLs for second tailnet. + if err = bootstrapClient.PolicyFile().Set(ctx, string(requiredACLs), ""); err != nil { + return 0, fmt.Errorf("failed to set policy file: %w", err) + } + + logger.Info("ACLs configured for second tailnet") + + // Create an OAuth client for the second tailnet to be used + // by the k8s-operator. + secondKey, err := bootstrapClient.Keys().CreateOAuthClient(ctx, tailscale.CreateOAuthClientRequest{ + Scopes: []string{"auth_keys", "devices:core", "services"}, + Tags: []string{"tag:k8s-operator"}, + Description: "k8s-operator client for e2e tests", + }) + if err != nil { + return 0, fmt.Errorf("failed to create OAuth client for second tailnet: %w", err) + } + secondClientID = secondKey.ID + secondClientSecret = secondKey.Key + + secondTSClient, err = tailscaleClientFromSecret(ctx, "http://localhost:31544", secondClientID, secondClientSecret) + if err != nil { + return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) + } + + } else { + clientSecret = os.Getenv("TS_API_CLIENT_SECRET") + if clientSecret == "" { + return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") + } + clientID, err = clientIDFromSecret(clientSecret) + if err != nil { + return 0, fmt.Errorf("failed to get client id from secret: %w", err) + } + tsClient, err = tailscaleClientFromSecret(ctx, ipn.DefaultControlURL, clientID, clientSecret) + if err != nil { + return 0, fmt.Errorf("failed to set up first tailnet client: %w", err) + } + secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET") + if secondClientSecret == "" { + return 0, fmt.Errorf("must use --devcontrol or set SECOND_TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") + } + secondClientID, err = clientIDFromSecret(secondClientSecret) + if err != nil { + return 0, fmt.Errorf("failed to get client id from secret: %w", err) + } + secondTSClient, err = tailscaleClientFromSecret(ctx, ipn.DefaultControlURL, secondClientID, secondClientSecret) + if err != nil { + return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) + } + } + + var ossTag string + if *fBuild { + // TODO(tomhjp): proper support for --build=false and layering pebble certs on top of existing images. + // TODO(tomhjp): support non-local platform. + // TODO(tomhjp): build tsrecorder as well. + + // Build tailscale/k8s-operator, tailscale/tailscale, tailscale/k8s-proxy, with pebble CAs added. + ossTag, err = tagForRepo(ossDir) + if err != nil { + return 0, err + } + logger.Infof("using OSS image tag: %q", ossTag) + ossImageToTarget := map[string]string{ + "local/k8s-operator": "publishdevoperator", + "local/tailscale": "publishdevimage", + "local/k8s-proxy": "publishdevproxy", + } + for img, target := range ossImageToTarget { + if err := buildImage(ctx, ossDir, img, target, ossTag, caPaths); err != nil { + return 0, err + } + nodes, err := kindProvider.ListInternalNodes(kindClusterName) + if err != nil { + return 0, fmt.Errorf("failed to list kind nodes: %w", err) + } + // TODO(tomhjp): can be made more efficient and portable if we + // stream built image tarballs straight to the node rather than + // going via the daemon. + // TODO(tomhjp): support --build with non-kind clusters. + imgRef, err := name.ParseReference(fmt.Sprintf("%s:%s", img, ossTag)) + if err != nil { + return 0, fmt.Errorf("failed to parse image reference: %w", err) + } + img, err := daemon.Image(imgRef) + if err != nil { + return 0, fmt.Errorf("failed to get image from daemon: %w", err) + } + pr, pw := io.Pipe() + go func() { + defer pw.Close() + if err := tarball.Write(imgRef, img, pw); err != nil { + logger.Infof("failed to write image to pipe: %v", err) + } + }() + for _, n := range nodes { + if err := nodeutils.LoadImageArchive(n, pr); err != nil { + return 0, fmt.Errorf("failed to load image into node %q: %w", n.String(), err) + } + } + } + } + + // Generate CRDs for the helm chart. + cmd := exec.CommandContext(ctx, "go", "run", "tailscale.com/cmd/k8s-operator/generate", "helmcrd") + cmd.Dir = ossDir + out, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("failed to generate CRD: %v: %s", err, out) + } + + // Load and install helm chart. + chart, err := loader.Load(filepath.Join(ossDir, "cmd", "k8s-operator", "deploy", "chart")) + if err != nil { + return 0, fmt.Errorf("failed to load helm chart: %w", err) + } + extraEnv := []map[string]any{ + { + "name": "K8S_PROXY_IMAGE", + "value": "local/k8s-proxy:" + ossTag, + }, + } + if *fDevcontrol { + extraEnv = append(extraEnv, map[string]any{"name": "TS_DEBUG_ACME_DIRECTORY_URL", "value": "https://pebble:14000/dir"}) + } + values := map[string]any{ + "loginServer": clusterLoginServer, + "oauth": map[string]any{ + "clientId": clientID, + "clientSecret": clientSecret, + }, + "apiServerProxyConfig": map[string]any{ + "mode": "true", + }, + "operatorConfig": map[string]any{ + "logging": "debug", + "extraEnv": extraEnv, + "image": map[string]any{ + "repo": "local/k8s-operator", + "tag": ossTag, + "pullPolicy": "IfNotPresent", + }, + }, + "proxyConfig": map[string]any{ + "defaultProxyClass": "default", + "image": map[string]any{ + "repository": "local/tailscale", + "tag": ossTag, + }, + }, + } + + settings := cli.New() + settings.KubeConfig = kubeconfig + settings.SetNamespace("tailscale") + helmCfg := &action.Configuration{} + if err := helmCfg.Init(settings.RESTClientGetter(), "tailscale", "", logger.Infof); err != nil { + return 0, fmt.Errorf("failed to initialize helm action configuration: %w", err) + } + + const relName = "tailscale-operator" // TODO(tomhjp): maybe configurable if others use a different value. + f := upgraderOrInstaller(helmCfg, relName) + if _, err := f(ctx, relName, chart, values); err != nil { + return 0, fmt.Errorf("failed to install %q via helm: %w", relName, err) + } + + if err := applyDefaultProxyClass(ctx, logger, kubeClient); err != nil { + return 0, fmt.Errorf("failed to apply default ProxyClass: %w", err) + } + + caps := tailscale.KeyCapabilities{} + caps.Devices.Create.Preauthorized = true + caps.Devices.Create.Ephemeral = true + caps.Devices.Create.Tags = []string{"tag:k8s"} + + authKey, err := tsClient.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{Capabilities: caps}) + if err != nil { + return 0, fmt.Errorf("failed to create auth key for first tailnet: %w", err) + } + defer tsClient.Keys().Delete(context.Background(), authKey.ID) + + secondAuthKey, err := secondTSClient.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{Capabilities: caps}) + if err != nil { + return 0, fmt.Errorf("failed to create auth key for second tailnet: %w", err) + } + defer secondTSClient.Keys().Delete(context.Background(), secondAuthKey.ID) + + tnClient = &tsnet.Server{ + ControlURL: tsClient.BaseURL.String(), + Hostname: "test-proxy", + Ephemeral: true, + Store: &mem.Store{}, + AuthKey: authKey.Key, + } + _, err = tnClient.Up(ctx) + if err != nil { + return 0, err + } + defer tnClient.Close() + + secondTNClient = &tsnet.Server{ + ControlURL: secondTSClient.BaseURL.String(), + Hostname: "test-proxy", + Ephemeral: true, + Store: &mem.Store{}, + AuthKey: secondAuthKey.Key, + } + _, err = secondTNClient.Up(ctx) + if err != nil { + return 0, err + } + defer secondTNClient.Close() + + // Create the tailnet Secret in the tailscale namespace. + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet-credentials", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte(secondClientID), + "client_secret": []byte(secondClientSecret), + }, + } + if err := createOrUpdate(ctx, kubeClient, secret); err != nil { + return 0, fmt.Errorf("failed to create second-tailnet-credentials Secret: %w", err) + } + defer kubeClient.Delete(context.Background(), secret) + + // Create the Tailnet resource. + tn := &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet", + }, + Spec: tsapi.TailnetSpec{ + LoginURL: clusterLoginServer, + Credentials: tsapi.TailnetCredentials{ + SecretName: "second-tailnet-credentials", + }, + }, + } + if err := createOrUpdate(ctx, kubeClient, tn); err != nil { + return 0, fmt.Errorf("failed to create second-tailnet Tailnet: %w", err) + } + defer kubeClient.Delete(context.Background(), tn) + + return m.Run(), nil +} + +func clientIDFromSecret(clientSecret string) (string, error) { + // Format is "tskey-client--". + parts := strings.Split(clientSecret, "-") + if len(parts) != 4 { + return "", fmt.Errorf("secret is not valid") + } + return parts[2], nil +} + +func upgraderOrInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc { + hist := action.NewHistory(cfg) + hist.Max = 1 + helmVersions, err := hist.Run(releaseName) + if err == driver.ErrReleaseNotFound || (len(helmVersions) > 0 && helmVersions[0].Info.Status == release.StatusUninstalled) { + return helmInstaller(cfg, releaseName) + } else { + return helmUpgrader(cfg) + } +} + +func helmUpgrader(cfg *action.Configuration) helmInstallerFunc { + upgrade := action.NewUpgrade(cfg) + upgrade.Namespace = "tailscale" + upgrade.Install = true + upgrade.Wait = true + upgrade.Timeout = 5 * time.Minute + return upgrade.RunWithContext +} + +func helmInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc { + install := action.NewInstall(cfg) + install.Namespace = "tailscale" + install.CreateNamespace = true + install.ReleaseName = releaseName + install.Wait = true + install.Timeout = 5 * time.Minute + install.Replace = true + return func(ctx context.Context, _ string, chart *chart.Chart, values map[string]any) (*release.Release, error) { + return install.RunWithContext(ctx, chart, values) + } +} + +type helmInstallerFunc func(context.Context, string, *chart.Chart, map[string]any) (*release.Release, error) + +// gitRootDir returns the top-level directory of the current git repo. Expects +// to be run from inside a git repo. +func gitRootDir() (string, error) { + top, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + return "", fmt.Errorf("failed to find git top level (not in corp git?): %w", err) + } + return strings.TrimSpace(string(top)), nil +} + +func tagForRepo(dir string) (string, error) { + cmd := exec.Command("git", "rev-parse", "--short", "HEAD") + cmd.Dir = dir + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get latest git tag for repo %q: %w", dir, err) + } + tag := strings.TrimSpace(string(out)) + + // If dirty, append an extra random tag to ensure unique image tags. + cmd = exec.Command("git", "status", "--porcelain") + cmd.Dir = dir + out, err = cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to check git status for repo %q: %w", dir, err) + } + if strings.TrimSpace(string(out)) != "" { + tag += "-" + strings.ToLower(rand.Text()) + } + + return tag, nil +} + +func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl client.Client) error { + var env []tsapi.Env + if *fDevcontrol { + env = []tsapi.Env{ + { + Name: "TS_DEBUG_ACME_DIRECTORY_URL", + Value: "https://pebble:14000/dir", + }, + } + } + pc := &tsapi.ProxyClass{ + TypeMeta: metav1.TypeMeta{ + APIVersion: tsapi.SchemeGroupVersion.String(), + Kind: tsapi.ProxyClassKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleInitContainer: &tsapi.Container{ + ImagePullPolicy: "IfNotPresent", + }, + TailscaleContainer: &tsapi.Container{ + ImagePullPolicy: "IfNotPresent", + Env: env, + }, + }, + }, + }, + } + + owner := client.FieldOwner("k8s-test") + if err := cl.Patch(ctx, pc, client.Apply, owner); err != nil { + return fmt.Errorf("failed to apply default ProxyClass: %w", err) + } + + // Wait for the ProxyClass to be marked ready. + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + for { + if err := cl.Get(ctx, client.ObjectKeyFromObject(pc), pc); err != nil { + return fmt.Errorf("failed to get default ProxyClass: %w", err) + } + if tsoperator.ProxyClassIsReady(pc) { + break + } + logger.Info("waiting for default ProxyClass to be ready...") + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for default ProxyClass to be ready") + case <-time.After(time.Second): + } + } + + return nil +} + +// forwardLocalPortToPod sets up port forwarding to the specified Pod and remote port. +// It runs until the provided ctx is done. +func forwardLocalPortToPod(ctx context.Context, logger *zap.SugaredLogger, cfg *rest.Config, ns, podName string, port int) error { + transport, upgrader, err := spdy.RoundTripperFor(cfg) + if err != nil { + return fmt.Errorf("failed to create round tripper: %w", err) + } + + u, err := url.Parse(fmt.Sprintf("%s%s/api/v1/namespaces/%s/pods/%s/portforward", cfg.Host, cfg.APIPath, ns, podName)) + if err != nil { + return fmt.Errorf("failed to parse URL: %w", err) + } + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", u) + + stopChan := make(chan struct{}, 1) + readyChan := make(chan struct{}, 1) + + ports := []string{fmt.Sprintf("%d:%d", port, port)} + + // TODO(tomhjp): work out how zap logger can be used instead of stdout/err. + pf, err := portforward.New(dialer, ports, stopChan, readyChan, os.Stdout, os.Stderr) + if err != nil { + return fmt.Errorf("failed to create port forwarder: %w", err) + } + + go func() { + if err := pf.ForwardPorts(); err != nil { + logger.Infof("Port forwarding error: %v\n", err) + } + }() + + var once sync.Once + go func() { + <-ctx.Done() + once.Do(func() { close(stopChan) }) + }() + + // Wait for port forwarding to be ready + select { + case <-readyChan: + logger.Infof("Port forwarding to Pod %s/%s ready", ns, podName) + case <-time.After(10 * time.Second): + once.Do(func() { close(stopChan) }) + return fmt.Errorf("timeout waiting for port forward to be ready") + } + + return nil +} + +// waitForPodReady waits for at least 1 Pod matching the label selector to be +// in Ready state. It returns the name of the first ready Pod it finds. +func waitForPodReady(ctx context.Context, logger *zap.SugaredLogger, cl client.WithWatch, ns string, labelSelector client.MatchingLabels) (string, error) { + pods := &corev1.PodList{} + w, err := cl.Watch(ctx, pods, client.InNamespace(ns), client.MatchingLabels(labelSelector)) + if err != nil { + return "", fmt.Errorf("failed to create pod watcher: %v", err) + } + defer w.Stop() + + for { + select { + case event, ok := <-w.ResultChan(): + if !ok { + return "", fmt.Errorf("watcher channel closed") + } + + switch event.Type { + case watch.Added, watch.Modified: + if pod, ok := event.Object.(*corev1.Pod); ok { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + logger.Infof("pod %s is ready", pod.Name) + return pod.Name, nil + } + } + } + case watch.Error: + return "", fmt.Errorf("watch error: %v", event.Object) + } + case <-ctx.Done(): + return "", fmt.Errorf("timeout waiting for pod to be ready") + } + } +} + +func pebbleGet(ctx context.Context, port uint16, path string) ([]byte, error) { + pebbleClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: testCAs, + }, + }, + Timeout: 10 * time.Second, + } + req, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://localhost:%d%s", port, path), nil) + resp, err := pebbleClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch pebble root CA: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d when fetching pebble root CA", resp.StatusCode) + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read pebble root CA response: %w", err) + } + + return b, nil +} + +func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts []string) error { + var files []string + for _, f := range extraCACerts { + files = append(files, fmt.Sprintf("%s:/etc/ssl/certs/%s", f, filepath.Base(f))) + } + cmd := exec.CommandContext(ctx, "make", target, + "PLATFORM=local", + fmt.Sprintf("TAGS=%s", tag), + fmt.Sprintf("REPO=%s", repo), + fmt.Sprintf("FILES=%s", strings.Join(files, ",")), + ) + cmd.Dir = dir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to build image %q: %w", target, err) + } + + return nil +} + +func createOrUpdate(ctx context.Context, cl client.Client, obj client.Object) error { + if err := cl.Create(ctx, obj); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + return cl.Update(ctx, obj) + } + return nil +} + +// createTailnet creates a new tailnet and returns a tailscale.Client +// authenticated against it using the bootstrap credentials included in the +// creation response. +func createTailnet(ctx context.Context, tsClient *tailscale.Client) (*tailscale.Client, error) { + tailnetName := fmt.Sprintf("second-tailnet-%d", time.Now().Unix()) + body, err := json.Marshal(map[string]any{"displayName": tailnetName}) + if err != nil { + return nil, fmt.Errorf("failed to marshal tailnet creation request: %w", err) + } + // TODO(beckypauley): change to use a method on tailscale.Client once this is available. + req, _ := http.NewRequestWithContext(ctx, "POST", tsClient.BaseURL.String()+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body)) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tsClient.APIKey)) + resp, err := tsClient.HTTP.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to create tailnet: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b)) + } + var result struct { + OauthClient struct { + ID string `json:"id"` + Secret string `json:"secret"` + } `json:"oauthClient"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return tailscaleClientFromSecret(ctx, tsClient.BaseURL.String(), result.OauthClient.ID, result.OauthClient.Secret) +} + +// tailscaleClientFromSecret exchanges OAuth client credentials for an access token and +// returns a tailscale.Client configured to use it. The token is valid for +// one hour, which is sufficient for the tests to run. No need for refresh logic. +func tailscaleClientFromSecret(ctx context.Context, baseURL, clientID, clientSecret string) (*tailscale.Client, error) { + cfg := clientcredentials.Config{ + ClientID: clientID, + ClientSecret: clientSecret, + TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", baseURL), + } + tk, err := cfg.Token(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get OAuth token for client %q: %w", clientID, err) + } + return &tailscale.Client{ + APIKey: tk.AccessToken, + BaseURL: must.Get(url.Parse(baseURL)), + }, nil +} diff --git a/cmd/k8s-operator/e2e/ssh.go b/cmd/k8s-operator/e2e/ssh.go new file mode 100644 index 0000000000000..9adcce6e3eee0 --- /dev/null +++ b/cmd/k8s-operator/e2e/ssh.go @@ -0,0 +1,352 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "encoding/pem" + "fmt" + "io" + "net" + "os" + "path/filepath" + "time" + + "go.uber.org/zap" + "golang.org/x/crypto/ssh" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + tailscaleroot "tailscale.com" +) + +const ( + keysFilePath = "/root/.ssh/authorized_keys" + sshdConfig = ` +Port 8022 + +# Allow reverse tunnels +GatewayPorts yes +AllowTcpForwarding yes + +# Auth +PermitRootLogin yes +PasswordAuthentication no +PubkeyAuthentication yes +AuthorizedKeysFile ` + keysFilePath +) + +var privateKeyPath = filepath.Join(tmp, "id_ed25519") + +func connectClusterToDevcontrol(ctx context.Context, logger *zap.SugaredLogger, cl client.WithWatch, restConfig *rest.Config, privKey ed25519.PrivateKey, pubKey []byte) (clusterIP string, _ error) { + logger.Info("Setting up SSH reverse tunnel from cluster to devcontrol...") + var err error + if clusterIP, err = applySSHResources(ctx, cl, tailscaleroot.AlpineDockerTag, pubKey); err != nil { + return "", fmt.Errorf("failed to apply ssh-server resources: %w", err) + } + sshPodName, err := waitForPodReady(ctx, logger, cl, ns, client.MatchingLabels{"app": "ssh-server"}) + if err != nil { + return "", fmt.Errorf("ssh-server Pod not ready: %w", err) + } + if err := forwardLocalPortToPod(ctx, logger, restConfig, ns, sshPodName, 8022); err != nil { + return "", fmt.Errorf("failed to set up port forwarding to ssh-server: %w", err) + } + if err := reverseTunnel(ctx, logger, privKey, fmt.Sprintf("localhost:%d", 8022), 31544, "localhost:31544"); err != nil { + return "", fmt.Errorf("failed to set up reverse tunnel: %w", err) + } + + return clusterIP, nil +} + +func reverseTunnel(ctx context.Context, logger *zap.SugaredLogger, privateKey ed25519.PrivateKey, sshHost string, remotePort uint16, fwdTo string) error { + signer, err := ssh.NewSignerFromKey(privateKey) + if err != nil { + return fmt.Errorf("failed to create signer: %w", err) + } + config := &ssh.ClientConfig{ + User: "root", + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 30 * time.Second, + } + + conn, err := ssh.Dial("tcp", sshHost, config) + if err != nil { + return fmt.Errorf("failed to connect to SSH server: %w", err) + } + logger.Infof("Connected to SSH server at %s\n", sshHost) + + go func() { + defer conn.Close() + + // Start listening on remote port. + remoteAddr := fmt.Sprintf("localhost:%d", remotePort) + remoteLn, err := conn.Listen("tcp", remoteAddr) + if err != nil { + logger.Infof("Failed to listen on remote port %d: %v", remotePort, err) + return + } + defer remoteLn.Close() + logger.Infof("Reverse tunnel ready on remote addr %s -> local addr %s", remoteAddr, fwdTo) + + for { + remoteConn, err := remoteLn.Accept() + if err != nil { + logger.Infof("Failed to accept remote connection: %v", err) + return + } + + go handleConnection(ctx, logger, remoteConn, fwdTo) + } + }() + + return nil +} + +func handleConnection(ctx context.Context, logger *zap.SugaredLogger, remoteConn net.Conn, fwdTo string) { + go func() { + <-ctx.Done() + remoteConn.Close() + }() + + var d net.Dialer + localConn, err := d.DialContext(ctx, "tcp", fwdTo) + if err != nil { + logger.Infof("Failed to connect to local service %s: %v", fwdTo, err) + return + } + go func() { + <-ctx.Done() + localConn.Close() + }() + + go func() { + if _, err := io.Copy(localConn, remoteConn); err != nil { + logger.Infof("Error copying remote->local: %v", err) + } + }() + + go func() { + if _, err := io.Copy(remoteConn, localConn); err != nil { + logger.Infof("Error copying local->remote: %v", err) + } + }() +} + +func readOrGenerateSSHKey(tmp string) (ed25519.PrivateKey, []byte, error) { + var privateKey ed25519.PrivateKey + b, err := os.ReadFile(privateKeyPath) + switch { + case os.IsNotExist(err): + _, privateKey, err = ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate key: %w", err) + } + privKeyPEM, err := ssh.MarshalPrivateKey(privateKey, "") + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal SSH private key: %w", err) + } + f, err := os.OpenFile(privateKeyPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return nil, nil, fmt.Errorf("failed to open SSH private key file: %w", err) + } + defer f.Close() + if err := pem.Encode(f, privKeyPEM); err != nil { + return nil, nil, fmt.Errorf("failed to write SSH private key: %w", err) + } + case err != nil: + return nil, nil, fmt.Errorf("failed to read SSH private key: %w", err) + default: + pKey, err := ssh.ParseRawPrivateKey(b) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse SSH private key: %w", err) + } + pKeyPointer, ok := pKey.(*ed25519.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("SSH private key is not ed25519: %T", pKey) + } + privateKey = *pKeyPointer + } + + sshPublicKey, err := ssh.NewPublicKey(privateKey.Public()) + if err != nil { + return nil, nil, fmt.Errorf("failed to create SSH public key: %w", err) + } + + return privateKey, ssh.MarshalAuthorizedKey(sshPublicKey), nil +} + +func applySSHResources(ctx context.Context, cl client.Client, alpineTag string, pubKey []byte) (string, error) { + owner := client.FieldOwner("k8s-test") + + if err := cl.Patch(ctx, sshDeployment(alpineTag, pubKey), client.Apply, owner); err != nil { + return "", fmt.Errorf("failed to apply ssh-server Deployment: %w", err) + } + if err := cl.Patch(ctx, sshConfigMap(pubKey), client.Apply, owner); err != nil { + return "", fmt.Errorf("failed to apply ssh-server ConfigMap: %w", err) + } + svc := sshService() + if err := cl.Patch(ctx, svc, client.Apply, owner); err != nil { + return "", fmt.Errorf("failed to apply ssh-server Service: %w", err) + } + + return svc.Spec.ClusterIP, nil +} + +func cleanupSSHResources(ctx context.Context, cl client.Client) error { + noGrace := &client.DeleteOptions{ + GracePeriodSeconds: new(int64(0)), + } + if err := cl.Delete(ctx, sshDeployment("", nil), noGrace); err != nil { + return fmt.Errorf("failed to delete ssh-server Deployment: %w", err) + } + if err := cl.Delete(ctx, sshConfigMap(nil), noGrace); err != nil { + return fmt.Errorf("failed to delete ssh-server ConfigMap: %w", err) + } + if err := cl.Delete(ctx, sshService(), noGrace); err != nil { + return fmt.Errorf("failed to delete control Service: %w", err) + } + + return nil +} + +func sshDeployment(tag string, pubKey []byte) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ssh-server", + Namespace: ns, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: new(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "ssh-server", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "ssh-server", + }, + Annotations: map[string]string{ + "pubkey": hex.EncodeToString(pubKey), // Ensure new key triggers rollout. + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "ssh-server", + Image: fmt.Sprintf("alpine:%s", tag), + Command: []string{ + "sh", "-c", + "apk add openssh-server; ssh-keygen -A; /usr/sbin/sshd -D -e", + }, + Ports: []corev1.ContainerPort{ + { + Name: "ctrl-port-fwd", + ContainerPort: 31544, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "ssh", + ContainerPort: 8022, + Protocol: corev1.ProtocolTCP, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(8022), + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 1, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "sshd-config", + MountPath: "/etc/ssh/sshd_config.d/reverse-tunnel.conf", + SubPath: "reverse-tunnel.conf", + }, + { + Name: "sshd-config", + MountPath: keysFilePath, + SubPath: "authorized_keys", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sshd-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "ssh-server-config", + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func sshConfigMap(pubKey []byte) *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ssh-server-config", + Namespace: ns, + }, + Data: map[string]string{ + "reverse-tunnel.conf": sshdConfig, + "authorized_keys": string(pubKey), + }, + } +} + +func sshService() *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "control", + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app": "ssh-server", + }, + Ports: []corev1.ServicePort{ + { + Name: "tunnel", + Port: 31544, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} diff --git a/cmd/k8s-operator/egress-eps.go b/cmd/k8s-operator/egress-eps.go index 3441e12ba93ec..b47009e36fd83 100644 --- a/cmd/k8s-operator/egress-eps.go +++ b/cmd/k8s-operator/egress-eps.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -20,8 +20,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/kube/egressservices" - "tailscale.com/types/ptr" ) // egressEpsReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies. @@ -36,21 +36,21 @@ type egressEpsReconciler struct { // It compares tailnet service state stored in egress proxy state Secrets by containerboot with the desired // configuration stored in proxy-cfg ConfigMap to determine if the endpoint is ready. func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { - l := er.logger.With("Service", req.NamespacedName) - l.Debugf("starting reconcile") - defer l.Debugf("reconcile finished") + lg := er.logger.With("Service", req.NamespacedName) + lg.Debugf("starting reconcile") + defer lg.Debugf("reconcile finished") eps := new(discoveryv1.EndpointSlice) err = er.Get(ctx, req.NamespacedName, eps) if apierrors.IsNotFound(err) { - l.Debugf("EndpointSlice not found") + lg.Debugf("EndpointSlice not found") return reconcile.Result{}, nil } if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get EndpointSlice: %w", err) } if !eps.DeletionTimestamp.IsZero() { - l.Debugf("EnpointSlice is being deleted") + lg.Debugf("EnpointSlice is being deleted") return res, nil } @@ -64,7 +64,7 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ } err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc) if apierrors.IsNotFound(err) { - l.Infof("ExternalName Service %s/%s not found, perhaps it was deleted", svc.Namespace, svc.Name) + lg.Infof("ExternalName Service %s/%s not found, perhaps it was deleted", svc.Namespace, svc.Name) return res, nil } if err != nil { @@ -77,7 +77,7 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ oldEps := eps.DeepCopy() tailnetSvc := tailnetSvcName(svc) - l = l.With("tailnet-service-name", tailnetSvc) + lg = lg.With("tailnet-service-name", tailnetSvc) // Retrieve the desired tailnet service configuration from the ConfigMap. proxyGroupName := eps.Labels[labelProxyGroup] @@ -88,12 +88,13 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ if cfgs == nil { // TODO(irbekrm): this path would be hit if egress service was once exposed on a ProxyGroup that later // got deleted. Probably the EndpointSlices then need to be deleted too- need to rethink this flow. - l.Debugf("No egress config found, likely because ProxyGroup has not been created") + lg.Debugf("No egress config found, likely because ProxyGroup has not been created") return res, nil } - cfg, ok := (*cfgs)[tailnetSvc] + + cfg, ok := cfgs[tailnetSvc] if !ok { - l.Infof("[unexpected] configuration for tailnet service %s not found", tailnetSvc) + lg.Warnf("configuration for tailnet service %q not found", tailnetSvc) return res, nil } @@ -105,7 +106,7 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ } newEndpoints := make([]discoveryv1.Endpoint, 0) for _, pod := range podList.Items { - ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, l) + ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, lg) if err != nil { return res, fmt.Errorf("error verifying if Pod is ready to route traffic: %w", err) } @@ -120,9 +121,9 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ Hostname: (*string)(&pod.UID), Addresses: []string{podIP}, Conditions: discoveryv1.EndpointConditions{ - Ready: ptr.To(true), - Serving: ptr.To(true), - Terminating: ptr.To(false), + Ready: new(true), + Serving: new(true), + Terminating: new(false), }, }) } @@ -130,11 +131,12 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // run a cleanup for deleted Pods etc. eps.Endpoints = newEndpoints if !reflect.DeepEqual(eps, oldEps) { - l.Infof("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods") - if err := er.Update(ctx, eps); err != nil { + lg.Info("Updating EndpointSlice to ensure traffic is routed to ready proxy Pods") + if err = er.Update(ctx, eps); err != nil { return res, fmt.Errorf("error updating EndpointSlice: %w", err) } } + return res, nil } @@ -154,61 +156,71 @@ func podIPv4(pod *corev1.Pod) (string, error) { // podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to // route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service // status written there to the desired service configuration. -func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, l *zap.SugaredLogger) (bool, error) { - l = l.With("proxy_pod", pod.Name) - l.Debugf("checking whether proxy is ready to route to egress service") +func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, lg *zap.SugaredLogger) (bool, error) { + lg = lg.With("proxy_pod", pod.Name) + lg.Debug("checking whether proxy is ready to route to egress service") if !pod.DeletionTimestamp.IsZero() { - l.Debugf("proxy Pod is being deleted, ignore") + lg.Debug("proxy Pod is being deleted, ignore") return false, nil } + podIP, err := podIPv4(&pod) - if err != nil { + switch { + case err != nil: return false, fmt.Errorf("error determining Pod IP address: %v", err) - } - if podIP == "" { - l.Infof("[unexpected] Pod does not have an IPv4 address, and IPv6 is not currently supported") + case podIP == "": + lg.Warn("Pod does not have an IPv4 address, and IPv6 is not currently supported") return false, nil } + stateS := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pod.Name, Namespace: pod.Namespace, }, } + err = er.Get(ctx, client.ObjectKeyFromObject(stateS), stateS) - if apierrors.IsNotFound(err) { - l.Debugf("proxy does not have a state Secret, waiting...") + switch { + case apierrors.IsNotFound(err): + lg.Debug("proxy does not yet have a state Secret, waiting...") return false, nil + case err != nil: + return false, fmt.Errorf("error retrieving state Secret: %w", err) } - if err != nil { - return false, fmt.Errorf("error getting state Secret: %w", err) - } + svcStatusBS := stateS.Data[egressservices.KeyEgressServices] if len(svcStatusBS) == 0 { - l.Debugf("proxy's state Secret does not contain egress services status, waiting...") + lg.Debug("proxy's state Secret does not contain egress services status, waiting...") return false, nil } + svcStatus := &egressservices.Status{} - if err := json.Unmarshal(svcStatusBS, svcStatus); err != nil { + if err = json.Unmarshal(svcStatusBS, svcStatus); err != nil { return false, fmt.Errorf("error unmarshalling egress service status: %w", err) } + if !strings.EqualFold(podIP, svcStatus.PodIPv4) { - l.Infof("proxy's egress service status is for Pod IP %s, current proxy's Pod IP %s, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP) + lg.Infof("proxy's egress service status is for Pod IP %q, current proxy's Pod IP %q, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP) return false, nil } - st, ok := (*svcStatus).Services[tailnetSvcName] + + st, ok := svcStatus.Services[tailnetSvcName] if !ok { - l.Infof("proxy's state Secret does not have egress service status, waiting...") + lg.Infof("proxy's state Secret does not have egress service status, waiting...") return false, nil } + if !reflect.DeepEqual(cfg.TailnetTarget, st.TailnetTarget) { - l.Infof("proxy has configured egress service for tailnet target %v, current target is %v, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget) + lg.Infof("proxy has configured egress service for tailnet target %q, current target is %q, waiting for proxy to reconfigure...", st.TailnetTarget, cfg.TailnetTarget) return false, nil } + if !reflect.DeepEqual(cfg.Ports, st.Ports) { - l.Debugf("proxy has configured egress service for ports %#+v, wants ports %#+v, waiting for proxy to reconfigure", st.Ports, cfg.Ports) + lg.Debugf("proxy has configured egress service for ports %#+v, wants ports %#+v, waiting for proxy to reconfigure", st.Ports, cfg.Ports) return false, nil } - l.Debugf("proxy is ready to route traffic to egress service") + + lg.Debug("proxy is ready to route traffic to egress service") return true, nil } diff --git a/cmd/k8s-operator/egress-eps_test.go b/cmd/k8s-operator/egress-eps_test.go index bd81071cb5e4f..6335b4eb8454b 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/cmd/k8s-operator/egress-eps_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -11,7 +11,6 @@ import ( "math/rand/v2" "testing" - "github.com/AlekSi/pointer" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -20,6 +19,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/util/mak" ) @@ -105,11 +105,11 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { expectReconciled(t, er, "operator-ns", "foo") eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{ Addresses: []string{"10.0.0.1"}, - Hostname: pointer.To("foo"), + Hostname: new("foo"), Conditions: discoveryv1.EndpointConditions{ - Serving: pointer.ToBool(true), - Ready: pointer.ToBool(true), - Terminating: pointer.ToBool(false), + Serving: new(true), + Ready: new(true), + Terminating: new(false), }, }) expectEqual(t, fc, eps) @@ -200,7 +200,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: pgSecretLabels(pg, "state"), + Labels: pgSecretLabels(pg, kubetypes.LabelSecretTypeState), }, } return p, s diff --git a/cmd/k8s-operator/egress-pod-readiness.go b/cmd/k8s-operator/egress-pod-readiness.go index 05cf1aa1abfed..17ba0c5ec484b 100644 --- a/cmd/k8s-operator/egress-pod-readiness.go +++ b/cmd/k8s-operator/egress-pod-readiness.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -23,10 +23,11 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" - "tailscale.com/logtail/backoff" "tailscale.com/tstime" + "tailscale.com/util/backoff" "tailscale.com/util/httpm" ) @@ -71,9 +72,9 @@ type egressPodsReconciler struct { // If the Pod does not appear to be serving the health check endpoint (pre-v1.80 proxies), the reconciler just sets the // readiness condition for backwards compatibility reasons. func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { - l := er.logger.With("Pod", req.NamespacedName) - l.Debugf("starting reconcile") - defer l.Debugf("reconcile finished") + lg := er.logger.With("Pod", req.NamespacedName) + lg.Debugf("starting reconcile") + defer lg.Debugf("reconcile finished") pod := new(corev1.Pod) err = er.Get(ctx, req.NamespacedName, pod) @@ -84,11 +85,12 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req return reconcile.Result{}, fmt.Errorf("failed to get Pod: %w", err) } if !pod.DeletionTimestamp.IsZero() { - l.Debugf("Pod is being deleted, do nothing") + lg.Debugf("Pod is being deleted, do nothing") return res, nil } + if pod.Labels[LabelParentType] != proxyTypeProxyGroup { - l.Infof("[unexpected] reconciler called for a Pod that is not a ProxyGroup Pod") + lg.Warn("reconciler called for a Pod that is not a ProxyGroup Pod") return res, nil } @@ -97,7 +99,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req if !slices.ContainsFunc(pod.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool { return r.ConditionType == tsEgressReadinessGate }) { - l.Debug("Pod does not have egress readiness gate set, skipping") + lg.Debug("Pod does not have egress readiness gate set, skipping") return res, nil } @@ -106,10 +108,12 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req if err := er.Get(ctx, types.NamespacedName{Name: proxyGroupName}, pg); err != nil { return res, fmt.Errorf("error getting ProxyGroup %q: %w", proxyGroupName, err) } + if pg.Spec.Type != typeEgress { - l.Infof("[unexpected] reconciler called for %q ProxyGroup Pod", pg.Spec.Type) + lg.Warnf("reconciler called for %q ProxyGroup Pod", pg.Spec.Type) return res, nil } + // Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup. lbls := map[string]string{ kubetypes.LabelManaged: "true", @@ -125,7 +129,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req return c.Type == tsEgressReadinessGate }) if idx != -1 { - l.Debugf("Pod is already ready, do nothing") + lg.Debugf("Pod is already ready, do nothing") return res, nil } @@ -134,7 +138,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req for _, svc := range svcs.Items { s := svc go func() { - ll := l.With("service_name", s.Name) + ll := lg.With("service_name", s.Name) d := retrieveClusterDomain(er.tsNamespace, ll) healthCheckAddr := healthCheckForSvc(&s, d) if healthCheckAddr == "" { @@ -175,25 +179,25 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req err = errors.Join(err, e) } if err != nil { - return res, fmt.Errorf("error verifying conectivity: %w", err) + return res, fmt.Errorf("error verifying connectivity: %w", err) } if rm := routesMissing.Load(); rm { - l.Info("Pod is not yet added as an endpoint for all egress targets, waiting...") + lg.Info("Pod is not yet added as an endpoint for all egress targets, waiting...") return reconcile.Result{RequeueAfter: shortRequeue}, nil } - if err := er.setPodReady(ctx, pod, l); err != nil { + if err := er.setPodReady(ctx, pod, lg); err != nil { return res, fmt.Errorf("error setting Pod as ready: %w", err) } return res, nil } -func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, l *zap.SugaredLogger) error { +func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, lg *zap.SugaredLogger) error { if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool { return c.Type == tsEgressReadinessGate }) { return nil } - l.Infof("Pod is ready to route traffic to all egress targets") + lg.Infof("Pod is ready to route traffic to all egress targets") pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ Type: tsEgressReadinessGate, Status: corev1.ConditionTrue, @@ -216,11 +220,11 @@ const ( ) // lookupPodRouteViaSvc attempts to reach a Pod using a health check endpoint served by a Service and returns the state of the health check. -func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, l *zap.SugaredLogger) (healthCheckState, error) { +func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, lg *zap.SugaredLogger) (healthCheckState, error) { if !slices.ContainsFunc(pod.Spec.Containers[0].Env, func(e corev1.EnvVar) bool { return e.Name == "TS_ENABLE_HEALTH_CHECK" && e.Value == "true" }) { - l.Debugf("Pod does not have health check enabled, unable to verify if it is currently routable via Service") + lg.Debugf("Pod does not have health check enabled, unable to verify if it is currently routable via Service") return cannotVerify, nil } wantsIP, err := podIPv4(pod) @@ -241,14 +245,14 @@ func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *c req.Close = true resp, err := er.httpClient.Do(req) if err != nil { - // This is most likely because this is the first Pod and is not yet added to Service endoints. Other + // This is most likely because this is the first Pod and is not yet added to service endpoints. Other // error types are possible, but checking for those would likely make the system too fragile. return unreachable, nil } defer resp.Body.Close() gotIP := resp.Header.Get(kubetypes.PodIPv4Header) if gotIP == "" { - l.Debugf("Health check does not return Pod's IP header, unable to verify if Pod is currently routable via Service") + lg.Debugf("Health check does not return Pod's IP header, unable to verify if Pod is currently routable via Service") return cannotVerify, nil } if !strings.EqualFold(wantsIP, gotIP) { diff --git a/cmd/k8s-operator/egress-pod-readiness_test.go b/cmd/k8s-operator/egress-pod-readiness_test.go index 3c35d9043ebe6..0cf9108f5cd20 100644 --- a/cmd/k8s-operator/egress-pod-readiness_test.go +++ b/cmd/k8s-operator/egress-pod-readiness_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -24,7 +24,6 @@ import ( tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" - "tailscale.com/types/ptr" ) func TestEgressPodReadiness(t *testing.T) { @@ -48,7 +47,7 @@ func TestEgressPodReadiness(t *testing.T) { }, Spec: tsapi.ProxyGroupSpec{ Type: "egress", - Replicas: ptr.To(int32(3)), + Replicas: new(int32(3)), }, } mustCreate(t, fc, pg) diff --git a/cmd/k8s-operator/egress-services-readiness.go b/cmd/k8s-operator/egress-services-readiness.go index 5e95a52790395..54504af9863ac 100644 --- a/cmd/k8s-operator/egress-services-readiness.go +++ b/cmd/k8s-operator/egress-services-readiness.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -20,6 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstime" @@ -47,13 +48,13 @@ type egressSvcsReadinessReconciler struct { // route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress // service to determine how many replicas are currently able to route traffic. func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { - l := esrr.logger.With("Service", req.NamespacedName) - l.Debugf("starting reconcile") - defer l.Debugf("reconcile finished") + lg := esrr.logger.With("Service", req.NamespacedName) + lg.Debugf("starting reconcile") + defer lg.Debugf("reconcile finished") svc := new(corev1.Service) if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) { - l.Debugf("Service not found") + lg.Debugf("Service not found") return res, nil } else if err != nil { return res, fmt.Errorf("failed to get Service: %w", err) @@ -64,7 +65,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re ) oldStatus := svc.Status.DeepCopy() defer func() { - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, lg) if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) { err = errors.Join(err, esrr.Status().Update(ctx, svc)) } @@ -79,7 +80,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re return res, err } if eps == nil { - l.Infof("EndpointSlice for Service does not yet exist, waiting...") + lg.Infof("EndpointSlice for Service does not yet exist, waiting...") reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady st = metav1.ConditionFalse return res, nil @@ -91,7 +92,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re } err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg) if apierrors.IsNotFound(err) { - l.Infof("ProxyGroup for Service does not exist, waiting...") + lg.Infof("ProxyGroup for Service does not exist, waiting...") reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady st = metav1.ConditionFalse return res, nil @@ -102,8 +103,8 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re msg = err.Error() return res, err } - if !tsoperator.ProxyGroupIsReady(pg) { - l.Infof("ProxyGroup for Service is not ready, waiting...") + if !tsoperator.ProxyGroupAvailable(pg) { + lg.Infof("ProxyGroup for Service is not ready, waiting...") reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady st = metav1.ConditionFalse return res, nil @@ -111,7 +112,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re replicas := pgReplicas(pg) if replicas == 0 { - l.Infof("ProxyGroup replicas set to 0") + lg.Infof("ProxyGroup replicas set to 0") reason, msg = reasonNoProxies, reasonNoProxies st = metav1.ConditionFalse return res, nil @@ -127,17 +128,19 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re msg = err.Error() return res, err } + if pod == nil { - l.Warnf("[unexpected] ProxyGroup is ready, but replica %d was not found", i) + lg.Warnf("ProxyGroup is ready, but replica %d was not found", i) reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady return res, nil } - l.Debugf("looking at Pod with IPs %v", pod.Status.PodIPs) + + lg.Debugf("looking at Pod with IPs %v", pod.Status.PodIPs) ready := false for _, ep := range eps.Endpoints { - l.Debugf("looking at endpoint with addresses %v", ep.Addresses) - if endpointReadyForPod(&ep, pod, l) { - l.Debugf("endpoint is ready for Pod") + lg.Debugf("looking at endpoint with addresses %v", ep.Addresses) + if endpointReadyForPod(&ep, pod, lg) { + lg.Debugf("endpoint is ready for Pod") ready = true break } @@ -163,12 +166,13 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re // endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic. // Endpoint must not be nil. -func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool { +func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, lg *zap.SugaredLogger) bool { podIP, err := podIPv4(pod) if err != nil { - l.Warnf("[unexpected] error retrieving Pod's IPv4 address: %v", err) + lg.Warnf("error retrieving Pod's IPv4 address: %v", err) return false } + // Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this. if len(ep.Addresses) != 1 { return false diff --git a/cmd/k8s-operator/egress-services-readiness_test.go b/cmd/k8s-operator/egress-services-readiness_test.go index ce947329ddfb8..96d76cc4e7252 100644 --- a/cmd/k8s-operator/egress-services-readiness_test.go +++ b/cmd/k8s-operator/egress-services-readiness_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -9,7 +9,6 @@ import ( "fmt" "testing" - "github.com/AlekSi/pointer" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -49,12 +48,12 @@ func TestEgressServiceReadiness(t *testing.T) { }, } fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}} - l := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "operator-ns", - Labels: l, + Labels: labels, }, AddressType: discoveryv1.AddressTypeIPv4, } @@ -118,26 +117,26 @@ func TestEgressServiceReadiness(t *testing.T) { }) } -func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger) { - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, l) +func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger) { + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, lg) } -func setNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas int32) { +func setNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, replicas int32) { msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, l) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, lg) } -func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas, readyReplicas int32) { +func setReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, replicas, readyReplicas int32) { reason := reasonPartiallyReady if readyReplicas == replicas { reason = reasonReady } msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, l) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, lg) } -func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) { - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l) +func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, lg *zap.SugaredLogger) { + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, lg) } func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) { @@ -145,22 +144,22 @@ func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1 eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{ Addresses: []string{p.Status.PodIPs[0].IP}, Conditions: discoveryv1.EndpointConditions{ - Ready: pointer.ToBool(true), - Serving: pointer.ToBool(true), - Terminating: pointer.ToBool(false), + Ready: new(true), + Serving: new(true), + Terminating: new(false), }, }) } func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod { - l := pgLabels(pg.Name, nil) - l[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal) + labels := pgLabels(pg.Name, nil) + labels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal) ip := fmt.Sprintf("10.0.0.%d", ordinal) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%d", pg.Name, ordinal), Namespace: "operator-ns", - Labels: l, + Labels: labels, }, Status: corev1.PodStatus{ PodIPs: []corev1.PodIP{{IP: ip}}, diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 7103205ac2c3f..b9a3f8eaba799 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" @@ -98,12 +99,12 @@ type egressSvcsReconciler struct { // - updates the egress service config in a ConfigMap mounted to the ProxyGroup proxies with the tailnet target and the // portmappings. func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { - l := esr.logger.With("Service", req.NamespacedName) - defer l.Info("reconcile finished") + lg := esr.logger.With("Service", req.NamespacedName) + defer lg.Info("reconcile finished") svc := new(corev1.Service) if err = esr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) { - l.Info("Service not found") + lg.Info("Service not found") return res, nil } else if err != nil { return res, fmt.Errorf("failed to get Service: %w", err) @@ -111,7 +112,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re // Name of the 'egress service', meaning the tailnet target. tailnetSvc := tailnetSvcName(svc) - l = l.With("tailnet-service", tailnetSvc) + lg = lg.With("tailnet-service", tailnetSvc) // Note that resources for egress Services are only cleaned up when the // Service is actually deleted (and not if, for example, user decides to @@ -119,8 +120,8 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re // assume that the egress ExternalName Services are always created for // Tailscale operator specifically. if !svc.DeletionTimestamp.IsZero() { - l.Info("Service is being deleted, ensuring resource cleanup") - return res, esr.maybeCleanup(ctx, svc, l) + lg.Info("Service is being deleted, ensuring resource cleanup") + return res, esr.maybeCleanup(ctx, svc, lg) } oldStatus := svc.Status.DeepCopy() @@ -131,7 +132,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re }() // Validate the user-created ExternalName Service and the associated ProxyGroup. - if ok, err := esr.validateClusterResources(ctx, svc, l); err != nil { + if ok, err := esr.validateClusterResources(ctx, svc, lg); err != nil { return res, fmt.Errorf("error validating cluster resources: %w", err) } else if !ok { return res, nil @@ -141,8 +142,8 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re svc.Finalizers = append(svc.Finalizers, FinalizerName) if err := esr.updateSvcSpec(ctx, svc); err != nil { err := fmt.Errorf("failed to add finalizer: %w", err) - r := svcConfiguredReason(svc, false, l) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l) + r := svcConfiguredReason(svc, false, lg) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg) return res, err } esr.mu.Lock() @@ -151,16 +152,16 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re esr.mu.Unlock() } - if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, l); err != nil { + if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, lg); err != nil { err = fmt.Errorf("cleaning up resources for previous ProxyGroup failed: %w", err) - r := svcConfiguredReason(svc, false, l) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l) + r := svcConfiguredReason(svc, false, lg) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg) return res, err } - if err := esr.maybeProvision(ctx, svc, l); err != nil { + if err := esr.maybeProvision(ctx, svc, lg); err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { - l.Infof("optimistic lock error, retrying: %s", err) + lg.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err } @@ -169,15 +170,15 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re return res, nil } -func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (err error) { - r := svcConfiguredReason(svc, false, l) +func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (err error) { + r := svcConfiguredReason(svc, false, lg) st := metav1.ConditionFalse defer func() { msg := r if st != metav1.ConditionTrue && err != nil { msg = err.Error() } - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, l) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, lg) }() crl := egressSvcChildResourceLabels(svc) @@ -189,36 +190,36 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1 if clusterIPSvc == nil { clusterIPSvc = esr.clusterIPSvcForEgress(crl) } - upToDate := svcConfigurationUpToDate(svc, l) + upToDate := svcConfigurationUpToDate(svc, lg) provisioned := true if !upToDate { - if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, l); err != nil { + if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, lg); err != nil { return err } } if !provisioned { - l.Infof("unable to provision cluster resources") + lg.Infof("unable to provision cluster resources") return nil } // Update ExternalName Service to point at the ClusterIP Service. - clusterDomain := retrieveClusterDomain(esr.tsNamespace, l) + clusterDomain := retrieveClusterDomain(esr.tsNamespace, lg) clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain) if svc.Spec.ExternalName != clusterIPSvcFQDN { - l.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN) + lg.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN) svc.Spec.ExternalName = clusterIPSvcFQDN if err = esr.updateSvcSpec(ctx, svc); err != nil { err = fmt.Errorf("error updating ExternalName Service: %w", err) return err } } - r = svcConfiguredReason(svc, true, l) + r = svcConfiguredReason(svc, true, lg) st = metav1.ConditionTrue return nil } -func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, l *zap.SugaredLogger) (*corev1.Service, bool, error) { - l.Infof("updating configuration...") +func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) (*corev1.Service, bool, error) { + lg.Infof("updating configuration...") usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName) if err != nil { return nil, false, fmt.Errorf("error calculating used ports for ProxyGroup %s: %w", proxyGroupName, err) @@ -246,7 +247,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s } } if !found { - l.Debugf("portmapping %s:%d -> %s:%d is no longer required, removing", pm.Protocol, pm.TargetPort.IntVal, pm.Protocol, pm.Port) + lg.Debugf("portmapping %s:%d -> %s:%d is no longer required, removing", pm.Protocol, pm.TargetPort.IntVal, pm.Protocol, pm.Port) clusterIPSvc.Spec.Ports = slices.Delete(clusterIPSvc.Spec.Ports, i, i+1) } } @@ -277,7 +278,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s return nil, false, fmt.Errorf("unable to allocate additional ports on ProxyGroup %s, %d ports already used. Create another ProxyGroup or open an issue if you believe this is unexpected.", proxyGroupName, maxPorts) } p := unusedPort(usedPorts) - l.Debugf("mapping tailnet target port %d to container port %d", wantsPM.Port, p) + lg.Debugf("mapping tailnet target port %d to container port %d", wantsPM.Port, p) usedPorts.Insert(p) clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{ Name: wantsPM.Name, @@ -343,15 +344,15 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s return nil, false, fmt.Errorf("error retrieving egress services configuration: %w", err) } if cm == nil { - l.Info("ConfigMap not yet created, waiting..") + lg.Info("ConfigMap not yet created, waiting..") return nil, false, nil } tailnetSvc := tailnetSvcName(svc) - gotCfg := (*cfgs)[tailnetSvc] - wantsCfg := egressSvcCfg(svc, clusterIPSvc, esr.tsNamespace, l) + gotCfg := cfgs[tailnetSvc] + wantsCfg := egressSvcCfg(svc, clusterIPSvc, esr.tsNamespace, lg) if !reflect.DeepEqual(gotCfg, wantsCfg) { - l.Debugf("updating egress services ConfigMap %s", cm.Name) - mak.Set(cfgs, tailnetSvc, wantsCfg) + lg.Debugf("updating egress services ConfigMap %s", cm.Name) + mak.Set(&cfgs, tailnetSvc, wantsCfg) bs, err := json.Marshal(cfgs) if err != nil { return nil, false, fmt.Errorf("error marshalling egress services configs: %w", err) @@ -361,7 +362,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s return nil, false, fmt.Errorf("error updating egress services ConfigMap: %w", err) } } - l.Infof("egress service configuration has been updated") + lg.Infof("egress service configuration has been updated") return clusterIPSvc, true, nil } @@ -402,7 +403,7 @@ func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.S return nil } -func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) error { +func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) error { wantsProxyGroup := svc.Annotations[AnnotationProxyGroup] cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured) if cond == nil { @@ -416,7 +417,7 @@ func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Contex return nil } esr.logger.Infof("egress Service configured on ProxyGroup %s, wants ProxyGroup %s, cleaning up...", ss[2], wantsProxyGroup) - if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, l); err != nil { + if err := esr.ensureEgressSvcCfgDeleted(ctx, svc, lg); err != nil { return fmt.Errorf("error deleting egress service config: %w", err) } return nil @@ -457,7 +458,8 @@ func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *c Labels: crl, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, + Type: corev1.ServiceTypeClusterIP, + IPFamilyPolicy: new(corev1.IPFamilyPolicyPreferDualStack), }, } } @@ -471,32 +473,32 @@ func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, Namespace: esr.tsNamespace, }, } - l := logger.With("ConfigMap", client.ObjectKeyFromObject(cm)) - l.Debug("ensuring that egress service configuration is removed from proxy config") + lggr := logger.With("ConfigMap", client.ObjectKeyFromObject(cm)) + lggr.Debug("ensuring that egress service configuration is removed from proxy config") if err := esr.Get(ctx, client.ObjectKeyFromObject(cm), cm); apierrors.IsNotFound(err) { - l.Debugf("ConfigMap not found") + lggr.Debugf("ConfigMap not found") return nil } else if err != nil { return fmt.Errorf("error retrieving ConfigMap: %w", err) } bs := cm.BinaryData[egressservices.KeyEgressServices] if len(bs) == 0 { - l.Debugf("ConfigMap does not contain egress service configs") + lggr.Debugf("ConfigMap does not contain egress service configs") return nil } - cfgs := &egressservices.Configs{} - if err := json.Unmarshal(bs, cfgs); err != nil { + cfgs := egressservices.Configs{} + if err := json.Unmarshal(bs, &cfgs); err != nil { return fmt.Errorf("error unmarshalling egress services configs") } tailnetSvc := tailnetSvcName(svc) - _, ok := (*cfgs)[tailnetSvc] + _, ok := cfgs[tailnetSvc] if !ok { - l.Debugf("ConfigMap does not contain egress service config, likely because it was already deleted") + lggr.Debugf("ConfigMap does not contain egress service config, likely because it was already deleted") return nil } - l.Infof("before deleting config %+#v", *cfgs) - delete(*cfgs, tailnetSvc) - l.Infof("after deleting config %+#v", *cfgs) + lggr.Infof("before deleting config %+#v", cfgs) + delete(cfgs, tailnetSvc) + lggr.Infof("after deleting config %+#v", cfgs) bs, err := json.Marshal(cfgs) if err != nil { return fmt.Errorf("error marshalling egress services configs: %w", err) @@ -505,7 +507,7 @@ func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, return esr.Update(ctx, cm) } -func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (bool, error) { +func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (bool, error) { proxyGroupName := svc.Annotations[AnnotationProxyGroup] pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -513,36 +515,36 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s }, } if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) { - l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) + lg.Infof("ProxyGroup %q not found, waiting...", proxyGroupName) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } else if err != nil { err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, lg) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, err } if violations := validateEgressService(svc, pg); len(violations) > 0 { msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", ")) esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) - l.Info(msg) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l) + lg.Info(msg) + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, lg) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } - if !tsoperator.ProxyGroupIsReady(pg) { - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) + if !tsoperator.ProxyGroupAvailable(pg) { + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) } - l.Debugf("egress service is valid") - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l) + lg.Debugf("egress service is valid") + tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, lg) return true, nil } -func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, l *zap.SugaredLogger) egressservices.Config { - d := retrieveClusterDomain(ns, l) +func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, lg *zap.SugaredLogger) egressservices.Config { + d := retrieveClusterDomain(ns, lg) tt := tailnetTargetFromSvc(externalNameSvc) hep := healthCheckForSvc(clusterIPSvc, d) cfg := egressservices.Config{ @@ -648,7 +650,7 @@ func isEgressSvcForProxyGroup(obj client.Object) bool { // egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well // as unmarshalled configuration from the ConfigMap. -func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) { +func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs egressservices.Configs, err error) { name := pgEgressCMName(proxyGroupName) cm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -663,9 +665,9 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts if err != nil { return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err) } - cfgs = &egressservices.Configs{} + cfgs = egressservices.Configs{} if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 { - if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], cfgs); err != nil { + if err := json.Unmarshal(cm.BinaryData[egressservices.KeyEgressServices], &cfgs); err != nil { return nil, nil, fmt.Errorf("error unmarshaling egress services config %v: %w", cm.BinaryData[egressservices.KeyEgressServices], err) } } @@ -691,18 +693,18 @@ func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string { // egressEpsLabels returns labels to be added to an EndpointSlice created for an egress service. func egressSvcEpsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string { - l := egressSvcChildResourceLabels(extNSvc) + lbels := egressSvcChildResourceLabels(extNSvc) // Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the // endpoints defined on this EndpointSlice. // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership - l[discoveryv1.LabelServiceName] = clusterIPSvc.Name + lbels[discoveryv1.LabelServiceName] = clusterIPSvc.Name // Kubernetes recommends setting this label. // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management - l[discoveryv1.LabelManagedBy] = "tailscale.com" - return l + lbels[discoveryv1.LabelManagedBy] = "tailscale.com" + return lbels } -func svcConfigurationUpToDate(svc *corev1.Service, l *zap.SugaredLogger) bool { +func svcConfigurationUpToDate(svc *corev1.Service, lg *zap.SugaredLogger) bool { cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured) if cond == nil { return false @@ -710,21 +712,21 @@ func svcConfigurationUpToDate(svc *corev1.Service, l *zap.SugaredLogger) bool { if cond.Status != metav1.ConditionTrue { return false } - wantsReadyReason := svcConfiguredReason(svc, true, l) + wantsReadyReason := svcConfiguredReason(svc, true, lg) return strings.EqualFold(wantsReadyReason, cond.Reason) } -func cfgHash(c cfg, l *zap.SugaredLogger) string { +func cfgHash(c cfg, lg *zap.SugaredLogger) string { bs, err := json.Marshal(c) if err != nil { // Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace. - l.Infof("error marhsalling Config: %v", err) + lg.Infof("error marhsalling Config: %v", err) return "" } h := sha256.New() if _, err := h.Write(bs); err != nil { // Don't use l.Error as that messes up component logs with, in this case, unnecessary stack trace. - l.Infof("error producing Config hash: %v", err) + lg.Infof("error producing Config hash: %v", err) return "" } return fmt.Sprintf("%x", h.Sum(nil)) @@ -736,7 +738,7 @@ type cfg struct { ProxyGroup string `json:"proxyGroup"` } -func svcConfiguredReason(svc *corev1.Service, configured bool, l *zap.SugaredLogger) string { +func svcConfiguredReason(svc *corev1.Service, configured bool, lg *zap.SugaredLogger) string { var r string if configured { r = "ConfiguredFor:" @@ -750,7 +752,7 @@ func svcConfiguredReason(svc *corev1.Service, configured bool, l *zap.SugaredLog TailnetTarget: tt, ProxyGroup: svc.Annotations[AnnotationProxyGroup], } - r += fmt.Sprintf(":Config:%s", cfgHash(s, l)) + r += fmt.Sprintf(":Config:%s", cfgHash(s, lg)) return r } diff --git a/cmd/k8s-operator/egress-services_test.go b/cmd/k8s-operator/egress-services_test.go index d8a5dfd32c1c2..a7dd79f7f1f84 100644 --- a/cmd/k8s-operator/egress-services_test.go +++ b/cmd/k8s-operator/egress-services_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" "tailscale.com/tstest" @@ -203,8 +204,9 @@ func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service { Labels: labels, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Ports: ports, + Type: corev1.ServiceTypeClusterIP, + IPFamilyPolicy: new(corev1.IPFamilyPolicyPreferDualStack), + Ports: ports, }, } } @@ -243,15 +245,15 @@ func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort { ports = append(ports, discoveryv1.EndpointPort{ Name: &p.Name, Protocol: &p.Protocol, - Port: pointer.ToInt32(p.TargetPort.IntVal), + Port: new(p.TargetPort.IntVal), }) } return ports } -func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap, l *zap.Logger) { +func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap, lg *zap.Logger) { t.Helper() - wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc, clusterIPSvc.Namespace, l.Sugar()) + wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc, clusterIPSvc.Namespace, lg.Sugar()) if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil { t.Fatalf("Error retrieving ConfigMap: %v", err) } @@ -283,11 +285,11 @@ func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressser if !ok { return nil } - cfgs := &egressservices.Configs{} - if err := json.Unmarshal(cfgBs, cfgs); err != nil { + cfgs := egressservices.Configs{} + if err := json.Unmarshal(cfgBs, &cfgs); err != nil { t.Fatalf("error unmarshalling config: %v", err) } - cfg, ok := (*cfgs)[svcName] + cfg, ok := cfgs[svcName] if ok { return &cfg } diff --git a/cmd/k8s-operator/generate/main.go b/cmd/k8s-operator/generate/main.go index 25435a47cf14a..840812ea3b248 100644 --- a/cmd/k8s-operator/generate/main.go +++ b/cmd/k8s-operator/generate/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -20,18 +20,22 @@ import ( ) const ( - operatorDeploymentFilesPath = "cmd/k8s-operator/deploy" - connectorCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml" - proxyClassCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml" - dnsConfigCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml" - recorderCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml" - proxyGroupCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml" - helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates" - connectorCRDHelmTemplatePath = helmTemplatesPath + "/connector.yaml" - proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml" - dnsConfigCRDHelmTemplatePath = helmTemplatesPath + "/dnsconfig.yaml" - recorderCRDHelmTemplatePath = helmTemplatesPath + "/recorder.yaml" - proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml" + operatorDeploymentFilesPath = "cmd/k8s-operator/deploy" + connectorCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml" + proxyClassCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml" + dnsConfigCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml" + recorderCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml" + proxyGroupCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml" + tailnetCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_tailnets.yaml" + proxyGroupPolicyCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygrouppolicies.yaml" + helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates" + connectorCRDHelmTemplatePath = helmTemplatesPath + "/connector.yaml" + proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml" + dnsConfigCRDHelmTemplatePath = helmTemplatesPath + "/dnsconfig.yaml" + recorderCRDHelmTemplatePath = helmTemplatesPath + "/recorder.yaml" + proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml" + tailnetCRDHelmTemplatePath = helmTemplatesPath + "/tailnet.yaml" + proxyGroupPolicyCRDHelmTemplatePath = helmTemplatesPath + "/proxygrouppolicy.yaml" helmConditionalStart = "{{ if .Values.installCRDs -}}\n" helmConditionalEnd = "{{- end -}}" @@ -41,11 +45,16 @@ func main() { if len(os.Args) < 2 { log.Fatalf("usage ./generate [staticmanifests|helmcrd]") } - repoRoot := "../../" + gitOut, err := exec.Command("git", "rev-parse", "--show-toplevel").CombinedOutput() + if err != nil { + log.Fatalf("error determining git root: %v: %s", err, gitOut) + } + + repoRoot := strings.TrimSpace(string(gitOut)) switch os.Args[1] { case "helmcrd": // insert CRDs to Helm templates behind a installCRDs=true conditional check log.Print("Adding CRDs to Helm templates") - if err := generate("./"); err != nil { + if err := generate(repoRoot); err != nil { log.Fatalf("error adding CRDs to Helm templates: %v", err) } return @@ -64,7 +73,7 @@ func main() { }() log.Print("Templating Helm chart contents") helmTmplCmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart", - "--namespace=tailscale") + "--namespace=tailscale", "--set=oauth.clientSecret=''") helmTmplCmd.Dir = repoRoot var out bytes.Buffer helmTmplCmd.Stdout = &out @@ -139,7 +148,7 @@ func generate(baseDir string) error { if _, err := file.Write([]byte(helmConditionalEnd)); err != nil { return fmt.Errorf("error writing helm if-statement end: %w", err) } - return nil + return file.Close() } for _, crd := range []struct { crdPath, templatePath string @@ -149,6 +158,8 @@ func generate(baseDir string) error { {dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath}, {recorderCRDPath, recorderCRDHelmTemplatePath}, {proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath}, + {tailnetCRDPath, tailnetCRDHelmTemplatePath}, + {proxyGroupPolicyCRDPath, proxyGroupPolicyCRDHelmTemplatePath}, } { if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil { return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err) @@ -165,6 +176,8 @@ func cleanup(baseDir string) error { dnsConfigCRDHelmTemplatePath, recorderCRDHelmTemplatePath, proxyGroupCRDHelmTemplatePath, + tailnetCRDHelmTemplatePath, + proxyGroupPolicyCRDHelmTemplatePath, } { if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) { return fmt.Errorf("error cleaning up %s: %w", path, err) diff --git a/cmd/k8s-operator/generate/main_test.go b/cmd/k8s-operator/generate/main_test.go index c7956dcdbef8f..775d16ba1e827 100644 --- a/cmd/k8s-operator/generate/main_test.go +++ b/cmd/k8s-operator/generate/main_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 && !windows @@ -7,26 +7,50 @@ package main import ( "bytes" + "context" + "net" "os" "os/exec" "path/filepath" "strings" "testing" + "time" + + "tailscale.com/tstest/nettest" + "tailscale.com/util/cibuild" ) func Test_generate(t *testing.T) { + nettest.SkipIfNoNetwork(t) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + if _, err := net.DefaultResolver.LookupIPAddr(ctx, "get.helm.sh"); err != nil { + // https://github.com/helm/helm/issues/31434 + t.Skipf("get.helm.sh seems down or unreachable; skipping test") + } + base, err := os.Getwd() base = filepath.Join(base, "../../../") if err != nil { t.Fatalf("error getting current working directory: %v", err) } defer cleanup(base) + + helmCLIPath := filepath.Join(base, "tool/helm") + if out, err := exec.Command(helmCLIPath, "version").CombinedOutput(); err != nil && cibuild.On() { + // It's not just DNS. Azure is generating bogus certs within GitHub Actions at least for + // helm. So try to run it and see if we can even fetch it. + // + // https://github.com/helm/helm/issues/31434 + t.Skipf("error fetching helm; skipping test in CI: %v, %s", err, out) + } + if err := generate(base); err != nil { t.Fatalf("CRD template generation: %v", err) } tempDir := t.TempDir() - helmCLIPath := filepath.Join(base, "tool/helm") helmChartTemplatesPath := filepath.Join(base, "cmd/k8s-operator/deploy/chart") helmPackageCmd := exec.Command(helmCLIPath, "package", helmChartTemplatesPath, "--destination", tempDir, "--version", "0.0.1") helmPackageCmd.Stderr = os.Stderr diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index ea31dbd63befb..6cacec61efce1 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -10,8 +10,8 @@ import ( "encoding/json" "errors" "fmt" + "maps" "math/rand/v2" - "net/http" "reflect" "slices" "strings" @@ -29,11 +29,12 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "tailscale.com/internal/client/tailscale" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" - "tailscale.com/ipn/ipnstate" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/util/clientmetric" @@ -43,22 +44,15 @@ import ( ) const ( - serveConfigKey = "serve-config.json" - TailscaleSvcOwnerRef = "tailscale.com/k8s-operator:owned-by:%s" + serveConfigKey = "serve-config.json" // FinalizerNamePG is the finalizer used by the IngressPGReconciler - FinalizerNamePG = "tailscale.com/ingress-pg-finalizer" - + FinalizerNamePG = "tailscale.com/ingress-pg-finalizer" indexIngressProxyGroup = ".metadata.annotations.ingress-proxy-group" // annotationHTTPEndpoint can be used to configure the Ingress to expose an HTTP endpoint to tailnet (as // well as the default HTTPS endpoint). - annotationHTTPEndpoint = "tailscale.com/http-endpoint" - - labelDomain = "tailscale.com/domain" - msgFeatureFlagNotEnabled = "Tailscale Service feature flag is not enabled for this tailnet, skipping provisioning. " + - "Please contact Tailscale support through https://tailscale.com/contact/support to enable the feature flag, then recreate the operator's Pod." - - warningTailscaleServiceFeatureFlagNotEnabled = "TailscaleServiceFeatureFlagNotEnabled" - managedTSServiceComment = "This Tailscale Service is managed by the Tailscale Kubernetes Operator, do not modify" + annotationHTTPEndpoint = "tailscale.com/http-endpoint" + labelDomain = "tailscale.com/domain" + managedTSServiceComment = "This Tailscale Service is managed by the Tailscale Kubernetes Operator, do not modify" ) var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount) @@ -68,14 +62,14 @@ var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGRes type HAIngressReconciler struct { client.Client - recorder record.EventRecorder - logger *zap.SugaredLogger - tsClient tsClient - tsnetServer tsnetServer - tsNamespace string - lc localClient - defaultTags []string - operatorID string // stableID of the operator's Tailscale device + recorder record.EventRecorder + logger *zap.SugaredLogger + clients ClientProvider + tsnetServer tsnetServer + tsNamespace string + defaultTags []string + operatorID string // stableID of the operator's Tailscale device + ingressClassName string mu sync.Mutex // protects following // managedIngresses is a set of all ingress resources that we're currently @@ -106,11 +100,12 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque ing := new(networkingv1.Ingress) err = r.Get(ctx, req.NamespacedName, ing) - if apierrors.IsNotFound(err) { + switch { + case apierrors.IsNotFound(err): // Request object not found, could have been deleted after reconcile request. logger.Debugf("Ingress not found, assuming it was deleted") return res, nil - } else if err != nil { + case err != nil: return res, fmt.Errorf("failed to get Ingress: %w", err) } @@ -120,6 +115,23 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque hostname := hostnameForIngress(ing) logger = logger.With("hostname", hostname) + pgName := ing.Annotations[AnnotationProxyGroup] + pg := &tsapi.ProxyGroup{} + + err = r.Get(ctx, client.ObjectKey{Name: pgName}, pg) + switch { + case apierrors.IsNotFound(err): + logger.Infof("ProxyGroup %q does not exist, it may have been deleted. Reconciliation for ingress %q will be skipped until the ProxyGroup is found", pgName, ing.Name) + return res, nil + case err != nil: + return res, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) + } + + tsClient, err := r.clients.For(pg.Spec.Tailnet) + if err != nil { + return res, fmt.Errorf("failed to get tailscale client: %w", err) + } + // needsRequeue is set to true if the underlying Tailscale Service has // changed as a result of this reconcile. If that is the case, we // reconcile the Ingress one more time to ensure that concurrent updates @@ -127,9 +139,9 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque // resulted in another actor overwriting our Tailscale Service update. needsRequeue := false if !ing.DeletionTimestamp.IsZero() || !r.shouldExpose(ing) { - needsRequeue, err = r.maybeCleanup(ctx, hostname, ing, logger) + needsRequeue, err = r.maybeCleanup(ctx, hostname, ing, logger, tsClient, pg) } else { - needsRequeue, err = r.maybeProvision(ctx, hostname, ing, logger) + needsRequeue, err = r.maybeProvision(ctx, hostname, ing, logger, tsClient, pg) } if err != nil { return res, err @@ -148,41 +160,28 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque // If a Tailscale Service exists, but does not have an owner reference from any operator, we error // out assuming that this is an owner reference created by an unknown actor. // Returns true if the operation resulted in a Tailscale Service update. -func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) (svcsChanged bool, err error) { +func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsclient.Client, pg *tsapi.ProxyGroup) (svcsChanged bool, err error) { // Currently (2025-05) Tailscale Services are behind an alpha feature flag that // needs to be explicitly enabled for a tailnet to be able to use them. serviceName := tailcfg.ServiceName("svc:" + hostname) - existingTSSvc, err := r.tsClient.GetVIPService(ctx, serviceName) - if isErrorFeatureFlagNotEnabled(err) { - logger.Warn(msgFeatureFlagNotEnabled) - r.recorder.Event(ing, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msgFeatureFlagNotEnabled) - return false, nil - } - if err != nil && !isErrorTailscaleServiceNotFound(err) { + existingTSSvc, err := tsClient.VIPServices().Get(ctx, serviceName.String()) + if err != nil && !tailscale.IsNotFound(err) { return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) } - if err := validateIngressClass(ctx, r.Client); err != nil { + if err = validateIngressClass(ctx, r.Client, r.ingressClassName); err != nil { logger.Infof("error validating tailscale IngressClass: %v.", err) return false, nil } - // Get and validate ProxyGroup readiness + + // We only act on services that are annotated as using a proxy group. pgName := ing.Annotations[AnnotationProxyGroup] if pgName == "" { - logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning") return false, nil } - logger = logger.With("ProxyGroup", pgName) - pg := &tsapi.ProxyGroup{} - if err := r.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil { - if apierrors.IsNotFound(err) { - logger.Infof("ProxyGroup does not exist") - return false, nil - } - return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) - } - if !tsoperator.ProxyGroupIsReady(pg) { + logger = logger.With("ProxyGroup", pgName) + if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return false, nil } @@ -222,7 +221,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // that in edge cases (a single update changed both hostname and removed // ProxyGroup annotation) the Tailscale Service is more likely to be // (eventually) removed. - svcsChanged, err = r.maybeCleanupProxyGroup(ctx, pgName, logger) + svcsChanged, err = r.maybeCleanupProxyGroup(ctx, logger, tsClient, pg) if err != nil { return false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err) } @@ -238,7 +237,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Ingress and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition" msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -247,12 +246,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, nil } // 3. Ensure that TLS Secret and RBAC exists - tcd, err := r.tailnetCertDomain(ctx) + dnsName, err := dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) if err != nil { - return false, fmt.Errorf("error determining DNS name base: %w", err) + return false, fmt.Errorf("error determining DNS name for service: %w", err) } - dnsName := hostname + "." + tcd - if err := r.ensureCertResources(ctx, pg, dnsName, ing); err != nil { + + if err = r.ensureCertResources(ctx, pg, dnsName, ing); err != nil { return false, fmt.Errorf("error ensuring cert resources: %w", err) } @@ -293,6 +292,25 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin ingCfg.Web[epHTTP] = &ipn.WebServerConfig{ Handlers: handlers, } + if isHTTPRedirectEnabled(ing) { + logger.Warnf("Both HTTP endpoint and HTTP redirect flags are enabled: ignoring HTTP redirect.") + } + } else if isHTTPRedirectEnabled(ing) { + logger.Infof("HTTP redirect enabled, setting up port 80 redirect handlers") + epHTTP := ipn.HostPort(fmt.Sprintf("%s:80", dnsName)) + ingCfg.TCP[80] = &ipn.TCPPortHandler{HTTP: true} + ingCfg.Web[epHTTP] = &ipn.WebServerConfig{ + Handlers: map[string]*ipn.HTTPHandler{}, + } + web80 := ingCfg.Web[epHTTP] + for mountPoint := range handlers { + // We send a 301 - Moved Permanently redirect from HTTP to HTTPS + redirectURL := "301:https://${HOST}${REQUEST_URI}" + logger.Debugf("Creating redirect handler: %s -> %s", mountPoint, redirectURL) + web80.Handlers[mountPoint] = &ipn.HTTPHandler{ + Redirect: redirectURL, + } + } } var gotCfg *ipn.ServiceConfig @@ -319,12 +337,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } tsSvcPorts := []string{"tcp:443"} // always 443 for Ingress - if isHTTPEndpointEnabled(ing) { + if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) { tsSvcPorts = append(tsSvcPorts, "tcp:80") } - tsSvc := &tailscale.VIPService{ - Name: serviceName, + tsSvc := tailscale.VIPService{ + Name: serviceName.String(), Tags: tags, Ports: tsSvcPorts, Comment: managedTSServiceComment, @@ -339,9 +357,9 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin if existingTSSvc == nil || !reflect.DeepEqual(tsSvc.Tags, existingTSSvc.Tags) || !reflect.DeepEqual(tsSvc.Ports, existingTSSvc.Ports) || - !ownersAreSetAndEqual(tsSvc, existingTSSvc) { + !ownersAreSetAndEqual(tsSvc, *existingTSSvc) { logger.Infof("Ensuring Tailscale Service exists and is up to date") - if err := r.tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil { + if err := tsClient.VIPServices().CreateOrUpdate(ctx, tsSvc); err != nil { return false, fmt.Errorf("error creating Tailscale Service: %w", err) } } @@ -349,15 +367,15 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // 5. Update tailscaled's AdvertiseServices config, which should add the Tailscale Service // IPs to the ProxyGroup Pods' AllowedIPs in the next netmap update if approved. mode := serviceAdvertisementHTTPS - if isHTTPEndpointEnabled(ing) { + if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) { mode = serviceAdvertisementHTTPAndHTTPS } - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, mode, logger); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg); err != nil { return false, fmt.Errorf("failed to update tailscaled config: %w", err) } // 6. Update Ingress status if ProxyGroup Pods are ready. - count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName) + count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName.String()) if err != nil { return false, fmt.Errorf("failed to check if any Pods are configured: %w", err) } @@ -369,7 +387,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin ing.Status.LoadBalancer.Ingress = nil default: var ports []networkingv1.IngressPortStatus - hasCerts, err := r.hasCerts(ctx, serviceName) + hasCerts, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) } @@ -380,7 +398,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin Port: 443, }) } - if isHTTPEndpointEnabled(ing) { + if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) { ports = append(ports, networkingv1.IngressPortStatus{ Protocol: "TCP", Port: 80, @@ -409,9 +427,10 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin logger.Infof("%s. %d Pod(s) advertising Tailscale Service", prefix, count) } - if err := r.Status().Update(ctx, ing); err != nil { + if err = r.Status().Update(ctx, ing); err != nil { return false, fmt.Errorf("failed to update Ingress status: %w", err) } + return svcsChanged, nil } @@ -421,9 +440,9 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // operator instances, else the owner reference is cleaned up. Returns true if // the operation resulted in an existing Tailscale Service updates (owner // reference removal). -func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger) (svcsChanged bool, err error) { +func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger *zap.SugaredLogger, tsClient tsclient.Client, pg *tsapi.ProxyGroup) (svcsChanged bool, err error) { // Get serve config for the ProxyGroup - cm, cfg, err := r.proxyGroupServeConfig(ctx, proxyGroupName) + cm, cfg, err := r.proxyGroupServeConfig(ctx, pg.Name) if err != nil { return false, fmt.Errorf("getting serve config: %w", err) } @@ -451,36 +470,33 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG if !found { logger.Infof("Tailscale Service %q is not owned by any Ingress, cleaning up", tsSvcName) - tsService, err := r.tsClient.GetVIPService(ctx, tsSvcName) - if isErrorFeatureFlagNotEnabled(err) { - msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled) - logger.Warn(msg) - return false, nil - } - if isErrorTailscaleServiceNotFound(err) { + tsService, err := tsClient.VIPServices().Get(ctx, tsSvcName.String()) + switch { + case tailscale.IsNotFound(err): return false, nil - } - if err != nil { + case err != nil: return false, fmt.Errorf("getting Tailscale Service %q: %w", tsSvcName, err) } // Delete the Tailscale Service from control if necessary. - svcsChanged, err = r.cleanupTailscaleService(ctx, tsService, logger) + svcsChanged, err = r.cleanupTailscaleService(ctx, tsService, logger, tsClient) if err != nil { return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err) } // Make sure the Tailscale Service is not advertised in tailscaled or serve config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, tsSvcName, serviceAdvertisementOff, logger); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } + _, ok := cfg.Services[tsSvcName] if ok { logger.Infof("Removing Tailscale Service %q from serve config", tsSvcName) delete(cfg.Services, tsSvcName) serveConfigChanged = true } - if err := r.cleanupCertResources(ctx, proxyGroupName, tsSvcName); err != nil { + + if err = cleanupCertResources(ctx, r.Client, r.tsNamespace, tsSvcName, pg); err != nil { return false, fmt.Errorf("failed to clean up cert resources: %w", err) } } @@ -503,26 +519,19 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG // Ingress is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only // deleted if it does not contain any other owner references. If it does the cleanup only removes the owner reference // corresponding to this Ingress. -func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) (svcChanged bool, err error) { +func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsclient.Client, pg *tsapi.ProxyGroup) (svcChanged bool, err error) { logger.Debugf("Ensuring any resources for Ingress are cleaned up") ix := slices.Index(ing.Finalizers, FinalizerNamePG) if ix < 0 { logger.Debugf("no finalizer, nothing to do") return false, nil } + logger.Infof("Ensuring that Tailscale Service %q configuration is cleaned up", hostname) serviceName := tailcfg.ServiceName("svc:" + hostname) - svc, err := r.tsClient.GetVIPService(ctx, serviceName) - if err != nil { - if isErrorFeatureFlagNotEnabled(err) { - msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled) - logger.Warn(msg) - r.recorder.Event(ing, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msg) - return false, nil - } - if isErrorTailscaleServiceNotFound(err) { - return false, nil - } + + svc, err := tsClient.VIPServices().Get(ctx, serviceName.String()) + if err != nil && !tailscale.IsNotFound(err) { return false, fmt.Errorf("error getting Tailscale Service: %w", err) } @@ -535,8 +544,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, }() // 1. Check if there is a Tailscale Service associated with this Ingress. - pg := ing.Annotations[AnnotationProxyGroup] - cm, cfg, err := r.proxyGroupServeConfig(ctx, pg) + cm, cfg, err := r.proxyGroupServeConfig(ctx, pg.Name) if err != nil { return false, fmt.Errorf("error getting ProxyGroup serve config: %w", err) } @@ -550,13 +558,13 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 2. Clean up the Tailscale Service resources. - svcChanged, err = r.cleanupTailscaleService(ctx, svc, logger) + svcChanged, err = r.cleanupTailscaleService(ctx, svc, logger, tsClient) if err != nil { return false, fmt.Errorf("error deleting Tailscale Service: %w", err) } // 3. Clean up any cluster resources - if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil { + if err = cleanupCertResources(ctx, r.Client, r.tsNamespace, serviceName, pg); err != nil { return false, fmt.Errorf("failed to clean up cert resources: %w", err) } @@ -565,12 +573,12 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 4. Unadvertise the Tailscale Service in tailscaled config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, serviceAdvertisementOff, logger); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } // 5. Remove the Tailscale Service from the serve config for the ProxyGroup. - logger.Infof("Removing TailscaleService %q from serve config for ProxyGroup %q", hostname, pg) + logger.Infof("Removing TailscaleService %q from serve config for ProxyGroup %q", hostname, pg.Name) delete(cfg.Services, serviceName) cfgBytes, err := json.Marshal(cfg) if err != nil { @@ -628,24 +636,11 @@ func (r *HAIngressReconciler) proxyGroupServeConfig(ctx context.Context, pg stri return cm, cfg, nil } -type localClient interface { - StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) -} - -// tailnetCertDomain returns the base domain (TCD) of the current tailnet. -func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, error) { - st, err := r.lc.StatusWithoutPeers(ctx) - if err != nil { - return "", fmt.Errorf("error getting tailscale status: %w", err) - } - return st.CurrentTailnet.MagicDNSSuffix, nil -} - // shouldExpose returns true if the Ingress should be exposed over Tailscale in HA mode (on a ProxyGroup). func (r *HAIngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { isTSIngress := ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == tailscaleIngressClassName + *ing.Spec.IngressClassName == r.ingressClassName pgAnnot := ing.Annotations[AnnotationProxyGroup] return isTSIngress && pgAnnot != "" } @@ -666,7 +661,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki } // Validate TLS configuration - if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { + if len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS)) } @@ -683,16 +678,17 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki } // Validate ProxyGroup readiness - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name)) } // It is invalid to have multiple Ingress resources for the same Tailscale Service in one cluster. ingList := &networkingv1.IngressList{} if err := r.List(ctx, ingList); err != nil { - errs = append(errs, fmt.Errorf("[unexpected] error listing Ingresses: %w", err)) + errs = append(errs, fmt.Errorf("failed to list ingresses: %w", err)) return errors.Join(errs...) } + for _, i := range ingList.Items { if r.shouldExpose(&i) && hostnameForIngress(&i) == hostname && i.UID != ing.UID { errs = append(errs, fmt.Errorf("found duplicate Ingress %q for hostname %q - multiple Ingresses for the same hostname in the same cluster are not allowed", client.ObjectKeyFromObject(&i), hostname)) @@ -705,10 +701,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki // If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference. // If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing. // It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred. -func (r *HAIngressReconciler) cleanupTailscaleService(ctx context.Context, svc *tailscale.VIPService, logger *zap.SugaredLogger) (updated bool, _ error) { - if svc == nil { - return false, nil - } +func (r *HAIngressReconciler) cleanupTailscaleService(ctx context.Context, svc *tailscale.VIPService, logger *zap.SugaredLogger, tsClient tsclient.Client) (updated bool, _ error) { o, err := parseOwnerAnnotation(svc) if err != nil { return false, fmt.Errorf("error parsing Tailscale Service's owner annotation") @@ -728,16 +721,21 @@ func (r *HAIngressReconciler) cleanupTailscaleService(ctx context.Context, svc * } if len(o.OwnerRefs) == 1 { logger.Infof("Deleting Tailscale Service %q", svc.Name) - return false, r.tsClient.DeleteVIPService(ctx, svc.Name) + if err = tsClient.VIPServices().Delete(ctx, svc.Name); err != nil && !tailscale.IsNotFound(err) { + return false, err + } + + return false, nil } + o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1) - logger.Infof("Deleting Tailscale Service %q", svc.Name) + logger.Infof("Creating/Updating Tailscale Service %q", svc.Name) json, err := json.Marshal(o) if err != nil { return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err) } svc.Annotations[ownerAnnotation] = string(json) - return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc) + return true, tsClient.VIPServices().CreateOrUpdate(ctx, *svc) } // isHTTPEndpointEnabled returns true if the Ingress has been configured to expose an HTTP endpoint to tailnet. @@ -757,10 +755,10 @@ const ( serviceAdvertisementHTTPAndHTTPS // Both ports 80 and 443 should be advertised ) -func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) { +func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (err error) { // Get all config Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil { + if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { return fmt.Errorf("failed to list config Secrets: %w", err) } @@ -772,7 +770,7 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // The only exception is Ingresses with an HTTP endpoint enabled - if an // Ingress has an HTTP endpoint enabled, it will be advertised even if the // TLS cert is not yet provisioned. - hasCert, err := a.hasCerts(ctx, serviceName) + hasCert, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) } @@ -812,7 +810,7 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con } if updated { - if err := a.Update(ctx, &secret); err != nil { + if err := r.Update(ctx, &secret); err != nil { return fmt.Errorf("error updating ProxyGroup config Secret: %w", err) } } @@ -821,10 +819,10 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con return nil } -func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) { +func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName string) (int, error) { // Get all state Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil { + if err := cl.List(ctx, secrets, client.InNamespace(tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil { return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err) } @@ -837,7 +835,7 @@ func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName if !ok { continue } - if slices.Contains(prefs.AdvertiseServices, serviceName.String()) { + if slices.Contains(prefs.AdvertiseServices, serviceName) { count++ } } @@ -858,7 +856,14 @@ type ownerAnnotationValue struct { // Kubernetes operator instance. type OwnerRef struct { // OperatorID is the stable ID of the operator's Tailscale device. - OperatorID string `json:"operatorID,omitempty"` + OperatorID string `json:"operatorID,omitempty"` + Resource *Resource `json:"resource,omitempty"` // optional, used to identify the ProxyGroup that owns this Tailscale Service. +} + +type Resource struct { + Kind string `json:"kind,omitempty"` // "ProxyGroup" + Name string `json:"name,omitempty"` // Name of the ProxyGroup that owns this Tailscale Service. Informational only. + UID string `json:"uid,omitempty"` // UID of the ProxyGroup that owns this Tailscale Service. } // ownerAnnotations returns the updated annotations required to ensure this @@ -866,20 +871,22 @@ type OwnerRef struct { // nil, but does not contain an owner reference we return an error as this likely means // that the Service was created by somthing other than a Tailscale // Kubernetes operator. -func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { +func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string]string, error) { ref := OwnerRef{ - OperatorID: r.operatorID, + OperatorID: operatorID, } if svc == nil { c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} - json, err := json.Marshal(c) + data, err := json.Marshal(c) if err != nil { - return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err) + return nil, fmt.Errorf("failed to marshal Tailscale Service's owner annotation contents: %w", err) } + return map[string]string{ - ownerAnnotation: string(json), + ownerAnnotation: string(data), }, nil } + o, err := parseOwnerAnnotation(svc) if err != nil { return nil, err @@ -890,6 +897,9 @@ func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[s if slices.Contains(o.OwnerRefs, ref) { // up to date return svc.Annotations, nil } + if o.OwnerRefs[0].Resource != nil { + return nil, fmt.Errorf("Tailscale Service %s is owned by another resource: %#v; cannot be reused for an Ingress", svc.Name, o.OwnerRefs[0].Resource) + } o.OwnerRefs = append(o.OwnerRefs, ref) json, err := json.Marshal(o) if err != nil { @@ -897,15 +907,17 @@ func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[s } newAnnots := make(map[string]string, len(svc.Annotations)+1) - for k, v := range svc.Annotations { - newAnnots[k] = v - } + maps.Copy(newAnnots, svc.Annotations) newAnnots[ownerAnnotation] = string(json) return newAnnots, nil } // parseOwnerAnnotation returns nil if no valid owner found. func parseOwnerAnnotation(tsSvc *tailscale.VIPService) (*ownerAnnotationValue, error) { + if tsSvc == nil { + return nil, nil + } + if tsSvc.Annotations == nil || tsSvc.Annotations[ownerAnnotation] == "" { return nil, nil } @@ -916,9 +928,8 @@ func parseOwnerAnnotation(tsSvc *tailscale.VIPService) (*ownerAnnotationValue, e return o, nil } -func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool { - return a != nil && b != nil && - a.Annotations != nil && b.Annotations != nil && +func ownersAreSetAndEqual(a, b tailscale.VIPService) bool { + return a.Annotations != nil && b.Annotations != nil && a.Annotations[ownerAnnotation] != "" && b.Annotations[ownerAnnotation] != "" && strings.EqualFold(a.Annotations[ownerAnnotation], b.Annotations[ownerAnnotation]) @@ -948,7 +959,7 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi }); err != nil { return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) } - rolebinding := certSecretRoleBinding(pg.Name, r.tsNamespace, domain) + rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) { // Labels and subjects might have changed if the Ingress has been updated to use a // different ProxyGroup. @@ -962,19 +973,19 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi // cleanupCertResources ensures that the TLS Secret and associated RBAC // resources that allow proxies to read/write to the Secret are deleted. -func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error { - domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name)) +func cleanupCertResources(ctx context.Context, cl client.Client, tsNamespace string, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup) error { + domainName, err := dnsNameForService(ctx, cl, serviceName, pg, tsNamespace) if err != nil { - return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", name, err) + return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", serviceName, err) } - labels := certResourceLabels(pgName, domainName) - if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + labels := certResourceLabels(pg.Name, domainName) + if err := cl.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err) } - if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err) } - if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err) } return nil @@ -1017,17 +1028,17 @@ func certSecretRole(pgName, namespace, domain string) *rbacv1.Role { // certSecretRoleBinding creates a RoleBinding for Role that will allow proxies // to manage the TLS Secret for the given domain. Domain must be a valid // Kubernetes resource name. -func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding { +func certSecretRoleBinding(pg *tsapi.ProxyGroup, namespace, domain string) *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: domain, Namespace: namespace, - Labels: certResourceLabels(pgName, domain), + Labels: certResourceLabels(pg.Name, domain), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: pgName, + Name: pgServiceAccountName(pg), Namespace: namespace, }, }, @@ -1040,14 +1051,17 @@ func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding // certSecret creates a Secret that will store the TLS certificate and private // key for the given domain. Domain must be a valid Kubernetes resource name. -func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *corev1.Secret { +func certSecret(pgName, namespace, domain string, parent client.Object) *corev1.Secret { labels := certResourceLabels(pgName, domain) - labels[kubetypes.LabelSecretType] = "certs" + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts // Labels that let us identify the Ingress resource lets us reconcile // the Ingress when the TLS Secret is updated (for example, when TLS // certs have been provisioned). - labels[LabelParentName] = ing.Name - labels[LabelParentNamespace] = ing.Namespace + labels[LabelParentType] = strings.ToLower(parent.GetObjectKind().GroupVersionKind().Kind) + labels[LabelParentName] = parent.GetName() + if ns := parent.GetNamespace(); ns != "" { + labels[LabelParentNamespace] = ns + } return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -1070,29 +1084,19 @@ func certResourceLabels(pgName, domain string) map[string]string { return map[string]string{ kubetypes.LabelManaged: "true", labelProxyGroup: pgName, - labelDomain: domain, - } -} - -// dnsNameForService returns the DNS name for the given Tailscale Service's name. -func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) { - s := svc.WithoutPrefix() - tcd, err := r.tailnetCertDomain(ctx) - if err != nil { - return "", fmt.Errorf("error determining DNS name base: %w", err) + labelDomain: tsoperator.TruncateLabelValue(domain), } - return s + "." + tcd, nil } // hasCerts checks if the TLS Secret for the given service has non-zero cert and key data. -func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceName) (bool, error) { - domain, err := r.dnsNameForService(ctx, svc) +func hasCerts(ctx context.Context, cl client.Client, ns string, svc tailcfg.ServiceName, pg *tsapi.ProxyGroup) (bool, error) { + domain, err := dnsNameForService(ctx, cl, svc, pg, ns) if err != nil { return false, fmt.Errorf("failed to get DNS name for service: %w", err) } secret := &corev1.Secret{} - err = r.Get(ctx, client.ObjectKey{ - Namespace: r.tsNamespace, + err = cl.Get(ctx, client.ObjectKey{ + Namespace: ns, Name: domain, }, secret) if err != nil { @@ -1108,20 +1112,6 @@ func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceN return len(cert) > 0 && len(key) > 0, nil } -func isErrorFeatureFlagNotEnabled(err error) bool { - // messageFFNotEnabled is the error message returned by - // Tailscale control plane when a Tailscale Service API call is made for a - // tailnet that does not have the Tailscale Services feature flag enabled. - const messageFFNotEnabled = "feature unavailable for tailnet" - return err != nil && strings.Contains(err.Error(), messageFFNotEnabled) -} - -func isErrorTailscaleServiceNotFound(err error) bool { - var errResp tailscale.ErrResponse - ok := errors.As(err, &errResp) - return ok && errResp.Status == http.StatusNotFound -} - func tagViolations(obj client.Object) []string { var violations []string if obj == nil { @@ -1132,7 +1122,7 @@ func tagViolations(obj client.Object) []string { return nil } - for _, tag := range strings.Split(tags, ",") { + for tag := range strings.SplitSeq(tags, ",") { tag = strings.TrimSpace(tag) if err := tailcfg.CheckTag(tag); err != nil { violations = append(violations, fmt.Sprintf("invalid tag %q: %v", tag, err)) diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 05f4827927923..8312dc5f70520 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -12,8 +12,10 @@ import ( "maps" "reflect" "slices" + "strings" "testing" + "github.com/google/go-cmp/cmp" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -23,14 +25,14 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "tailscale.com/internal/client/tailscale" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" - "tailscale.com/ipn/ipnstate" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" - "tailscale.com/types/ptr" ) func TestIngressPGReconciler(t *testing.T) { @@ -47,7 +49,7 @@ func TestIngressPGReconciler(t *testing.T) { }, }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), DefaultBackend: &networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ Name: "test", @@ -65,7 +67,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify initial reconciliation expectReconciled(t, ingPGR, "default", "test-ingress") - populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net") + populateTLSSecret(t, fc, "test-pg", "my-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) @@ -73,8 +75,13 @@ func TestIngressPGReconciler(t *testing.T) { // Verify that Role and RoleBinding have been created for the first Ingress. // Do not verify the cert Secret as that was already verified implicitly above. + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg", + }, + } expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net")) + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net")) mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test" @@ -82,7 +89,7 @@ func TestIngressPGReconciler(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") // Verify Tailscale Service uses custom tags - tsSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc") + tsSvc, err := ft.VIPServices().Get(t.Context(), "svc:my-svc") if err != nil { t.Fatalf("getting Tailscale Service: %v", err) } @@ -109,7 +116,7 @@ func TestIngressPGReconciler(t *testing.T) { }, }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), DefaultBackend: &networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ Name: "test", @@ -127,7 +134,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify second Ingress reconciliation expectReconciled(t, ingPGR, "default", "my-other-ingress") - populateTLSSecret(context.Background(), fc, "test-pg", "my-other-svc.ts.net") + populateTLSSecret(t, fc, "test-pg", "my-other-svc.ts.net") expectReconciled(t, ingPGR, "default", "my-other-ingress") verifyServeConfig(t, fc, "svc:my-other-svc", false) verifyTailscaleService(t, ft, "svc:my-other-svc", []string{"tcp:443"}) @@ -135,7 +142,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify that Role and RoleBinding have been created for the second Ingress. // Do not verify the cert Secret as that was already verified implicitly above. expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net")) + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-other-svc.ts.net")) // Verify first Ingress is still working verifyServeConfig(t, fc, "svc:my-svc", false) @@ -144,14 +151,14 @@ func TestIngressPGReconciler(t *testing.T) { verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc", "svc:my-other-svc"}) // Delete second Ingress - if err := fc.Delete(context.Background(), ing2); err != nil { + if err := fc.Delete(t.Context(), ing2); err != nil { t.Fatalf("deleting second Ingress: %v", err) } expectReconciled(t, ingPGR, "default", "my-other-ingress") // Verify second Ingress cleanup cm := &corev1.ConfigMap{} - if err := fc.Get(context.Background(), types.NamespacedName{ + if err := fc.Get(t.Context(), types.NamespacedName{ Name: "test-pg-ingress-config", Namespace: "operator-ns", }, cm); err != nil { @@ -184,10 +191,15 @@ func TestIngressPGReconciler(t *testing.T) { }) expectReconciled(t, ingPGR, "default", "test-ingress") expectEqual(t, fc, certSecretRole("test-pg-second", "operator-ns", "my-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg-second", "operator-ns", "my-svc.ts.net")) + pg = &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg-second", + }, + } + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net")) // Delete the first Ingress and verify cleanup - if err := fc.Delete(context.Background(), ing); err != nil { + if err := fc.Delete(t.Context(), ing); err != nil { t.Fatalf("deleting Ingress: %v", err) } @@ -195,7 +207,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify the ConfigMap was cleaned up cm = &corev1.ConfigMap{} - if err := fc.Get(context.Background(), types.NamespacedName{ + if err := fc.Get(t.Context(), types.NamespacedName{ Name: "test-pg-second-ingress-config", Namespace: "operator-ns", }, cm); err != nil { @@ -216,6 +228,47 @@ func TestIngressPGReconciler(t *testing.T) { expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net") expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-svc.ts.net") expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-svc.ts.net") + + // Create a third ingress + ing3 := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-other-ingress", + Namespace: "default", + UID: types.UID("5678-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"my-other-svc.tailnetxyz.ts.net"}}, + }, + }, + } + + mustCreate(t, fc, ing3) + expectReconciled(t, ingPGR, ing3.Namespace, ing3.Name) + + // Delete the service from "control" + ft.vipServices = make(map[string]tailscale.VIPService) + + // Delete the ingress and confirm we don't get stuck due to the VIP service not existing. + if err = fc.Delete(t.Context(), ing3); err != nil { + t.Fatalf("deleting Ingress: %v", err) + } + + expectReconciled(t, ingPGR, ing3.Namespace, ing3.Name) + expectMissing[networkingv1.Ingress](t, fc, ing3.Namespace, ing3.Name) } func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { @@ -232,7 +285,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { }, }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), DefaultBackend: &networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ Name: "test", @@ -250,7 +303,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { // Verify initial reconciliation expectReconciled(t, ingPGR, "default", "test-ingress") - populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net") + populateTLSSecret(t, fc, "test-pg", "my-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) @@ -261,17 +314,17 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { ing.Spec.TLS[0].Hosts[0] = "updated-svc" }) expectReconciled(t, ingPGR, "default", "test-ingress") - populateTLSSecret(context.Background(), fc, "test-pg", "updated-svc.ts.net") + populateTLSSecret(t, fc, "test-pg", "updated-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:updated-svc", false) verifyTailscaleService(t, ft, "svc:updated-svc", []string{"tcp:443"}) verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:updated-svc"}) - _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc")) + _, err := ft.VIPServices().Get(context.Background(), "svc:my-svc") if err == nil { t.Fatalf("svc:my-svc not cleaned up") } - if !isErrorTailscaleServiceNotFound(err) { + if !tailscale.IsNotFound(err) { t.Fatalf("unexpected error: %v", err) } } @@ -287,7 +340,7 @@ func TestValidateIngress(t *testing.T) { }, }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), TLS: []networkingv1.IngressTLS{ {Hosts: []string{"test"}}, }, @@ -305,7 +358,7 @@ func TestValidateIngress(t *testing.T) { Status: tsapi.ProxyGroupStatus{ Conditions: []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, @@ -399,7 +452,7 @@ func TestValidateIngress(t *testing.T) { Status: tsapi.ProxyGroupStatus{ Conditions: []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionFalse, ObservedGeneration: 1, }, @@ -421,7 +474,7 @@ func TestValidateIngress(t *testing.T) { }, }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), TLS: []networkingv1.IngressTLS{ {Hosts: []string{"test"}}, }, @@ -438,7 +491,12 @@ func TestValidateIngress(t *testing.T) { WithObjects(tt.ing). WithLists(&networkingv1.IngressList{Items: tt.existingIngs}). Build() + r := &HAIngressReconciler{Client: fc} + if tt.ing.Spec.IngressClassName != nil { + r.ingressClassName = *tt.ing.Spec.IngressClassName + } + err := r.validateIngress(context.Background(), tt.ing, tt.pg) if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) { t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr) @@ -463,7 +521,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { }, }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), DefaultBackend: &networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ Name: "test", @@ -483,7 +541,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { // Verify initial reconciliation with HTTP enabled expectReconciled(t, ingPGR, "default", "test-ingress") - populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net") + populateTLSSecret(t, fc, "test-pg", "my-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:80", "tcp:443"}) verifyServeConfig(t, fc, "svc:my-svc", true) @@ -504,16 +562,18 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { } // Add the Tailscale Service to prefs to have the Ingress recognised as ready. - mustCreate(t, fc, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pg-0", - Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "state"), - }, - Data: map[string][]byte{ - "_current-profile": []byte("profile-foo"), - "profile-foo": []byte(`{"AdvertiseServices":["svc:my-svc"],"Config":{"NodeID":"node-foo"}}`), - }, + mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { + var p prefs + var err error + if err = json.Unmarshal(o.Data["test"], &p); err != nil { + t.Errorf("failed to unmarshal preferences: %v", err) + } + + p.AdvertiseServices = []string{"svc:my-svc"} + o.Data["test"], err = json.Marshal(p) + if err != nil { + t.Errorf("failed to marshal preferences: %v", err) + } }) // Reconcile and re-fetch Ingress. @@ -559,6 +619,240 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { } } +func TestIngressPGReconciler_HTTPRedirect(t *testing.T) { + ingPGR, fc, ft := setupIngressTest(t) + + // Create backend Service that the Ingress will route to + backendSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.0.0.1", + Ports: []corev1.ServicePort{ + { + Port: 8080, + }, + }, + }, + } + mustCreate(t, fc, backendSvc) + + // Create test Ingress with HTTP redirect enabled + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/http-redirect": "true", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"my-svc"}}, + }, + }, + } + if err := fc.Create(context.Background(), ing); err != nil { + t.Fatal(err) + } + + // Verify initial reconciliation with HTTP redirect enabled + expectReconciled(t, ingPGR, "default", "test-ingress") + populateTLSSecret(t, fc, "test-pg", "my-svc.ts.net") + expectReconciled(t, ingPGR, "default", "test-ingress") + + // Verify Tailscale Service includes both tcp:80 and tcp:443 + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:80", "tcp:443"}) + + // Verify Ingress status includes port 80 + ing = &networkingv1.Ingress{} + if err := fc.Get(context.Background(), types.NamespacedName{ + Name: "test-ingress", + Namespace: "default", + }, ing); err != nil { + t.Fatal(err) + } + + // Add the Tailscale Service to prefs to have the Ingress recognised as ready. + mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { + var p prefs + var err error + if err = json.Unmarshal(o.Data["test"], &p); err != nil { + t.Errorf("failed to unmarshal preferences: %v", err) + } + + p.AdvertiseServices = []string{"svc:my-svc"} + o.Data["test"], err = json.Marshal(p) + if err != nil { + t.Errorf("failed to marshal preferences: %v", err) + } + }) + + // Reconcile and re-fetch Ingress + expectReconciled(t, ingPGR, "default", "test-ingress") + if err := fc.Get(context.Background(), client.ObjectKeyFromObject(ing), ing); err != nil { + t.Fatal(err) + } + + wantStatus := []networkingv1.IngressPortStatus{ + {Port: 443, Protocol: "TCP"}, + {Port: 80, Protocol: "TCP"}, + } + if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) { + t.Errorf("incorrect status ports: got %v, want %v", + ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) + } + + // Remove HTTP redirect annotation + mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { + delete(ing.Annotations, "tailscale.com/http-redirect") + }) + + // Verify reconciliation after removing HTTP redirect + expectReconciled(t, ingPGR, "default", "test-ingress") + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) + + // Verify Ingress status no longer includes port 80 + ing = &networkingv1.Ingress{} + if err := fc.Get(context.Background(), types.NamespacedName{ + Name: "test-ingress", + Namespace: "default", + }, ing); err != nil { + t.Fatal(err) + } + + wantStatus = []networkingv1.IngressPortStatus{ + {Port: 443, Protocol: "TCP"}, + } + if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) { + t.Errorf("incorrect status ports after removing redirect: got %v, want %v", + ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) + } +} + +func TestIngressPGReconciler_HTTPEndpointAndRedirectConflict(t *testing.T) { + ingPGR, fc, ft := setupIngressTest(t) + + // Create backend Service that the Ingress will route to + backendSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.0.0.1", + Ports: []corev1.ServicePort{ + { + Port: 8080, + }, + }, + }, + } + mustCreate(t, fc, backendSvc) + + // Create test Ingress with both HTTP endpoint and HTTP redirect enabled + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/http-endpoint": "enabled", + "tailscale.com/http-redirect": "true", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"my-svc"}}, + }, + }, + } + if err := fc.Create(context.Background(), ing); err != nil { + t.Fatal(err) + } + + // Verify initial reconciliation - HTTP endpoint should take precedence + expectReconciled(t, ingPGR, "default", "test-ingress") + populateTLSSecret(t, fc, "test-pg", "my-svc.ts.net") + expectReconciled(t, ingPGR, "default", "test-ingress") + + // Verify Tailscale Service includes both tcp:80 and tcp:443 + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:80", "tcp:443"}) + + // Verify the serve config has HTTP endpoint handlers on port 80, NOT redirect handlers + cm := &corev1.ConfigMap{} + if err := fc.Get(context.Background(), types.NamespacedName{ + Name: "test-pg-ingress-config", + Namespace: "operator-ns", + }, cm); err != nil { + t.Fatalf("getting ConfigMap: %v", err) + } + + // Verify Ingress status includes port 80 + ing = &networkingv1.Ingress{} + if err := fc.Get(context.Background(), types.NamespacedName{ + Name: "test-ingress", + Namespace: "default", + }, ing); err != nil { + t.Fatal(err) + } + + // Add the Tailscale Service to prefs to have the Ingress recognised as ready. + mustUpdate(t, fc, "operator-ns", "test-pg-0", func(o *corev1.Secret) { + var p prefs + var err error + if err = json.Unmarshal(o.Data["test"], &p); err != nil { + t.Errorf("failed to unmarshal preferences: %v", err) + } + + p.AdvertiseServices = []string{"svc:my-svc"} + o.Data["test"], err = json.Marshal(p) + if err != nil { + t.Errorf("failed to marshal preferences: %v", err) + } + }) + + // Reconcile and re-fetch Ingress + expectReconciled(t, ingPGR, "default", "test-ingress") + if err := fc.Get(context.Background(), client.ObjectKeyFromObject(ing), ing); err != nil { + t.Fatal(err) + } + + wantStatus := []networkingv1.IngressPortStatus{ + {Port: 443, Protocol: "TCP"}, + {Port: 80, Protocol: "TCP"}, + } + if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) { + t.Errorf("incorrect status ports: got %v, want %v", + ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) + } +} + func TestIngressPGReconciler_MultiCluster(t *testing.T) { ingPGR, fc, ft := setupIngressTest(t) ingPGR.operatorID = "operator-1" @@ -575,7 +869,7 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) { }, }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), TLS: []networkingv1.IngressTLS{ {Hosts: []string{"my-svc"}}, }, @@ -584,20 +878,18 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) { mustCreate(t, fc, ing) // Simulate existing Tailscale Service from another cluster - existingVIPSvc := &tailscale.VIPService{ + existingVIPSvc := tailscale.VIPService{ Name: "svc:my-svc", Annotations: map[string]string{ ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`, }, } - ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{ - "svc:my-svc": existingVIPSvc, - } + ft.VIPServices().CreateOrUpdate(t.Context(), existingVIPSvc) // Verify reconciliation adds our operator reference expectReconciled(t, ingPGR, "default", "test-ingress") - tsSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc") + tsSvc, err := ft.VIPServices().Get(context.Background(), "svc:my-svc") if err != nil { t.Fatalf("getting Tailscale Service: %v", err) } @@ -624,7 +916,7 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) { } expectRequeue(t, ingPGR, "default", "test-ingress") - tsSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc") + tsSvc, err = ft.VIPServices().Get(context.Background(), "svc:my-svc") if err != nil { t.Fatalf("getting Tailscale Service after deletion: %v", err) } @@ -645,7 +937,64 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) { } } -func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error { +func TestOwnerAnnotations(t *testing.T) { + singleSelfOwner := map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, + } + + for name, tc := range map[string]struct { + svc *tailscale.VIPService + wantAnnotations map[string]string + wantErr string + }{ + "no_svc": { + svc: nil, + wantAnnotations: singleSelfOwner, + }, + "empty_svc": { + svc: &tailscale.VIPService{}, + wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator", + }, + "already_owner": { + svc: &tailscale.VIPService{ + Annotations: singleSelfOwner, + }, + wantAnnotations: singleSelfOwner, + }, + "add_owner": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`, + }, + }, + wantAnnotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`, + }, + }, + "owned_by_proxygroup": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"1234-UID"}}]}`, + }, + }, + wantErr: "owned by another resource", + }, + } { + t.Run(name, func(t *testing.T) { + got, err := ownerAnnotations("self-id", tc.svc) + if tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("ownerAnnotations() error = %v, wantErr %v", err, tc.wantErr) + } + if diff := cmp.Diff(tc.wantAnnotations, got); diff != "" { + t.Errorf("ownerAnnotations() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func populateTLSSecret(t *testing.T, c client.Client, pgName, domain string) { + t.Helper() + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: domain, @@ -654,7 +1003,7 @@ func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain stri kubetypes.LabelManaged: "true", labelProxyGroup: pgName, labelDomain: domain, - kubetypes.LabelSecretType: "certs", + kubetypes.LabelSecretType: kubetypes.LabelSecretTypeCerts, }, }, Type: corev1.SecretTypeTLS, @@ -664,15 +1013,17 @@ func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain stri }, } - _, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) { + _, err := createOrUpdate(t.Context(), c, "operator-ns", secret, func(s *corev1.Secret) { s.Data = secret.Data }) - return err + if err != nil { + t.Fatalf("failed to populate TLS secret: %v", err) + } } func verifyTailscaleService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) { t.Helper() - tsSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName)) + tsSvc, err := ft.VIPServices().Get(context.Background(), serviceName) if err != nil { t.Fatalf("getting Tailscale Service %q: %v", serviceName, err) } @@ -752,16 +1103,17 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expec ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels(pgName, "config"), + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): fmt.Appendf(nil, `{"Version":""%s}`, expected), }, }) } func createPGResources(t *testing.T, fc client.Client, pgName string) { t.Helper() + // Pre-create the ProxyGroup pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -791,16 +1143,40 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels(pgName, "config"), + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte("{}"), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte("{}"), }, } mustCreate(t, fc, pgCfgSecret) + + pr := prefs{} + pr.Config.UserProfile.LoginName = "test.ts.net" + pr.Config.NodeID = "test" + + p, err := json.Marshal(pr) + if err != nil { + t.Fatalf("marshaling prefs: %v", err) + } + + // Pre-create a state secret for the ProxyGroup + pgStateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pgName, 0), + Namespace: "operator-ns", + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState), + }, + Data: map[string][]byte{ + currentProfileKey: []byte("test"), + "test": p, + }, + } + mustCreate(t, fc, pgStateSecret) + pg.Status.Conditions = []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, @@ -826,29 +1202,23 @@ func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeT fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} - ft := &fakeTSClient{} + ft := &fakeTSClient{ + vipServices: make(map[string]tailscale.VIPService), + } zl, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } - lc := &fakeLocalClient{ - status: &ipnstate.Status{ - CurrentTailnet: &ipnstate.TailnetStatus{ - MagicDNSSuffix: "ts.net", - }, - }, - } - ingPGR := &HAIngressReconciler{ - Client: fc, - tsClient: ft, - defaultTags: []string{"tag:k8s"}, - tsNamespace: "operator-ns", - tsnetServer: fakeTsnetServer, - logger: zl.Sugar(), - recorder: record.NewFakeRecorder(10), - lc: lc, + Client: fc, + clients: tsclient.NewProvider(ft), + defaultTags: []string{"tag:k8s"}, + tsNamespace: "operator-ns", + tsnetServer: fakeTsnetServer, + logger: zl.Sugar(), + recorder: record.NewFakeRecorder(10), + ingressClassName: tsIngressClass.Name, } return ingPGR, fc, ft diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 6c50e10b2ba94..8ea6d2f47f711 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "net" "slices" "strings" "sync" @@ -22,6 +23,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/ipn" "tailscale.com/kube/kubetypes" "tailscale.com/types/opt" @@ -31,9 +33,9 @@ import ( ) const ( - tailscaleIngressClassName = "tailscale" // ingressClass.metadata.name for tailscale IngressClass resource tailscaleIngressControllerName = "tailscale.com/ts-ingress" // ingressClass.spec.controllerName for tailscale IngressClass resource ingressClassDefaultAnnotation = "ingressclass.kubernetes.io/is-default-class" // we do not support this https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class + indexIngressProxyClass = ".metadata.annotations.ingress-proxy-class" ) type IngressReconciler struct { @@ -50,6 +52,7 @@ type IngressReconciler struct { managedIngresses set.Slice[types.UID] defaultProxyClass string + ingressClassName string } var ( @@ -100,7 +103,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare return nil } - if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil { + if done, err := a.ssr.Cleanup(ctx, operatorTailnet, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil { return fmt.Errorf("failed to cleanup: %w", err) } else if !done { logger.Debugf("cleanup not done yet, waiting for next reconcile") @@ -130,7 +133,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare // This function adds a finalizer to ing, ensuring that we can handle orderly // deprovisioning later. func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error { - if err := validateIngressClass(ctx, a.Client); err != nil { + if err := validateIngressClass(ctx, a.Client, a.ingressClassName); err != nil { logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err) } if !slices.Contains(ing.Finalizers, FinalizerName) { @@ -202,6 +205,27 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga return nil } + if isHTTPRedirectEnabled(ing) { + logger.Infof("HTTP redirect enabled, setting up port 80 redirect handlers") + const magic80 = "${TS_CERT_DOMAIN}:80" + sc.TCP[80] = &ipn.TCPPortHandler{HTTP: true} + sc.Web[magic80] = &ipn.WebServerConfig{ + Handlers: map[string]*ipn.HTTPHandler{}, + } + if sc.AllowFunnel != nil && sc.AllowFunnel[magic443] { + sc.AllowFunnel[magic80] = true + } + web80 := sc.Web[magic80] + for mountPoint := range handlers { + // We send a 301 - Moved Permanently redirect from HTTP to HTTPS + redirectURL := "301:https://${HOST}${REQUEST_URI}" + logger.Debugf("Creating redirect handler: %s -> %s", mountPoint, redirectURL) + web80.Handlers[mountPoint] = &ipn.HTTPHandler{ + Redirect: redirectURL, + } + } + } + crl := childResourceLabels(ing.Name, ing.Namespace, "ingress") var tags []string if tstr, ok := ing.Annotations[AnnotationTags]; ok { @@ -210,6 +234,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga hostname := hostnameForIngress(ing) sts := &tailscaleSTSConfig{ + Replicas: 1, Hostname: hostname, ParentResourceName: ing.Name, ParentResourceUID: string(ing.UID), @@ -218,62 +243,68 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ChildResourceLabels: crl, ProxyClassName: proxyClass, proxyType: proxyTypeIngressResource, + LoginServer: a.ssr.loginServer, } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { sts.ForwardClusterTrafficViaL7IngressProxy = true } - if _, err := a.ssr.Provision(ctx, logger, sts); err != nil { + if _, err = a.ssr.Provision(ctx, logger, sts); err != nil { return fmt.Errorf("failed to provision: %w", err) } - dev, err := a.ssr.DeviceInfo(ctx, crl, logger) + devices, err := a.ssr.DeviceInfo(ctx, crl, logger) if err != nil { return fmt.Errorf("failed to retrieve Ingress HTTPS endpoint status: %w", err) } - if dev == nil || dev.ingressDNSName == "" { - logger.Debugf("no Ingress DNS name known yet, waiting for proxy Pod initialize and start serving Ingress") - // No hostname yet. Wait for the proxy pod to auth. - ing.Status.LoadBalancer.Ingress = nil - if err := a.Status().Update(ctx, ing); err != nil { - return fmt.Errorf("failed to update ingress status: %w", err) + + ing.Status.LoadBalancer.Ingress = nil + for _, dev := range devices { + if dev.ingressDNSName == "" { + continue } - return nil - } - logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName) - ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{ - { - Hostname: dev.ingressDNSName, - Ports: []networkingv1.IngressPortStatus{ - { - Protocol: "TCP", - Port: 443, - }, + logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName) + ports := []networkingv1.IngressPortStatus{ + { + Protocol: "TCP", + Port: 443, }, - }, + } + if isHTTPRedirectEnabled(ing) { + ports = append(ports, networkingv1.IngressPortStatus{ + Protocol: "TCP", + Port: 80, + }) + } + ing.Status.LoadBalancer.Ingress = append(ing.Status.LoadBalancer.Ingress, networkingv1.IngressLoadBalancerIngress{ + Hostname: dev.ingressDNSName, + Ports: ports, + }) } - if err := a.Status().Update(ctx, ing); err != nil { + + if err = a.Status().Update(ctx, ing); err != nil { return fmt.Errorf("failed to update ingress status: %w", err) } + return nil } func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { return ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == tailscaleIngressClassName && + *ing.Spec.IngressClassName == a.ingressClassName && ing.Annotations[AnnotationProxyGroup] == "" } // validateIngressClass attempts to validate that 'tailscale' IngressClass // included in Tailscale installation manifests exists and has not been modified // to attempt to enable features that we do not support. -func validateIngressClass(ctx context.Context, cl client.Client) error { +func validateIngressClass(ctx context.Context, cl client.Client, ingressClassName string) error { ic := &networkingv1.IngressClass{ ObjectMeta: metav1.ObjectMeta{ - Name: tailscaleIngressClassName, + Name: ingressClassName, }, } if err := cl.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) { @@ -334,7 +365,7 @@ func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl clien proto = "https+insecure://" } mak.Set(&handlers, path, &ipn.HTTPHandler{ - Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path, + Proxy: proto + net.JoinHostPort(svc.Spec.ClusterIP, fmt.Sprint(port)) + path, }) } addIngressBackend(ing.Spec.DefaultBackend, "/") @@ -361,6 +392,12 @@ func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl clien return handlers, nil } +// isHTTPRedirectEnabled returns true if HTTP redirect is enabled for the Ingress. +// The annotation is tailscale.com/http-redirect and it should be set to "true". +func isHTTPRedirectEnabled(ing *networkingv1.Ingress) bool { + return ing.Annotations != nil && opt.Bool(ing.Annotations[AnnotationHTTPRedirect]).EqualBool(true) +} + // hostnameForIngress returns the hostname for an Ingress resource. // If the Ingress has TLS configured with a host, it returns the first component of that host. // Otherwise, it returns a hostname derived from the Ingress name and namespace. diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index dbd6961d7d7ff..92065b43d2cf6 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,6 +7,7 @@ package main import ( "context" + "reflect" "testing" "go.uber.org/zap" @@ -14,32 +15,38 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" - "tailscale.com/types/ptr" "tailscale.com/util/mak" ) func TestTailscaleIngress(t *testing.T) { fc := fake.NewFakeClient(ingressClass()) - ft := &fakeTSClient{} + ft := &fakeTSClient{ + vipServices: make(map[string]tailscale.VIPService), + } fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} zl, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), tsnetServer: fakeTsnetServer, defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", @@ -56,22 +63,26 @@ func TestTailscaleIngress(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "ingress") opts := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", parentType: "ingress", hostname: "default-test", app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, + }, + }, } - serveConfig := &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, - Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, - } - opts.serveConfig = serveConfig expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress status gets updated with ingress proxy's MagicDNS name // once that becomes available. @@ -98,11 +109,11 @@ func TestTailscaleIngress(t *testing.T) { }) opts.shouldEnableForwardingClusterTrafficViaIngress = true expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 4. Resources get cleaned up when Ingress class is unset mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { - ing.Spec.IngressClassName = ptr.To("nginx") + ing.Spec.IngressClassName = new("nginx") }) expectReconciled(t, ingR, "default", "test") expectReconciled(t, ingR, "default", "test") // deleting Ingress STS requires two reconciles @@ -120,10 +131,11 @@ func TestTailscaleIngressHostname(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), tsnetServer: fakeTsnetServer, defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", @@ -153,16 +165,19 @@ func TestTailscaleIngressHostname(t *testing.T) { parentType: "ingress", hostname: "default-test", app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, + }, + }, } - serveConfig := &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, - Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, - } - opts.serveConfig = serveConfig expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint set mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { @@ -230,7 +245,18 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, - Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + Pod: &tsapi.Pod{ + Annotations: map[string]string{"foo.io/bar": "some-val"}, + TailscaleContainer: &tsapi.Container{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("28Mi"), + }, + }, + }, + }, + }}, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -244,10 +270,11 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), tsnetServer: fakeTsnetServer, defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", @@ -271,24 +298,27 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { parentType: "ingress", hostname: "default-test", app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, + }, + }, } - serveConfig := &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, - Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, - } - opts.serveConfig = serveConfig expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) // 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet // ready, so proxy resource configuration does not change. mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { - mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata") + mak.Set(&ing.ObjectMeta.Labels, LabelAnnotationProxyClass, "custom-metadata") }) expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) // 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get // reconciled and configuration from the ProxyClass is applied to the @@ -299,21 +329,22 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: pc.Generation, - }}} + }}, + } }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = pc.Name - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) // 4. tailscale.com/proxy-class label is removed from the Ingress, the // Ingress gets reconciled and the custom ProxyClass configuration is // removed from the proxy resources. mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { - delete(ing.ObjectMeta.Labels, LabelProxyClass) + delete(ing.ObjectMeta.Labels, LabelAnnotationProxyClass) }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = "" - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) } func TestTailscaleIngressWithServiceMonitor(t *testing.T) { @@ -325,14 +356,15 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: 1, - }}}, + }}, + }, } crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} // Create fake client with ProxyClass, IngressClass, Ingress with metrics ProxyClass, and Service ing := ingress() ing.Labels = map[string]string{ - LabelProxyClass: "metrics", + LabelAnnotationProxyClass: "metrics", } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -347,10 +379,11 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), tsnetServer: fakeTsnetServer, defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", @@ -360,10 +393,6 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { } expectReconciled(t, ingR, "default", "test") fullName, shortName := findGenName(t, fc, "default", "test", "ingress") - serveConfig := &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, - Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, - } opts := configOpts{ stsName: shortName, secretName: fullName, @@ -374,8 +403,15 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { app: kubetypes.AppIngressResource, namespaced: true, proxyType: proxyTypeIngressResource, - serveConfig: serveConfig, - resourceVersion: "1", + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, + }, + }, + resourceVersion: "1", } // 1. Enable metrics- expect metrics Service to be created @@ -421,6 +457,114 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { // ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here. } +func TestIngressProxyClassAnnotation(t *testing.T) { + cl := tstest.NewClock(tstest.ClockOpts{}) + zl := zap.Must(zap.NewDevelopment()) + + pcLEStaging, pcLEStagingFalse, _ := proxyClassesForLEStagingTest() + + testCases := []struct { + name string + proxyClassAnnotation string + proxyClassLabel string + proxyClassDefault string + expectedProxyClass string + expectEvents []string + }{ + { + name: "via_label", + proxyClassLabel: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_annotation", + proxyClassAnnotation: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_default", + proxyClassDefault: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_label_override_annotation", + proxyClassLabel: pcLEStaging.Name, + proxyClassAnnotation: pcLEStagingFalse.Name, + expectedProxyClass: pcLEStaging.Name, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + builder := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme) + + builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse). + WithStatusSubresource(pcLEStaging, pcLEStagingFalse) + + fc := builder.Build() + + if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" { + name := tt.proxyClassDefault + if name == "" { + name = tt.proxyClassLabel + if name == "" { + name = tt.proxyClassAnnotation + } + } + setProxyClassReady(t, fc, cl, name) + } + + mustCreate(t, fc, ingressClass()) + mustCreate(t, fc, service()) + ing := ingress() + if tt.proxyClassLabel != "" { + ing.Labels = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassLabel, + } + } + if tt.proxyClassAnnotation != "" { + ing.Annotations = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassAnnotation, + } + } + mustCreate(t, fc, ing) + + ingR := &IngressReconciler{ + Client: fc, + ingressClassName: "tailscale", + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(&fakeTSClient{}), + tsnetServer: &fakeTSNetServer{certDomains: []string{"test-host"}}, + defaultTags: []string{"tag:test"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale:test", + }, + logger: zl.Sugar(), + defaultProxyClass: tt.proxyClassDefault, + } + + expectReconciled(t, ingR, "default", "test") + + _, shortName := findGenName(t, fc, "default", "test", "ingress") + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + switch tt.expectedProxyClass { + case pcLEStaging.Name: + verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) + case pcLEStagingFalse.Name: + verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL") + default: + t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass) + } + }) + } +} + func TestIngressLetsEncryptStaging(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) zl := zap.Must(zap.NewDevelopment()) @@ -452,16 +596,17 @@ func TestIngressLetsEncryptStaging(t *testing.T) { ing := ingress() if tt.proxyClassPerResource != "" { ing.Labels = map[string]string{ - LabelProxyClass: tt.proxyClassPerResource, + LabelAnnotationProxyClass: tt.proxyClassPerResource, } } mustCreate(t, fc, ing) ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: &fakeTSClient{}, + clients: tsclient.NewProvider(&fakeTSClient{}), tsnetServer: &fakeTSNetServer{certDomains: []string{"test-host"}}, defaultTags: []string{"tag:test"}, operatorNamespace: "operator-ns", @@ -498,7 +643,7 @@ func TestEmptyPath(t *testing.T) { name: "empty_path_with_prefix_type", paths: []networkingv1.HTTPIngressPath{ { - PathType: ptrPathType(networkingv1.PathTypePrefix), + PathType: new(networkingv1.PathTypePrefix), Path: "", Backend: *backend(), }, @@ -511,7 +656,7 @@ func TestEmptyPath(t *testing.T) { name: "empty_path_with_implementation_specific_type", paths: []networkingv1.HTTPIngressPath{ { - PathType: ptrPathType(networkingv1.PathTypeImplementationSpecific), + PathType: new(networkingv1.PathTypeImplementationSpecific), Path: "", Backend: *backend(), }, @@ -524,7 +669,7 @@ func TestEmptyPath(t *testing.T) { name: "empty_path_with_exact_type", paths: []networkingv1.HTTPIngressPath{ { - PathType: ptrPathType(networkingv1.PathTypeExact), + PathType: new(networkingv1.PathTypeExact), Path: "", Backend: *backend(), }, @@ -538,12 +683,12 @@ func TestEmptyPath(t *testing.T) { name: "two_competing_but_not_identical_paths_including_one_empty", paths: []networkingv1.HTTPIngressPath{ { - PathType: ptrPathType(networkingv1.PathTypeImplementationSpecific), + PathType: new(networkingv1.PathTypeImplementationSpecific), Path: "", Backend: *backend(), }, { - PathType: ptrPathType(networkingv1.PathTypeImplementationSpecific), + PathType: new(networkingv1.PathTypeImplementationSpecific), Path: "/", Backend: *backend(), }, @@ -565,11 +710,12 @@ func TestEmptyPath(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - recorder: fr, - Client: fc, + recorder: fr, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), tsnetServer: fakeTsnetServer, defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", @@ -599,27 +745,25 @@ func TestEmptyPath(t *testing.T) { parentType: "ingress", hostname: "foo", app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, + }, + }, } - serveConfig := &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, - Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, - } - opts.serveConfig = serveConfig expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) expectEvents(t, fr, tt.expectedEvents) }) } } -// ptrPathType is a helper function to return a pointer to the pathtype string (required for TestEmptyPath) -func ptrPathType(p networkingv1.PathType) *networkingv1.PathType { - return &p -} - func ingressClass() *networkingv1.IngressClass { return &networkingv1.IngressClass{ ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, @@ -635,9 +779,11 @@ func service() *corev1.Service { }, Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.4", - Ports: []corev1.ServicePort{{ - Port: 8080, - Name: "http"}, + Ports: []corev1.ServicePort{ + { + Port: 8080, + Name: "http", + }, }, }, } @@ -649,10 +795,10 @@ func ingress() *networkingv1.Ingress { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), DefaultBackend: backend(), TLS: []networkingv1.IngressTLS{ {Hosts: []string{"default-test"}}, @@ -670,7 +816,7 @@ func ingressWithPaths(paths []networkingv1.HTTPIngressPath) *networkingv1.Ingres UID: types.UID("1234-UID"), }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), + IngressClassName: new("tailscale"), Rules: []networkingv1.IngressRule{ { Host: "foo.tailnetxyz.ts.net", @@ -698,3 +844,189 @@ func backend() *networkingv1.IngressBackend { }, } } + +func TestTailscaleIngressWithHTTPRedirect(t *testing.T) { + fc := fake.NewFakeClient(ingressClass()) + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + ingR := &IngressReconciler{ + Client: fc, + ingressClassName: "tailscale", + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(ft), + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // 1. Create Ingress with HTTP redirect annotation + ing := ingress() + mak.Set(&ing.Annotations, AnnotationHTTPRedirect, "true") + mustCreate(t, fc, ing) + mustCreate(t, fc, service()) + + expectReconciled(t, ingR, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + opts := configOpts{ + replicas: new(int32(1)), + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test", + app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://1.2.3.4:8080/"}, + }}, + "${TS_CERT_DOMAIN}:80": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Redirect: "301:https://${HOST}${REQUEST_URI}"}, + }}, + }, + }, + } + + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) + + // 2. Update device info to get status updated + mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { + mak.Set(&secret.Data, "device_id", []byte("1234")) + mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net")) + }) + expectReconciled(t, ingR, "default", "test") + + // Verify Ingress status includes both ports 80 and 443 + ing = &networkingv1.Ingress{} + if err := fc.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: "default"}, ing); err != nil { + t.Fatal(err) + } + wantPorts := []networkingv1.IngressPortStatus{ + {Port: 443, Protocol: "TCP"}, + {Port: 80, Protocol: "TCP"}, + } + if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantPorts) { + t.Errorf("incorrect status ports: got %v, want %v", ing.Status.LoadBalancer.Ingress[0].Ports, wantPorts) + } + + // 3. Remove HTTP redirect annotation + mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { + delete(ing.Annotations, AnnotationHTTPRedirect) + }) + expectReconciled(t, ingR, "default", "test") + + // 4. Verify Ingress status no longer includes port 80 + ing = &networkingv1.Ingress{} + if err := fc.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: "default"}, ing); err != nil { + t.Fatal(err) + } + wantPorts = []networkingv1.IngressPortStatus{ + {Port: 443, Protocol: "TCP"}, + } + if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantPorts) { + t.Errorf("incorrect status ports after removing redirect: got %v, want %v", ing.Status.LoadBalancer.Ingress[0].Ports, wantPorts) + } +} + +func TestTailscaleIngressIPv6(t *testing.T) { + fc := fake.NewFakeClient(ingressClass()) + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + // Create a Service with an IPv6 ClusterIP + ipv6Svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ipv6", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "fda9:e575:6e22:2::25", + Ports: []corev1.ServicePort{ + { + Port: 2283, + Name: "http", + }, + }, + }, + } + mustCreate(t, fc, ipv6Svc) + + // Create an Ingress that routes to the IPv6 service + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ipv6", + Namespace: "default", + UID: "1234-UID-IPV6", + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test-ipv6", + Port: networkingv1.ServiceBackendPort{ + Number: 2283, + }, + }, + }, + }, + } + mustCreate(t, fc, ing) + + ingR := &IngressReconciler{ + Client: fc, + ingressClassName: "tailscale", + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(&fakeTSClient{}), + tsnetServer: &fakeTSNetServer{certDomains: []string{"test-host"}}, + defaultTags: []string{"tag:test"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + expectReconciled(t, ingR, "default", "test-ipv6") + + // Verify the generated serveConfig has properly bracketed IPv6 address + fullName, _ := findGenName(t, fc, "default", "test-ipv6", "ingress") + opts := configOpts{ + replicas: new(int32(1)), + stsName: "tailscale-ipv6-ingress-test-ipv6", + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test-ipv6-ingress", + app: kubetypes.AppIngressResource, + serveConfig: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://[fda9:e575:6e22:2::25]:2283/"}, + }}, + }, + }, + } + // expectedSecret hardcodes the parent-resource label to "test", so fix it for our IPv6 test + secret := expectedSecret(t, fc, opts) + secret.Labels[LabelParentName] = "test-ipv6" + expectEqual(t, fc, secret) +} diff --git a/cmd/k8s-operator/logger.go b/cmd/k8s-operator/logger.go new file mode 100644 index 0000000000000..45018e37eaf30 --- /dev/null +++ b/cmd/k8s-operator/logger.go @@ -0,0 +1,26 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "io" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// wrapZapCore returns a zapcore.Core implementation that splits the core chain using zapcore.NewTee. This causes +// logs to be simultaneously written to both the original core and the provided io.Writer implementation. +func wrapZapCore(core zapcore.Core, writer io.Writer) zapcore.Core { + encoder := &kzap.KubeAwareEncoder{ + Encoder: zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + } + + // We use a tee logger here so that logs are written to stdout/stderr normally while at the same time being + // sent upstream. + return zapcore.NewTee(core, zapcore.NewCore(encoder, zapcore.AddSync(writer), zap.DebugLevel)) +} diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go index 0579e34661a11..4384f4cba40bd 100644 --- a/cmd/k8s-operator/metrics_resources.go +++ b/cmd/k8s-operator/metrics_resources.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "maps" "reflect" "go.uber.org/zap" @@ -18,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" ) @@ -226,13 +228,13 @@ func metricsResourceLabels(opts *metricsOpts) map[string]string { kubetypes.LabelManaged: "true", labelMetricsTarget: opts.proxyStsName, labelPromProxyType: opts.proxyType, - labelPromProxyParentName: opts.proxyLabels[LabelParentName], + labelPromProxyParentName: kube.TruncateLabelValue(opts.proxyLabels[LabelParentName]), } // Include namespace label for proxies created for a namespaced type. if isNamespacedProxyType(opts.proxyType) { - lbls[labelPromProxyParentNamespace] = opts.proxyLabels[LabelParentNamespace] + lbls[labelPromProxyParentNamespace] = kube.TruncateLabelValue(opts.proxyLabels[LabelParentNamespace]) } - lbls[labelPromJob] = promJobName(opts) + lbls[labelPromJob] = kube.TruncateLabelValue(promJobName(opts)) return lbls } @@ -249,11 +251,11 @@ func promJobName(opts *metricsOpts) string { func metricsSvcSelector(proxyLabels map[string]string, proxyType string) map[string]string { sel := map[string]string{ labelPromProxyType: proxyType, - labelPromProxyParentName: proxyLabels[LabelParentName], + labelPromProxyParentName: kube.TruncateLabelValue(proxyLabels[LabelParentName]), } // Include namespace label for proxies created for a namespaced type. if isNamespacedProxyType(proxyType) { - sel[labelPromProxyParentNamespace] = proxyLabels[LabelParentNamespace] + sel[labelPromProxyParentNamespace] = kube.TruncateLabelValue(proxyLabels[LabelParentNamespace]) } return sel } @@ -286,11 +288,7 @@ func isNamespacedProxyType(typ string) bool { func mergeMapKeys(a, b map[string]string) map[string]string { m := make(map[string]string, len(a)+len(b)) - for key, val := range b { - m[key] = val - } - for key, val := range a { - m[key] = val - } + maps.Copy(m, b) + maps.Copy(m, a) return m } diff --git a/cmd/k8s-operator/nameserver.go b/cmd/k8s-operator/nameserver.go index 20d66f7d0766a..f5565e5d30cee 100644 --- a/cmd/k8s-operator/nameserver.go +++ b/cmd/k8s-operator/nameserver.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,14 +7,13 @@ package main import ( "context" + _ "embed" "errors" "fmt" "slices" "strings" "sync" - _ "embed" - "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -27,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -45,10 +45,7 @@ const ( messageMultipleDNSConfigsPresent = "Multiple DNSConfig resources found in cluster. Please ensure no more than one is present." defaultNameserverImageRepo = "tailscale/k8s-nameserver" - // TODO (irbekrm): once we start publishing nameserver images for stable - // track, replace 'unstable' here with the version of this operator - // instance. - defaultNameserverImageTag = "unstable" + defaultNameserverImageTag = "stable" ) // NameserverReconciler knows how to create nameserver resources in cluster in @@ -131,7 +128,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ return setStatus(&dnsCfg, metav1.ConditionFalse, reasonNameserverCreationFailed, msg) } } - if err := a.maybeProvision(ctx, &dnsCfg, logger); err != nil { + if err = a.maybeProvision(ctx, &dnsCfg); err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil @@ -168,7 +165,7 @@ func nameserverResourceLabels(name, namespace string) map[string]string { return labels } -func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsapi.DNSConfig, logger *zap.SugaredLogger) error { +func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsapi.DNSConfig) error { labels := nameserverResourceLabels(tsDNSCfg.Name, a.tsNamespace) dCfg := &deployConfig{ ownerRefs: []metav1.OwnerReference{*metav1.NewControllerRef(tsDNSCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))}, @@ -176,6 +173,11 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa labels: labels, imageRepo: defaultNameserverImageRepo, imageTag: defaultNameserverImageTag, + replicas: 1, + } + + if tsDNSCfg.Spec.Nameserver.Replicas != nil { + dCfg.replicas = *tsDNSCfg.Spec.Nameserver.Replicas } if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Repo != "" { dCfg.imageRepo = tsDNSCfg.Spec.Nameserver.Image.Repo @@ -183,6 +185,15 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Tag != "" { dCfg.imageTag = tsDNSCfg.Spec.Nameserver.Image.Tag } + if tsDNSCfg.Spec.Nameserver.Service != nil { + dCfg.clusterIP = tsDNSCfg.Spec.Nameserver.Service.ClusterIP + } + if tsDNSCfg.Spec.Nameserver.Pod != nil { + dCfg.tolerations = tsDNSCfg.Spec.Nameserver.Pod.Tolerations + dCfg.affinity = tsDNSCfg.Spec.Nameserver.Pod.Affinity + dCfg.nodeSelector = tsDNSCfg.Spec.Nameserver.Pod.NodeSelector + } + for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} { if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil { return fmt.Errorf("error reconciling %s: %w", deployable.kind, err) @@ -208,11 +219,16 @@ type deployable struct { } type deployConfig struct { - imageRepo string - imageTag string - labels map[string]string - ownerRefs []metav1.OwnerReference - namespace string + replicas int32 + imageRepo string + imageTag string + labels map[string]string + ownerRefs []metav1.OwnerReference + namespace string + clusterIP string + tolerations []corev1.Toleration + affinity *corev1.Affinity + nodeSelector map[string]string } var ( @@ -232,10 +248,14 @@ var ( if err := yaml.Unmarshal(deployYaml, &d); err != nil { return fmt.Errorf("error unmarshalling Deployment yaml: %w", err) } + d.Spec.Replicas = new(cfg.replicas) d.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", cfg.imageRepo, cfg.imageTag) d.ObjectMeta.Namespace = cfg.namespace d.ObjectMeta.Labels = cfg.labels d.ObjectMeta.OwnerReferences = cfg.ownerRefs + d.Spec.Template.Spec.Tolerations = cfg.tolerations + d.Spec.Template.Spec.Affinity = cfg.affinity + d.Spec.Template.Spec.NodeSelector = cfg.nodeSelector updateF := func(oldD *appsv1.Deployment) { oldD.Spec = d.Spec } @@ -267,6 +287,7 @@ var ( svc.ObjectMeta.Labels = cfg.labels svc.ObjectMeta.OwnerReferences = cfg.ownerRefs svc.ObjectMeta.Namespace = cfg.namespace + svc.Spec.ClusterIP = cfg.clusterIP _, err := createOrUpdate[corev1.Service](ctx, kubeClient, cfg.namespace, svc, func(*corev1.Service) {}) return err }, diff --git a/cmd/k8s-operator/nameserver_test.go b/cmd/k8s-operator/nameserver_test.go index cec95b84ee719..3ec00d5ed8859 100644 --- a/cmd/k8s-operator/nameserver_test.go +++ b/cmd/k8s-operator/nameserver_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -19,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/yaml" + operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstest" @@ -26,102 +27,202 @@ import ( ) func TestNameserverReconciler(t *testing.T) { - dnsCfg := &tsapi.DNSConfig{ + dnsConfig := &tsapi.DNSConfig{ TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"}, ObjectMeta: metav1.ObjectMeta{ Name: "test", }, Spec: tsapi.DNSConfigSpec{ Nameserver: &tsapi.Nameserver{ + Replicas: new(int32(3)), Image: &tsapi.NameserverImage{ Repo: "test", Tag: "v0.0.1", }, + Service: &tsapi.NameserverService{ + ClusterIP: "5.4.3.2", + }, + Pod: &tsapi.NameserverPod{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + Tolerations: []corev1.Toleration{ + { + Key: "some-key", + Operator: corev1.TolerationOpEqual, + Value: "some-value", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "some-key", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"some-value"}, + }, + }, + }, + }, + }, + }, + }, + }, }, }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). - WithObjects(dnsCfg). - WithStatusSubresource(dnsCfg). + WithObjects(dnsConfig). + WithStatusSubresource(dnsConfig). Build() - zl, err := zap.NewDevelopment() + + logger, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } - cl := tstest.NewClock(tstest.ClockOpts{}) - nr := &NameserverReconciler{ + + clock := tstest.NewClock(tstest.ClockOpts{}) + reconciler := &NameserverReconciler{ Client: fc, - clock: cl, - logger: zl.Sugar(), - tsNamespace: "tailscale", - } - expectReconciled(t, nr, "", "test") - // Verify that nameserver Deployment has been created and has the expected fields. - wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: "tailscale"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}} - if err := yaml.Unmarshal(deployYaml, wantsDeploy); err != nil { - t.Fatalf("unmarshalling yaml: %v", err) + clock: clock, + logger: logger.Sugar(), + tsNamespace: tsNamespace, } - dnsCfgOwnerRef := metav1.NewControllerRef(dnsCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) - wantsDeploy.OwnerReferences = []metav1.OwnerReference{*dnsCfgOwnerRef} - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1" - wantsDeploy.Namespace = "tailscale" - labels := nameserverResourceLabels("test", "tailscale") - wantsDeploy.ObjectMeta.Labels = labels - expectEqual(t, fc, wantsDeploy) - - // Verify that DNSConfig advertizes the nameserver's Service IP address, - // has the ready status condition and tailscale finalizer. - mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { - svc.Spec.ClusterIP = "1.2.3.4" + expectReconciled(t, reconciler, "", "test") + + ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) + nameserverLabels := nameserverResourceLabels(dnsConfig.Name, tsNamespace) + + wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}} + t.Run("deployment-expected-fields", func(t *testing.T) { + if err = yaml.Unmarshal(deployYaml, wantsDeploy); err != nil { + t.Fatalf("unmarshalling yaml: %v", err) + } + wantsDeploy.OwnerReferences = []metav1.OwnerReference{*ownerReference} + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1" + wantsDeploy.Spec.Replicas = new(int32(3)) + wantsDeploy.Namespace = tsNamespace + wantsDeploy.ObjectMeta.Labels = nameserverLabels + wantsDeploy.Spec.Template.Spec.Tolerations = []corev1.Toleration{ + { + Key: "some-key", + Operator: corev1.TolerationOpEqual, + Value: "some-value", + Effect: corev1.TaintEffectNoSchedule, + }, + } + wantsDeploy.Spec.Template.Spec.Affinity = &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "some-key", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"some-value"}, + }, + }, + }, + }, + }, + }, + } + wantsDeploy.Spec.Template.Spec.NodeSelector = map[string]string{ + "foo": "bar", + } + + expectEqual(t, fc, wantsDeploy) }) - expectReconciled(t, nr, "", "test") - dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{ - IP: "1.2.3.4", - } - dnsCfg.Finalizers = []string{FinalizerName} - dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{ - Type: string(tsapi.NameserverReady), - Status: metav1.ConditionTrue, - Reason: reasonNameserverCreated, - Message: reasonNameserverCreated, - LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + + wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}} + t.Run("service-expected-fields", func(t *testing.T) { + if err = yaml.Unmarshal(svcYaml, wantsSvc); err != nil { + t.Fatalf("unmarshalling yaml: %v", err) + } + wantsSvc.Spec.ClusterIP = dnsConfig.Spec.Nameserver.Service.ClusterIP + wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference} + wantsSvc.Namespace = tsNamespace + wantsSvc.ObjectMeta.Labels = nameserverLabels + expectEqual(t, fc, wantsSvc) }) - expectEqual(t, fc, dnsCfg) - // // Verify that nameserver image gets updated to match DNSConfig spec. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { - dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" + t.Run("dns-config-status-set", func(t *testing.T) { + // Verify that DNSConfig advertizes the nameserver's Service IP address, + // has the ready status condition and tailscale finalizer. + mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { + svc.Spec.ClusterIP = "1.2.3.4" + }) + expectReconciled(t, reconciler, "", "test") + + dnsConfig.Finalizers = []string{FinalizerName} + dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{ + IP: "1.2.3.4", + } + dnsConfig.Status.Conditions = append(dnsConfig.Status.Conditions, metav1.Condition{ + Type: string(tsapi.NameserverReady), + Status: metav1.ConditionTrue, + Reason: reasonNameserverCreated, + Message: reasonNameserverCreated, + LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)}, + }) + + expectEqual(t, fc, dnsConfig) }) - expectReconciled(t, nr, "", "test") - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" - expectEqual(t, fc, wantsDeploy) - - // Verify that when another actor sets ConfigMap data, it does not get - // overwritten by nameserver reconciler. - dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}} - bs, err := json.Marshal(dnsRecords) - if err != nil { - t.Fatalf("error marshalling ConfigMap contents: %v", err) - } - mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { - mak.Set(&cm.Data, "records.json", string(bs)) + + t.Run("nameserver-image-updated", func(t *testing.T) { + // Verify that nameserver image gets updated to match DNSConfig spec. + mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" + }) + expectReconciled(t, reconciler, "", "test") + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" + expectEqual(t, fc, wantsDeploy) + }) + + t.Run("reconciler-preserves-custom-config", func(t *testing.T) { + // Verify that when another actor sets ConfigMap data, it does not get + // overwritten by nameserver reconciler. + dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}} + bs, err := json.Marshal(dnsRecords) + if err != nil { + t.Fatalf("error marshalling ConfigMap contents: %v", err) + } + + mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { + mak.Set(&cm.Data, "records.json", string(bs)) + }) + + expectReconciled(t, reconciler, "", "test") + + wantCm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dnsrecords", + Namespace: "tailscale", + Labels: nameserverLabels, + OwnerReferences: []metav1.OwnerReference{*ownerReference}, + }, + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + Data: map[string]string{"records.json": string(bs)}, + } + + expectEqual(t, fc, wantCm) }) - expectReconciled(t, nr, "", "test") - wantCm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", - Namespace: "tailscale", Labels: labels, OwnerReferences: []metav1.OwnerReference{*dnsCfgOwnerRef}}, - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - Data: map[string]string{"records.json": string(bs)}, - } - expectEqual(t, fc, wantCm) - // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, - // the nameserver image defaults to tailscale/k8s-nameserver:unstable. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { - dnsCfg.Spec.Nameserver.Image = nil + t.Run("uses-default-nameserver-image", func(t *testing.T) { + // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, + // the nameserver image defaults to tailscale/k8s-nameserver:unstable. + mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + dnsCfg.Spec.Nameserver.Image = nil + }) + expectReconciled(t, reconciler, "", "test") + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:stable" + expectEqual(t, fc, wantsDeploy) }) - expectReconciled(t, nr, "", "test") - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable" - expectEqual(t, fc, wantsDeploy) } diff --git a/cmd/k8s-operator/nodeport-service-ports.go b/cmd/k8s-operator/nodeport-service-ports.go new file mode 100644 index 0000000000000..f8d28860bf84e --- /dev/null +++ b/cmd/k8s-operator/nodeport-service-ports.go @@ -0,0 +1,203 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "context" + "fmt" + "math/rand/v2" + "regexp" + "sort" + "strconv" + "strings" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + k8soperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" +) + +const ( + tailscaledPortMax = 65535 + tailscaledPortMin = 1024 + testSvcName = "test-node-port-range" + + invalidSvcNodePort = 777777 +) + +// getServicesNodePortRange is a hacky function that attempts to determine Service NodePort range by +// creating a deliberately invalid Service with a NodePort that is too large and parsing the returned +// validation error. Returns nil if unable to determine port range. +// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport +func getServicesNodePortRange(ctx context.Context, c client.Client, tsNamespace string, logger *zap.SugaredLogger) *tsapi.PortRange { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSvcName, + Namespace: tsNamespace, + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + { + Name: testSvcName, + Port: 8080, + TargetPort: intstr.FromInt32(8080), + Protocol: corev1.ProtocolUDP, + NodePort: invalidSvcNodePort, + }, + }, + }, + } + + // NOTE(ChaosInTheCRD): ideally this would be a server side dry-run but could not get it working + err := c.Create(ctx, svc) + if err == nil { + return nil + } + + if validPorts := getServicesNodePortRangeFromErr(err.Error()); validPorts != "" { + pr, err := parseServicesNodePortRange(validPorts) + if err != nil { + logger.Debugf("failed to parse NodePort range set for Kubernetes Cluster: %w", err) + return nil + } + + return pr + } + + return nil +} + +func getServicesNodePortRangeFromErr(err string) string { + reg := regexp.MustCompile(`\d{1,5}-\d{1,5}`) + matches := reg.FindAllString(err, -1) + if len(matches) != 1 { + return "" + } + + return matches[0] +} + +// parseServicesNodePortRange converts the `ValidPorts` string field in the Kubernetes PortAllocator error and converts it to +// PortRange +func parseServicesNodePortRange(p string) (*tsapi.PortRange, error) { + parts := strings.Split(p, "-") + s, err := strconv.ParseUint(parts[0], 10, 16) + if err != nil { + return nil, fmt.Errorf("failed to parse string as uint16: %w", err) + } + + var e uint64 + switch len(parts) { + case 1: + e = uint64(s) + case 2: + e, err = strconv.ParseUint(parts[1], 10, 16) + if err != nil { + return nil, fmt.Errorf("failed to parse string as uint16: %w", err) + } + default: + return nil, fmt.Errorf("failed to parse port range %q", p) + } + + portRange := &tsapi.PortRange{Port: uint16(s), EndPort: uint16(e)} + if !portRange.IsValid() { + return nil, fmt.Errorf("port range %q is not valid", portRange.String()) + } + + return portRange, nil +} + +// validateNodePortRanges checks that the port range specified is valid. It also ensures that the specified ranges +// lie within the NodePort Service port range specified for the Kubernetes API Server. +func validateNodePortRanges(ctx context.Context, c client.Client, kubeRange *tsapi.PortRange, pc *tsapi.ProxyClass) error { + if pc.Spec.StaticEndpoints == nil { + return nil + } + + portRanges := pc.Spec.StaticEndpoints.NodePort.Ports + + if kubeRange != nil { + for _, pr := range portRanges { + if !kubeRange.Contains(pr.Port) || (pr.EndPort != 0 && !kubeRange.Contains(pr.EndPort)) { + return fmt.Errorf("range %q is not within Cluster configured range %q", pr.String(), kubeRange.String()) + } + } + } + + for _, r := range portRanges { + if !r.IsValid() { + return fmt.Errorf("port range %q is invalid", r.String()) + } + } + + // TODO(ChaosInTheCRD): if a ProxyClass that made another invalid (due to port range clash) is deleted, + // the invalid ProxyClass doesn't get reconciled on, and therefore will not go valid. We should fix this. + proxyClassRanges, err := getPortsForProxyClasses(ctx, c) + if err != nil { + return fmt.Errorf("failed to get port ranges for ProxyClasses: %w", err) + } + + for _, r := range portRanges { + for pcName, pcr := range proxyClassRanges { + if pcName == pc.Name { + continue + } + if pcr.ClashesWith(r) { + return fmt.Errorf("port ranges for ProxyClass %q clash with existing ProxyClass %q", pc.Name, pcName) + } + } + } + + if len(portRanges) == 1 { + return nil + } + + sort.Slice(portRanges, func(i, j int) bool { + return portRanges[i].Port < portRanges[j].Port + }) + + for i := 1; i < len(portRanges); i++ { + prev := portRanges[i-1] + curr := portRanges[i] + if curr.Port <= prev.Port || curr.Port <= prev.EndPort { + return fmt.Errorf("overlapping ranges: %q and %q", prev.String(), curr.String()) + } + } + + return nil +} + +// getPortsForProxyClasses gets the port ranges for all the other existing ProxyClasses +func getPortsForProxyClasses(ctx context.Context, c client.Client) (map[string]tsapi.PortRanges, error) { + pcs := new(tsapi.ProxyClassList) + + err := c.List(ctx, pcs) + if err != nil { + return nil, fmt.Errorf("failed to list ProxyClasses: %w", err) + } + + portRanges := make(map[string]tsapi.PortRanges) + for _, i := range pcs.Items { + if !k8soperator.ProxyClassIsReady(&i) { + continue + } + if se := i.Spec.StaticEndpoints; se != nil && se.NodePort != nil { + portRanges[i.Name] = se.NodePort.Ports + } + } + + return portRanges, nil +} + +func getRandomPort() uint16 { + return uint16(rand.IntN(tailscaledPortMax-tailscaledPortMin+1) + tailscaledPortMin) +} diff --git a/cmd/k8s-operator/nodeport-services-ports_test.go b/cmd/k8s-operator/nodeport-services-ports_test.go new file mode 100644 index 0000000000000..9c147f79aecbd --- /dev/null +++ b/cmd/k8s-operator/nodeport-services-ports_test.go @@ -0,0 +1,277 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" +) + +func TestGetServicesNodePortRangeFromErr(t *testing.T) { + tests := []struct { + name string + errStr string + want string + }{ + { + name: "valid_error_string", + errStr: "NodePort 777777 is not in the allowed range 30000-32767", + want: "30000-32767", + }, + { + name: "error_string_with_different_message", + errStr: "some other error without a port range", + want: "", + }, + { + name: "error_string_with_multiple_port_ranges", + errStr: "range 1000-2000 and another range 3000-4000", + want: "", + }, + { + name: "empty_error_string", + errStr: "", + want: "", + }, + { + name: "error_string_with_range_at_start", + errStr: "30000-32767 is the range", + want: "30000-32767", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getServicesNodePortRangeFromErr(tt.errStr); got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseServicesNodePortRange(t *testing.T) { + tests := []struct { + name string + p string + want *tsapi.PortRange + wantErr bool + }{ + { + name: "valid_range", + p: "30000-32767", + want: &tsapi.PortRange{Port: 30000, EndPort: 32767}, + wantErr: false, + }, + { + name: "single_port_range", + p: "30000", + want: &tsapi.PortRange{Port: 30000, EndPort: 30000}, + wantErr: false, + }, + { + name: "invalid_format_non_numeric_end", + p: "30000-abc", + want: nil, + wantErr: true, + }, + { + name: "invalid_format_non_numeric_start", + p: "abc-32767", + want: nil, + wantErr: true, + }, + { + name: "empty_string", + p: "", + want: nil, + wantErr: true, + }, + { + name: "too_many_parts", + p: "1-2-3", + want: nil, + wantErr: true, + }, + { + name: "port_too_large_start", + p: "65536-65537", + want: nil, + wantErr: true, + }, + { + name: "port_too_large_end", + p: "30000-65536", + want: nil, + wantErr: true, + }, + { + name: "inverted_range", + p: "32767-30000", + want: nil, + wantErr: true, // IsValid() will fail + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + portRange, err := parseServicesNodePortRange(tt.p) + if (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.wantErr { + return + } + + if portRange == nil { + t.Fatalf("got nil port range, expected %v", tt.want) + } + + if portRange.Port != tt.want.Port || portRange.EndPort != tt.want.EndPort { + t.Errorf("got = %v, want %v", portRange, tt.want) + } + }) + } +} + +func TestValidateNodePortRanges(t *testing.T) { + tests := []struct { + name string + portRanges []tsapi.PortRange + wantErr bool + }{ + { + name: "valid_ranges_with_unknown_kube_range", + portRanges: []tsapi.PortRange{ + {Port: 30003, EndPort: 30005}, + {Port: 30006, EndPort: 30007}, + }, + wantErr: false, + }, + { + name: "overlapping_ranges", + portRanges: []tsapi.PortRange{ + {Port: 30000, EndPort: 30010}, + {Port: 30005, EndPort: 30015}, + }, + wantErr: true, + }, + { + name: "adjacent_ranges_no_overlap", + portRanges: []tsapi.PortRange{ + {Port: 30010, EndPort: 30020}, + {Port: 30021, EndPort: 30022}, + }, + wantErr: false, + }, + { + name: "identical_ranges_are_overlapping", + portRanges: []tsapi.PortRange{ + {Port: 30005, EndPort: 30010}, + {Port: 30005, EndPort: 30010}, + }, + wantErr: true, + }, + { + name: "range_clashes_with_existing_proxyclass", + portRanges: []tsapi.PortRange{ + {Port: 31005, EndPort: 32070}, + }, + wantErr: true, + }, + } + + // as part of this test, we want to create an adjacent ProxyClass in order to ensure that if it clashes with the one created in this test + // that we get an error + cl := tstest.NewClock(tstest.ClockOpts{}) + opc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 31000}, {Port: 32000}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + + fc := fake.NewClientBuilder(). + WithObjects(opc). + WithStatusSubresource(opc). + WithScheme(tsapi.GlobalScheme). + Build() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: tt.portRanges, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + err := validateNodePortRanges(context.Background(), fc, &tsapi.PortRange{Port: 30000, EndPort: 32767}, pc) + if (err != nil) != tt.wantErr { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestGetRandomPort(t *testing.T) { + for range 100 { + port := getRandomPort() + if port < tailscaledPortMin || port > tailscaledPortMax { + t.Errorf("generated port %d which is out of range [%d, %d]", port, tailscaledPortMin, tailscaledPortMax) + } + } +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index a08dd4da8c52f..af44dd4f7405f 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -20,13 +20,18 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + klabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" @@ -39,18 +44,25 @@ import ( kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/client/tailscale/v2" + "tailscale.com/client/local" - "tailscale.com/client/tailscale" + "tailscale.com/envknob" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/store/kubestore" apiproxy "tailscale.com/k8s-operator/api-proxy" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" + "tailscale.com/k8s-operator/reconciler/tailnet" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tsnet" "tailscale.com/tstime" "tailscale.com/types/logger" + "tailscale.com/util/set" "tailscale.com/version" ) @@ -60,23 +72,32 @@ import ( // Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart. //go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests +// Generate the helm chart's CRDs (which are ignored from git). +//go:generate go run tailscale.com/cmd/k8s-operator/generate helmcrd + // Generate CRD API docs. //go:generate go run github.com/elastic/crd-ref-docs --renderer=markdown --source-path=../../k8s-operator/apis/ --config=../../k8s-operator/api-docs-config.yaml --output-path=../../k8s-operator/api.md -func main() { - // Required to use our client API. We're fine with the instability since the - // client lives in the same repo as this code. - tailscale.I_Acknowledge_This_API_Is_Unstable = true +const ( + indexServiceProxyClass = ".metadata.annotations.service-proxy-class" + indexServiceExposed = ".metadata.annotations.service-expose" + indexServiceType = ".metadata.annotations.service-type" +) +func main() { var ( tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") tslogging = defaultEnv("OPERATOR_LOGGING", "info") image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") + k8sProxyImage = defaultEnv("K8S_PROXY_IMAGE", "tailscale/k8s-proxy:latest") priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") tags = defaultEnv("PROXY_TAGS", "tag:k8s") tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") defaultProxyClass = defaultEnv("PROXY_DEFAULT_CLASS", "") isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) + loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") + ingressClassName = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale") + operatorSAName = defaultEnv("OPERATOR_SERVICE_ACCOUNT_NAME", "operator") ) var opts []kzap.Opts @@ -103,56 +124,84 @@ func main() { // The operator can run either as a plain operator or it can // additionally act as api-server proxy // https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy. - mode := apiproxy.ParseAPIProxyMode() - if mode == apiproxy.APIServerProxyModeDisabled { + mode := parseAPIProxyMode() + if mode == nil { hostinfo.SetApp(kubetypes.AppOperator) } else { - hostinfo.SetApp(kubetypes.AppAPIServerProxy) + hostinfo.SetApp(kubetypes.AppInProcessAPIServerProxy) } - s, tsc := initTSNet(zlog) + s, tsc := initTSNet(zlog, loginServer) defer s.Close() restConfig := config.GetConfigOrDie() - apiproxy.MaybeLaunchAPIServerProxy(zlog, restConfig, s, mode) - rOpts := reconcilerOpts{ + if mode != nil { + ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, *mode, true) + if err != nil { + zlog.Fatalf("error creating API server proxy: %v", err) + } + go func() { + if err := ap.Run(context.Background()); err != nil { + zlog.Fatalf("error running API server proxy: %v", err) + } + }() + } + + // Operator log uploads can be opted-out using the "TS_NO_LOGS_NO_SUPPORT" environment variable. + if !envknob.NoLogsNoSupport() { + zlog = zlog.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return wrapZapCore(core, s.LogtailWriter()) + })) + } + + runReconcilers(reconcilerOpts{ log: zlog, tsServer: s, tsClient: tsc, tailscaleNamespace: tsNamespace, + operatorSAName: operatorSAName, restConfig: restConfig, proxyImage: image, + k8sProxyImage: k8sProxyImage, proxyPriorityClassName: priorityClassName, proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer, proxyTags: tags, proxyFirewallMode: tsFirewallMode, defaultProxyClass: defaultProxyClass, - } - runReconcilers(rOpts) + loginServer: loginServer, + ingressClassName: ingressClassName, + }) } -// initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the -// CLIENT_ID_FILE and CLIENT_SECRET_FILE environment variables to authenticate -// with Tailscale. -func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { +// initTSNet initializes the tsnet.Server and logs in to Tailscale. If CLIENT_ID +// is set, it authenticates to the Tailscale API using the federated OIDC workload +// identity flow. Otherwise, it uses the CLIENT_ID_FILE and CLIENT_SECRET_FILE +// environment variables to authenticate with static credentials. +func initTSNet(zlog *zap.SugaredLogger, loginServer string) (*tsnet.Server, *tailscale.Client) { var ( - clientIDPath = defaultEnv("CLIENT_ID_FILE", "") - clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "") + clientID = defaultEnv("CLIENT_ID", "") // Used for workload identity federation. + clientIDPath = defaultEnv("CLIENT_ID_FILE", "") // Used for static client credentials. + clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "") // Used for static client credentials. hostname = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator") kubeSecret = defaultEnv("OPERATOR_SECRET", "") operatorTags = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator") ) + startlog := zlog.Named("startup") - if clientIDPath == "" || clientSecretPath == "" { - startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set") + if clientID == "" && (clientIDPath == "" || clientSecretPath == "") { + startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set") // TODO(tomhjp): error message can mention WIF once it's publicly available. } - tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath) + + tsc, err := newTSClient(zlog.Named("ts-api-client"), clientID, clientIDPath, clientSecretPath, loginServer) if err != nil { startlog.Fatalf("error creating Tailscale client: %v", err) } + s := &tsnet.Server{ - Hostname: hostname, - Logf: zlog.Named("tailscaled").Debugf, + Hostname: hostname, + Logf: zlog.Named("tailscaled").Debugf, + ControlURL: loginServer, } + if p := os.Getenv("TS_PORT"); p != "" { port, err := strconv.ParseUint(p, 10, 16) if err != nil { @@ -160,6 +209,7 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { } s.Port = uint16(port) } + if kubeSecret != "" { st, err := kubestore.New(logger.Discard, kubeSecret) if err != nil { @@ -167,6 +217,7 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { } s.Store = st } + if err := s.Start(); err != nil { startlog.Fatalf("starting tailscale server: %v", err) } @@ -192,27 +243,29 @@ waitOnline: if loginDone { break } - caps := tailscale.KeyCapabilities{ - Devices: tailscale.KeyDeviceCapabilities{ - Create: tailscale.KeyDeviceCreateCapabilities{ - Reusable: false, - Preauthorized: true, - Tags: strings.Split(operatorTags, ","), - }, - }, - } - authkey, _, err := tsc.CreateKey(ctx, caps) + + var caps tailscale.KeyCapabilities + caps.Devices.Create.Reusable = false + caps.Devices.Create.Preauthorized = true + caps.Devices.Create.Tags = strings.Split(operatorTags, ",") + + authKey, err := tsc.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{Capabilities: caps}) if err != nil { startlog.Fatalf("creating operator authkey: %v", err) } - if err := lc.Start(ctx, ipn.Options{ - AuthKey: authkey, - }); err != nil { + + opts := ipn.Options{ + AuthKey: authKey.Key, + } + + if err = lc.Start(ctx, opts); err != nil { startlog.Fatalf("starting tailscale: %v", err) } - if err := lc.StartLoginInteractive(ctx); err != nil { + + if err = lc.StartLoginInteractive(ctx); err != nil { startlog.Fatalf("starting login: %v", err) } + startlog.Debugf("requested login by authkey") loginDone = true case "NeedsMachineAuth": @@ -228,6 +281,23 @@ waitOnline: return s, tsc } +// predicate function for filtering to ensure we *don't* reconcile on tailscale managed Kubernetes Services +func serviceManagedResourceFilterPredicate() predicate.Predicate { + return predicate.NewPredicateFuncs(func(object client.Object) bool { + if svc, ok := object.(*corev1.Service); !ok { + return false + } else { + return !isManagedResource(svc) + } + }) +} + +type ( + ClientProvider interface { + For(tailnet string) (tsclient.Client, error) + } +) + // runReconcilers starts the controller-runtime manager and registers the // ServiceReconciler. It blocks forever. func runReconcilers(opts reconcilerOpts) { @@ -276,23 +346,53 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not create manager: %v", err) } + clients := tsclient.NewProvider(tsclient.Wrap(opts.tsClient)) + + tailnetOptions := tailnet.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + OperatorSAName: opts.operatorSAName, + Clock: tstime.DefaultClock{}, + Logger: opts.log, + Registry: clients, + } + + if err = tailnet.NewReconciler(tailnetOptions).Register(mgr); err != nil { + startlog.Fatalf("could not register tailnet reconciler: %v", err) + } + + proxyGroupPolicyOptions := proxygrouppolicy.ReconcilerOptions{ + Client: mgr.GetClient(), + } + + if err = proxygrouppolicy.NewReconciler(proxyGroupPolicyOptions).Register(mgr); err != nil { + startlog.Fatalf("could not register proxygrouppolicy reconciler: %v", err) + } + svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler) svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc")) // If a ProxyClass changes, enqueue all Services labeled with that // ProxyClass's name. - proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc(mgr.GetClient(), startlog)) + proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc( + mgr.GetClient(), + startlog, + opts.defaultProxyClass, + opts.proxyActAsDefaultLoadBalancer, + )) eventRecorder := mgr.GetEventRecorderFor("tailscale-operator") ssr := &tailscaleSTSReconciler{ Client: mgr.GetClient(), tsnetServer: opts.tsServer, - tsClient: opts.tsClient, + clients: clients, defaultTags: strings.Split(opts.proxyTags, ","), operatorNamespace: opts.tailscaleNamespace, proxyImage: opts.proxyImage, proxyPriorityClassName: opts.proxyPriorityClassName, tsFirewallMode: opts.proxyFirewallMode, + loginServer: opts.tsServer.ControlURL, } + err = builder. ControllerManagedBy(mgr). Named("service-reconciler"). @@ -313,12 +413,28 @@ func runReconcilers(opts reconcilerOpts) { if err != nil { startlog.Fatalf("could not create service reconciler: %v", err) } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceProxyClass, indexProxyClass); err != nil { + startlog.Fatalf("failed setting up ProxyClass indexer for Services: %v", err) + } + if opts.defaultProxyClass != "" { + // If a default ProxyClass is specified, we'll need to list all objects + // that could be affected. For L3 ingress, this is Services with the + // "tailscale.com/expose" annotation and LoadBalancer services (either + // with the loadBalancerClass "tailscale", or unset if we're the default). + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceExposed, indexExposed); err != nil { + startlog.Fatalf("failed setting up exposed indexer for Services: %v", err) + } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceType, indexType); err != nil { + startlog.Fatalf("failed setting up type indexer for Services: %v", err) + } + } + ingressChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("ingress")) // If a ProxyClassChanges, enqueue all Ingresses labeled with that // ProxyClass's name. proxyClassFilterForIngress := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForIngress(mgr.GetClient(), startlog)) // Enque Ingress if a managed Service or backend Service associated with a tailscale Ingress changes. - svcHandlerForIngress := handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngress(mgr.GetClient(), startlog)) + svcHandlerForIngress := handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngress(mgr.GetClient(), startlog, opts.ingressClassName)) err = builder. ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). @@ -333,10 +449,15 @@ func runReconcilers(opts reconcilerOpts) { Client: mgr.GetClient(), logger: opts.log.Named("ingress-reconciler"), defaultProxyClass: opts.defaultProxyClass, + ingressClassName: opts.ingressClassName, }) if err != nil { startlog.Fatalf("could not create ingress reconciler: %v", err) } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyClass, indexProxyClass); err != nil { + startlog.Fatalf("failed setting up ProxyClass indexer for Ingresses: %v", err) + } + lc, err := opts.tsServer.LocalClient() if err != nil { startlog.Fatalf("could not get local client: %v", err) @@ -350,19 +471,19 @@ func runReconcilers(opts reconcilerOpts) { ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). Named("ingress-pg-reconciler"). - Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))). + Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog, opts.ingressClassName))). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAIngressesFromSecret(mgr.GetClient(), startlog))). Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter). Complete(&HAIngressReconciler{ - recorder: eventRecorder, - tsClient: opts.tsClient, - tsnetServer: opts.tsServer, - defaultTags: strings.Split(opts.proxyTags, ","), - Client: mgr.GetClient(), - logger: opts.log.Named("ingress-pg-reconciler"), - lc: lc, - operatorID: id, - tsNamespace: opts.tailscaleNamespace, + recorder: eventRecorder, + clients: clients, + tsnetServer: opts.tsServer, + defaultTags: strings.Split(opts.proxyTags, ","), + Client: mgr.GetClient(), + logger: opts.log.Named("ingress-pg-reconciler"), + operatorID: id, + tsNamespace: opts.tailscaleNamespace, + ingressClassName: opts.ingressClassName, }) if err != nil { startlog.Fatalf("could not create ingress-pg-reconciler: %v", err) @@ -374,19 +495,17 @@ func runReconcilers(opts reconcilerOpts) { ingressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(ingressSvcFromEps(mgr.GetClient(), opts.log.Named("service-pg-reconciler"))) err = builder. ControllerManagedBy(mgr). - For(&corev1.Service{}). + For(&corev1.Service{}, builder.WithPredicates(serviceManagedResourceFilterPredicate())). Named("service-pg-reconciler"). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAServicesFromSecret(mgr.GetClient(), startlog))). Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter). Watches(&discoveryv1.EndpointSlice{}, ingressSvcFromEpsFilter). Complete(&HAServiceReconciler{ recorder: eventRecorder, - tsClient: opts.tsClient, - tsnetServer: opts.tsServer, + clients: clients, defaultTags: strings.Split(opts.proxyTags, ","), Client: mgr.GetClient(), logger: opts.log.Named("service-pg-reconciler"), - lc: lc, clock: tstime.DefaultClock{}, operatorID: id, tsNamespace: opts.tailscaleNamespace, @@ -519,16 +638,19 @@ func runReconcilers(opts reconcilerOpts) { // ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that // define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get // reconciled if the CRD is applied at a later point. + kPortRange := getServicesNodePortRange(context.Background(), mgr.GetClient(), opts.tailscaleNamespace, startlog) serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyClass{}). Named("proxyclass-reconciler"). Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter). Complete(&ProxyClassReconciler{ - Client: mgr.GetClient(), - recorder: eventRecorder, - logger: opts.log.Named("proxyclass-reconciler"), - clock: tstime.DefaultClock{}, + Client: mgr.GetClient(), + nodePortRange: kPortRange, + recorder: eventRecorder, + tsNamespace: opts.tailscaleNamespace, + logger: opts.log.Named("proxyclass-reconciler"), + clock: tstime.DefaultClock{}, }) if err != nil { startlog.Fatal("could not create proxyclass reconciler: %v", err) @@ -573,42 +695,77 @@ func runReconcilers(opts reconcilerOpts) { Watches(&rbacv1.Role{}, recorderFilter). Watches(&rbacv1.RoleBinding{}, recorderFilter). Complete(&RecorderReconciler{ + recorder: eventRecorder, + tsNamespace: opts.tailscaleNamespace, + Client: mgr.GetClient(), + log: opts.log.Named("recorder-reconciler"), + clock: tstime.DefaultClock{}, + clients: clients, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + }) + if err != nil { + startlog.Fatalf("could not create Recorder reconciler: %v", err) + } + + // kube-apiserver's Tailscale Service reconciler. + err = builder. + ControllerManagedBy(mgr). + For(&tsapi.ProxyGroup{}, builder.WithPredicates( + predicate.NewPredicateFuncs(func(obj client.Object) bool { + pg, ok := obj.(*tsapi.ProxyGroup) + return ok && pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer + }), + )). + Named("kube-apiserver-ts-service-reconciler"). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(kubeAPIServerPGsFromSecret(mgr.GetClient(), startlog))). + Complete(&KubeAPIServerTSServiceReconciler{ + Client: mgr.GetClient(), recorder: eventRecorder, + logger: opts.log.Named("kube-apiserver-ts-service-reconciler"), + clients: clients, tsNamespace: opts.tailscaleNamespace, - Client: mgr.GetClient(), - l: opts.log.Named("recorder-reconciler"), + defaultTags: strings.Split(opts.proxyTags, ","), + operatorID: id, clock: tstime.DefaultClock{}, - tsClient: opts.tsClient, }) if err != nil { - startlog.Fatalf("could not create Recorder reconciler: %v", err) + startlog.Fatalf("could not create Kubernetes API server Tailscale Service reconciler: %v", err) } // ProxyGroup reconciler. ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{}) proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) + nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog)) + saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). Named("proxygroup-reconciler"). + Watches(&corev1.Service{}, ownedByProxyGroupFilter). Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter). Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). - Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter). + Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup). Watches(&corev1.Secret{}, ownedByProxyGroupFilter). Watches(&rbacv1.Role{}, ownedByProxyGroupFilter). Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter). Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup). + Watches(&corev1.Node{}, nodeFilterForProxyGroup). Complete(&ProxyGroupReconciler{ recorder: eventRecorder, Client: mgr.GetClient(), - l: opts.log.Named("proxygroup-reconciler"), + log: opts.log.Named("proxygroup-reconciler"), clock: tstime.DefaultClock{}, - tsClient: opts.tsClient, + clients: clients, tsNamespace: opts.tailscaleNamespace, - proxyImage: opts.proxyImage, + tsProxyImage: opts.proxyImage, + k8sProxyImage: opts.k8sProxyImage, defaultTags: strings.Split(opts.proxyTags, ","), tsFirewallMode: opts.proxyFirewallMode, defaultProxyClass: opts.defaultProxyClass, + loginServer: opts.tsServer.ControlURL, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), }) if err != nil { startlog.Fatalf("could not create ProxyGroup reconciler: %v", err) @@ -623,10 +780,11 @@ func runReconcilers(opts reconcilerOpts) { type reconcilerOpts struct { log *zap.SugaredLogger tsServer *tsnet.Server - tsClient tsClient + tsClient *tailscale.Client tailscaleNamespace string // namespace in which operator resources will be deployed restConfig *rest.Config // config for connecting to the kube API server proxyImage string // : + k8sProxyImage string // : // proxyPriorityClassName isPriorityClass to be set for proxy Pods. This // is a legacy mechanism for cluster resource configuration options - // going forward use ProxyClass. @@ -658,6 +816,15 @@ type reconcilerOpts struct { // class for proxies that do not have a ProxyClass set. // this is defined by an operator env variable. defaultProxyClass string + // loginServer is the coordination server URL that should be used by managed resources. + loginServer string + // ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults + // to "tailscale" but can be customised. + ingressClassName string + // operatorSAName is the name of the ServiceAccount that the operator pod runs as. It is used as the target + // ServiceAccount when minting tokens via the Kubernetes TokenRequest API for Tailnets that authenticate using + // workload identity federation. + operatorSAName string } // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each @@ -777,23 +944,100 @@ func managedResourceHandlerForType(typ string) handler.MapFunc { } } +// indexProxyClass is used to select ProxyClass-backed objects which are +// locally indexed in the cache for efficient listing without requiring labels. +func indexProxyClass(o client.Object) []string { + if !hasProxyClassAnnotation(o) { + return nil + } + + return []string{o.GetAnnotations()[LabelAnnotationProxyClass]} +} + +func indexExposed(o client.Object) []string { + if o.GetAnnotations()[AnnotationExpose] != "true" { + return nil + } + + return []string{o.GetAnnotations()[AnnotationExpose]} +} + +func indexType(o client.Object) []string { + svc, ok := o.(*corev1.Service) + if !ok { + return nil + } + + return []string{string(svc.Spec.Type)} +} + // proxyClassHandlerForSvc returns a handler that, for a given ProxyClass, // returns a list of reconcile requests for all Services labeled with // tailscale.com/proxy-class: . -func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger, defaultProxyClass string, isDefaultLoadBalancer bool) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { svcList := new(corev1.ServiceList) labels := map[string]string{ - LabelProxyClass: o.GetName(), + LabelAnnotationProxyClass: o.GetName(), } + if err := cl.List(ctx, svcList, client.MatchingLabels(labels)); err != nil { logger.Debugf("error listing Services for ProxyClass: %v", err) return nil } + reqs := make([]reconcile.Request, 0) + seenSvcs := make(set.Set[string]) for _, svc := range svcList.Items { reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name)) + } + + if err := cl.List(ctx, svcList, client.MatchingFields{indexServiceProxyClass: o.GetName()}); err != nil { + logger.Debugf("error listing Services for ProxyClass: %v", err) + return nil + } + + for _, svc := range svcList.Items { + nsname := fmt.Sprintf("%s/%s", svc.Namespace, svc.Name) + if seenSvcs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(nsname) + } + + if o.GetName() == defaultProxyClass { + // For the default ProxyClass, we also need to reconcile all exposed + // Services that don't have an explicit ProxyClass set. + for _, matcher := range []client.ListOption{ + client.MatchingFields{indexServiceExposed: "true"}, + client.MatchingFields{indexServiceType: string(corev1.ServiceTypeLoadBalancer)}, + } { + if err := cl.List(ctx, svcList, matcher); err != nil { + logger.Debugf("error listing exposed Services for ProxyClass: %v", err) + return nil + } + + for _, svc := range svcList.Items { + if hasProxyClassAnnotation(&svc) { + continue + } + if !shouldExpose(&svc, isDefaultLoadBalancer) { + continue + } + nsname := fmt.Sprintf("%s/%s", svc.Namespace, svc.Name) + if seenSvcs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(nsname) + } + } } + return reqs } } @@ -805,16 +1049,36 @@ func proxyClassHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) ha return func(ctx context.Context, o client.Object) []reconcile.Request { ingList := new(networkingv1.IngressList) labels := map[string]string{ - LabelProxyClass: o.GetName(), + LabelAnnotationProxyClass: o.GetName(), } if err := cl.List(ctx, ingList, client.MatchingLabels(labels)); err != nil { logger.Debugf("error listing Ingresses for ProxyClass: %v", err) return nil } + reqs := make([]reconcile.Request, 0) + seenIngs := make(set.Set[string]) for _, ing := range ingList.Items { reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + seenIngs.Add(fmt.Sprintf("%s/%s", ing.Namespace, ing.Name)) + } + + ingAnnotationList := new(networkingv1.IngressList) + if err := cl.List(ctx, ingAnnotationList, client.MatchingFields{indexIngressProxyClass: o.GetName()}); err != nil { + logger.Debugf("error listing Ingreses for ProxyClass: %v", err) + return nil + } + + for _, ing := range ingAnnotationList.Items { + nsname := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) + if seenIngs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + seenIngs.Add(nsname) } + return reqs } } @@ -840,9 +1104,69 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger) } } +// nodeHandlerForProxyGroup returns a handler that, for a given Node, returns a +// list of reconcile requests for ProxyGroups that should be reconciled for the +// Node event. ProxyGroups need to be reconciled for Node events if they are +// configured to expose tailscaled static endpoints to tailnet using NodePort +// Services. +func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err) + return nil + } + + reqs := make([]reconcile.Request, 0) + for _, pg := range pgList.Items { + if pg.Spec.ProxyClass == "" && defaultProxyClass == "" { + continue + } + + pc := defaultProxyClass + if pc == "" { + pc = pg.Spec.ProxyClass + } + + proxyClass := &tsapi.ProxyClass{} + if err := cl.Get(ctx, types.NamespacedName{Name: pc}, proxyClass); err != nil { + if !apierrors.IsNotFound(err) { + logger.Debugf("error getting ProxyClass %q: %v", pg.Spec.ProxyClass, err) + } + return nil + } + + stat := proxyClass.Spec.StaticEndpoints + if stat == nil { + continue + } + + // If the selector is empty, all nodes match. + // TODO(ChaosInTheCRD): think about how this must be handled if we want to limit the number of nodes used + if len(stat.NodePort.Selector) == 0 { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + continue + } + + selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: stat.NodePort.Selector, + }) + if err != nil { + logger.Debugf("error converting `spec.staticEndpoints.nodePort.selector` to Selector: %v", err) + return nil + } + + if selector.Matches(klabels.Set(o.GetLabels())) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + } + return reqs + } +} + // proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass, -// returns a list of reconcile requests for all Connectors that have -// .spec.proxyClass set. +// returns a list of reconcile requests for all ProxyGroups that have +// .spec.proxyClass set to that ProxyClass. func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { pgList := new(tsapi.ProxyGroupList) @@ -861,13 +1185,44 @@ func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) } } +// serviceAccountHandlerForProxyGroup returns a handler that, for a given ServiceAccount, +// returns a list of reconcile requests for all ProxyGroups that use that ServiceAccount. +// For most ProxyGroups, this will be a dedicated ServiceAccount owned by a specific +// ProxyGroup. But for kube-apiserver ProxyGroups running in auth mode, they use a shared +// static ServiceAccount named "kube-apiserver-auth-proxy". +func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for ServiceAccount: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + saName := o.GetName() + for _, pg := range pgList.Items { + if saName == authAPIServerProxySAName && isAuthAPIServerProxy(&pg) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + expectedOwner := pgOwnerReference(&pg)[0] + saOwnerRefs := o.GetOwnerReferences() + for _, ref := range saOwnerRefs { + if apiequality.Semantic.DeepEqual(ref, expectedOwner) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + break + } + } + } + return reqs + } +} + // serviceHandlerForIngress returns a handler for Service events for ingress // reconciler that ensures that if the Service associated with an event is of // interest to the reconciler, the associated Ingress(es) gets be reconciled. // The Services of interest are backend Services for tailscale Ingress and // managed Services for an StatefulSet for a proxy configured for tailscale // Ingress -func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger, ingressClassName string) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { if isManagedByType(o, "ingress") { ingName := parentFromObjectLabels(o) @@ -880,8 +1235,8 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl } reqs := make([]reconcile.Request, 0) for _, ing := range ingList.Items { - if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { - return nil + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != ingressClassName { + continue } if hasProxyGroupAnnotation(&ing) { // We don't want to reconcile backend Services for Ingresses for ProxyGroups. @@ -1013,7 +1368,7 @@ func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc { if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" { return nil } - if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" { + if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState { return nil } pg, ok := o.GetLabels()[LabelParentName] @@ -1034,7 +1389,9 @@ func ingressSvcFromEps(cl client.Client, logger *zap.SugaredLogger) handler.MapF svc := &corev1.Service{} ns := o.GetNamespace() if err := cl.Get(ctx, types.NamespacedName{Name: svcName, Namespace: ns}, svc); err != nil { - logger.Errorf("failed to get service: %v", err) + if !apierrors.IsNotFound(err) { + logger.Debugf("failed to get service: %v", err) + } return nil } @@ -1103,7 +1460,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile. func isTLSSecret(secret *corev1.Secret) bool { return secret.Type == corev1.SecretTypeTLS && secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" && - secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "certs" && + secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeCerts && secret.ObjectMeta.Labels[labelDomain] != "" && secret.ObjectMeta.Labels[labelProxyGroup] != "" } @@ -1111,7 +1468,7 @@ func isTLSSecret(secret *corev1.Secret) bool { func isPGStateSecret(secret *corev1.Secret) bool { return secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" && secret.ObjectMeta.Labels[LabelParentType] == "proxygroup" && - secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "state" + secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeState } // HAIngressesFromSecret returns a handler that returns reconcile requests for @@ -1120,9 +1477,10 @@ func HAIngressesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler. return func(ctx context.Context, o client.Object) []reconcile.Request { secret, ok := o.(*corev1.Secret) if !ok { - logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret") + logger.Warn("Secret handler triggered for an object that is not a Secret") return nil } + if isTLSSecret(secret) { return []reconcile.Request{ { @@ -1159,15 +1517,16 @@ func HAIngressesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler. } } -// HAServiceFromSecret returns a handler that returns reconcile requests for +// HAServicesFromSecret returns a handler that returns reconcile requests for // all HA Services that should be reconciled in response to a Secret event. func HAServicesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { secret, ok := o.(*corev1.Secret) if !ok { - logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret") + logger.Warn("Secret handler triggered for an object that is not a Secret") return nil } + if !isPGStateSecret(secret) { return nil } @@ -1193,15 +1552,55 @@ func HAServicesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.M } } +// kubeAPIServerPGsFromSecret finds ProxyGroups of type "kube-apiserver" that +// need to be reconciled after a ProxyGroup-owned Secret is updated. +func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + secret, ok := o.(*corev1.Secret) + if !ok { + logger.Warn("Secret handler triggered for an object that is not a Secret") + return nil + } + + if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" || + secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" { + return nil + } + + var pg tsapi.ProxyGroup + if err := cl.Get(ctx, types.NamespacedName{Name: secret.ObjectMeta.Labels[LabelParentName]}, &pg); err != nil { + if !apierrors.IsNotFound(err) { + logger.Debugf("error getting ProxyGroup %s: %v", secret.ObjectMeta.Labels[LabelParentName], err) + } + return nil + } + + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + return nil + } + + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: secret.ObjectMeta.Labels[LabelParentNamespace], + Name: secret.ObjectMeta.Labels[LabelParentName], + }, + }, + } + + } +} + // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all // user-created ExternalName Services that should be exposed on this ProxyGroup. func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { pg, ok := o.(*tsapi.ProxyGroup) if !ok { - logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup") + logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup") return nil } + if pg.Spec.Type != tsapi.ProxyGroupTypeEgress { return nil } @@ -1229,9 +1628,10 @@ func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) return func(ctx context.Context, o client.Object) []reconcile.Request { pg, ok := o.(*tsapi.ProxyGroup) if !ok { - logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup") + logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup") return nil } + if pg.Spec.Type != tsapi.ProxyGroupTypeIngress { return nil } @@ -1259,9 +1659,10 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns return func(ctx context.Context, o client.Object) []reconcile.Request { svc, ok := o.(*corev1.Service) if !ok { - logger.Infof("[unexpected] Service handler triggered for an object that is not a Service") + logger.Warn("Service handler triggered for an object that is not a Service") return nil } + if !isEgressSvcForProxyGroup(svc) { return nil } @@ -1288,9 +1689,10 @@ func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) h return func(ctx context.Context, o client.Object) []reconcile.Request { eps, ok := o.(*discoveryv1.EndpointSlice) if !ok { - logger.Infof("[unexpected] EndpointSlice handler triggered for an object that is not a EndpointSlice") + logger.Warn("EndpointSlice handler triggered for an object that is not a EndpointSlice") return nil } + if eps.Labels[labelProxyGroup] == "" { return nil } @@ -1327,18 +1729,21 @@ func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) return func(ctx context.Context, o client.Object) []reconcile.Request { crd, ok := o.(*apiextensionsv1.CustomResourceDefinition) if !ok { - logger.Debugf("[unexpected] ServiceMonitor CRD handler received an object that is not a CustomResourceDefinition") + logger.Warn("ServiceMonitor CRD handler received an object that is not a CustomResourceDefinition") return nil } + if crd.Name != serviceMonitorCRD { - logger.Debugf("[unexpected] ServiceMonitor CRD handler received an unexpected CRD %q", crd.Name) + logger.Warnf("ServiceMonitor CRD handler received an unexpected CRD %q", crd.Name) return nil } + pcl := &tsapi.ProxyClassList{} if err := cl.List(ctx, pcl); err != nil { - logger.Debugf("[unexpected] error listing ProxyClasses: %v", err) + logger.Errorf("failed to list ProxyClass resources: %v", err) return nil } + reqs := make([]reconcile.Request, 0) for _, pc := range pcl.Items { if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable { @@ -1347,6 +1752,7 @@ func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) }) } } + return reqs } } @@ -1356,9 +1762,10 @@ func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc { return func(o any) (any, error) { crd, ok := o.(*apiextensionsv1.CustomResourceDefinition) if !ok { - log.Infof("[unexpected] CRD transformer called for a non-CRD type") + log.Warn("CRD transformer called for a non-CRD type") return crd, nil } + crd.Spec = apiextensionsv1.CustomResourceDefinitionSpec{} return crd, nil } @@ -1385,7 +1792,7 @@ func indexPGIngresses(o client.Object) []string { // serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service // associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation, // the associated Ingress gets reconciled. -func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger, ingressClassName string) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { ingList := networkingv1.IngressList{} if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil { @@ -1394,7 +1801,7 @@ func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) han } reqs := make([]reconcile.Request, 0) for _, ing := range ingList.Items { - if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != ingressClassName { continue } if !hasProxyGroupAnnotation(&ing) { @@ -1422,6 +1829,10 @@ func hasProxyGroupAnnotation(obj client.Object) bool { return obj.GetAnnotations()[AnnotationProxyGroup] != "" } +func hasProxyClassAnnotation(obj client.Object) bool { + return obj.GetAnnotations()[LabelAnnotationProxyClass] != "" +} + func id(ctx context.Context, lc *local.Client) (string, error) { st, err := lc.StatusWithoutPeers(ctx) if err != nil { diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 33bf23e844d9a..b775a36fb5d11 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,6 +7,7 @@ package main import ( "context" + "encoding/json" "fmt" "testing" "time" @@ -20,14 +21,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "tailscale.com/k8s-operator/apis/v1alpha1" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/net/dns/resolvconffile" "tailscale.com/tstest" "tailscale.com/tstime" - "tailscale.com/types/ptr" "tailscale.com/util/dnsname" "tailscale.com/util/mak" ) @@ -35,16 +39,13 @@ import ( func TestLoadBalancerClass(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -63,7 +64,7 @@ func TestLoadBalancerClass(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ AnnotationTailnetTargetFQDN: "invalid.example.com", }, @@ -71,7 +72,7 @@ func TestLoadBalancerClass(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, }) @@ -94,7 +95,7 @@ func TestLoadBalancerClass(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, Status: corev1.ServiceStatus{ Conditions: []metav1.Condition{{ @@ -119,6 +120,7 @@ func TestLoadBalancerClass(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") opts := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -130,7 +132,7 @@ func TestLoadBalancerClass(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) want.Annotations = nil want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"} @@ -169,6 +171,10 @@ func TestLoadBalancerClass(t *testing.T) { }, }, } + + // Perform an additional reconciliation loop here to ensure resources don't change through side effects. Mainly + // to prevent infinite reconciliation + expectReconciled(t, sr, "default", "test") expectEqual(t, fc, want) // Turn the service back into a ClusterIP service, which should make the @@ -199,7 +205,7 @@ func TestLoadBalancerClass(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", @@ -212,17 +218,14 @@ func TestLoadBalancerClass(t *testing.T) { func TestTailnetTargetFQDNAnnotation(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) tailnetTargetFQDN := "foo.bar.ts.net." clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -240,7 +243,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ AnnotationTailnetTargetFQDN: tailnetTargetFQDN, }, @@ -257,6 +260,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -268,7 +272,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -291,7 +295,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Change the tailscale-target-fqdn annotation which should update the // StatefulSet @@ -324,17 +328,14 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { func TestTailnetTargetIPAnnotation(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) tailnetTargetIP := "100.66.66.66" clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -352,7 +353,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ AnnotationTailnetTargetIP: tailnetTargetIP, }, @@ -369,6 +370,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -380,7 +382,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -403,7 +405,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Change the tailscale-target-ip annotation which should update the // StatefulSet @@ -421,12 +423,12 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { }) expectReconciled(t, sr, "default", "test") - // // synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet - // // didn't create any child resources since this is all faked, so the - // // deletion goes through immediately. + // synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet + // didn't create any child resources since this is all faked, so the + // deletion goes through immediately. expectReconciled(t, sr, "default", "test") expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) - // // The deletion triggers another reconcile, to finish the cleanup. + // The deletion triggers another reconcile, to finish the cleanup. expectReconciled(t, sr, "default", "test") expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) expectMissing[corev1.Service](t, fc, "operator-ns", shortName) @@ -436,16 +438,13 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -460,7 +459,7 @@ func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) { Name: "test", Namespace: "default", - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ AnnotationTailnetTargetIP: tailnetTargetIP, }, @@ -468,7 +467,7 @@ func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, }) @@ -488,7 +487,7 @@ func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, Status: corev1.ServiceStatus{ Conditions: []metav1.Condition{{ @@ -507,16 +506,13 @@ func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) { func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -531,7 +527,7 @@ func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) { Name: "test", Namespace: "default", - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ AnnotationTailnetTargetIP: tailnetTargetIP, }, @@ -539,7 +535,7 @@ func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, }) @@ -559,7 +555,7 @@ func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, Status: corev1.ServiceStatus{ Conditions: []metav1.Condition{{ @@ -578,16 +574,13 @@ func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) { func TestAnnotations(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -605,7 +598,7 @@ func TestAnnotations(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ "tailscale.com/expose": "true", }, @@ -620,6 +613,7 @@ func TestAnnotations(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -631,7 +625,7 @@ func TestAnnotations(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -671,7 +665,7 @@ func TestAnnotations(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", @@ -684,16 +678,13 @@ func TestAnnotations(t *testing.T) { func TestAnnotationIntoLB(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -711,7 +702,7 @@ func TestAnnotationIntoLB(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ "tailscale.com/expose": "true", }, @@ -726,6 +717,7 @@ func TestAnnotationIntoLB(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -737,7 +729,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, since it would have normally happened at @@ -776,12 +768,12 @@ func TestAnnotationIntoLB(t *testing.T) { mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { delete(s.ObjectMeta.Annotations, "tailscale.com/expose") s.Spec.Type = corev1.ServiceTypeLoadBalancer - s.Spec.LoadBalancerClass = ptr.To("tailscale") + s.Spec.LoadBalancerClass = new("tailscale") }) expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // ... but the service should have a LoadBalancer status. want = &corev1.Service{ @@ -789,12 +781,12 @@ func TestAnnotationIntoLB(t *testing.T) { Name: "test", Namespace: "default", Finalizers: []string{"tailscale.com/finalizer"}, - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, Status: corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{ @@ -816,16 +808,13 @@ func TestAnnotationIntoLB(t *testing.T) { func TestLBIntoAnnotation(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -843,12 +832,12 @@ func TestLBIntoAnnotation(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, }) @@ -856,6 +845,7 @@ func TestLBIntoAnnotation(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -867,7 +857,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -891,7 +881,7 @@ func TestLBIntoAnnotation(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, Status: corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{ @@ -927,7 +917,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -937,7 +927,7 @@ func TestLBIntoAnnotation(t *testing.T) { Annotations: map[string]string{ "tailscale.com/expose": "true", }, - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", @@ -953,16 +943,13 @@ func TestLBIntoAnnotation(t *testing.T) { func TestCustomHostname(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -980,7 +967,7 @@ func TestCustomHostname(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ "tailscale.com/expose": "true", "tailscale.com/hostname": "reindeer-flotilla", @@ -996,6 +983,7 @@ func TestCustomHostname(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -1007,7 +995,7 @@ func TestCustomHostname(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -1048,7 +1036,7 @@ func TestCustomHostname(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ "tailscale.com/hostname": "reindeer-flotilla", }, @@ -1064,16 +1052,13 @@ func TestCustomHostname(t *testing.T) { func TestCustomPriorityClassName(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -1092,7 +1077,7 @@ func TestCustomPriorityClassName(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ "tailscale.com/expose": "true", "tailscale.com/hostname": "tailscale-critical", @@ -1108,6 +1093,7 @@ func TestCustomPriorityClassName(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -1118,7 +1104,183 @@ func TestCustomPriorityClassName(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) +} + +func TestServiceProxyClassAnnotation(t *testing.T) { + cl := tstest.NewClock(tstest.ClockOpts{}) + zl := zap.Must(zap.NewDevelopment()) + + pcIfNotPresent := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "if-not-present", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &v1alpha1.Container{ + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + }, + }, + } + + pcAlways := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "always", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &v1alpha1.Container{ + ImagePullPolicy: corev1.PullAlways, + }, + }, + }, + }, + } + + builder := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme) + builder = builder.WithObjects(pcIfNotPresent, pcAlways). + WithStatusSubresource(pcIfNotPresent, pcAlways) + fc := builder.Build() + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + }, + } + + mustCreate(t, fc, svc) + + testCases := []struct { + name string + proxyClassAnnotation string + proxyClassLabel string + proxyClassDefault string + expectedProxyClass string + expectEvents []string + }{ + { + name: "via_label", + proxyClassLabel: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_annotation", + proxyClassAnnotation: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_default", + proxyClassDefault: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_label_override_annotation", + proxyClassLabel: pcIfNotPresent.Name, + proxyClassAnnotation: pcAlways.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ft := &fakeTSClient{} + + if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" { + name := tt.proxyClassDefault + if name == "" { + name = tt.proxyClassLabel + if name == "" { + name = tt.proxyClassAnnotation + } + } + setProxyClassReady(t, fc, cl, name) + } + + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(ft), + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + defaultProxyClass: tt.proxyClassDefault, + logger: zl.Sugar(), + clock: cl, + isDefaultLoadBalancer: true, + } + + if tt.proxyClassLabel != "" { + svc.Labels = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassLabel, + } + } + if tt.proxyClassAnnotation != "" { + svc.Annotations = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassAnnotation, + } + } + + mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) { + s.Labels = svc.Labels + s.Annotations = svc.Annotations + }) + + expectReconciled(t, sr, "default", "test") + + list := &corev1.ServiceList{} + fc.List(context.Background(), list, client.InNamespace("default")) + + for _, i := range list.Items { + t.Logf("found service %s", i.Name) + } + + slist := &corev1.SecretList{} + fc.List(context.Background(), slist, client.InNamespace("operator-ns")) + for _, i := range slist.Items { + labels, _ := json.Marshal(i.Labels) + t.Logf("found secret %q with labels %q ", i.Name, string(labels)) + } + + _, shortName := findGenName(t, fc, "default", "test", "svc") + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + switch tt.expectedProxyClass { + case pcIfNotPresent.Name: + for _, cont := range sts.Spec.Template.Spec.Containers { + if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullIfNotPresent { + t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcIfNotPresent.Name, pcIfNotPresent.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy) + } + } + case pcAlways.Name: + for _, cont := range sts.Spec.Template.Spec.Containers { + if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcAlways.Name, pcAlways.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy) + } + } + default: + t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass) + } + }) + } } func TestProxyClassForService(t *testing.T) { @@ -1132,7 +1294,9 @@ func TestProxyClassForService(t *testing.T) { StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, - Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}, + }, + }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -1140,16 +1304,13 @@ func TestProxyClassForService(t *testing.T) { WithStatusSubresource(pc). Build() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -1167,17 +1328,18 @@ func TestProxyClassForService(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, }) expectReconciled(t, sr, "default", "test") fullName, shortName := findGenName(t, fc, "default", "test", "svc") opts := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -1188,16 +1350,16 @@ func TestProxyClassForService(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. The Service gets updated with tailscale.com/proxy-class label // pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not // yet ready, so no changes are actually applied to the proxy resources. mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { - mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata") + mak.Set(&svc.Labels, LabelAnnotationProxyClass, "custom-metadata") }) expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) expectEqual(t, fc, expectedSecret(t, fc, opts)) // 3. ProxyClass is set to Ready, the Service gets reconciled by the @@ -1209,37 +1371,35 @@ func TestProxyClassForService(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: pc.Generation, - }}} + }}, + } }) opts.proxyClass = pc.Name expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t)) // 4. tailscale.com/proxy-class label is removed from the Service, the // configuration from the ProxyClass is removed from the cluster // resources. mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { - delete(svc.Labels, LabelProxyClass) + delete(svc.Labels, LabelAnnotationProxyClass) }) opts.proxyClass = "" expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func TestDefaultLoadBalancer(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -1258,7 +1418,7 @@ func TestDefaultLoadBalancer(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", @@ -1272,6 +1432,7 @@ func TestDefaultLoadBalancer(t *testing.T) { expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -1280,22 +1441,19 @@ func TestDefaultLoadBalancer(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func TestProxyFirewallMode(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -1315,7 +1473,7 @@ func TestProxyFirewallMode(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", @@ -1327,6 +1485,7 @@ func TestProxyFirewallMode(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -1336,29 +1495,33 @@ func TestProxyFirewallMode(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func Test_isMagicDNSName(t *testing.T) { tests := []struct { + name string in string want bool }{ { + name: "foo-tail4567-ts-net", in: "foo.tail4567.ts.net", want: true, }, { + name: "foo-tail4567-ts-net-trailing-dot", in: "foo.tail4567.ts.net.", want: true, }, { + name: "foo-tail4567", in: "foo.tail4567", want: false, }, } for _, tt := range tests { - t.Run(tt.in, func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { if got := isMagicDNSName(tt.in); got != tt.want { t.Errorf("isMagicDNSName(%q) = %v, want %v", tt.in, got, tt.want) } @@ -1366,13 +1529,71 @@ func Test_isMagicDNSName(t *testing.T) { } } -func Test_serviceHandlerForIngress(t *testing.T) { +func Test_HeadlessService(t *testing.T) { fc := fake.NewFakeClient() - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) + zl := zap.Must(zap.NewDevelopment()) + clock := tstest.NewClock(tstest.ClockOpts{}) + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + }, + logger: zl.Sugar(), + clock: clock, + recorder: record.NewFakeRecorder(100), + } + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + + UID: "1234-UID", + Annotations: map[string]string{ + AnnotationExpose: "true", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, + }, + }) + + expectReconciled(t, sr, "default", "test") + + t0 := conditionTime(clock) + + want := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + AnnotationExpose: "true", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, + }, + Status: corev1.ServiceStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyReady), + Status: metav1.ConditionFalse, + LastTransitionTime: t0, + Reason: reasonProxyInvalid, + Message: `unable to provision proxy resources: invalid Service: headless Services are not supported.`, + }}, + }, } + expectEqual(t, fc, want) +} + +func Test_serviceHandlerForIngress(t *testing.T) { + const tailscaleIngressClassName = "tailscale" + fc := fake.NewFakeClient() + zl := zap.Must(zap.NewDevelopment()) + // 1. An event on a headless Service for a tailscale Ingress results in // the Ingress being reconciled. mustCreate(t, fc, &networkingv1.Ingress{ @@ -1380,7 +1601,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { Name: "ing-1", Namespace: "ns-1", }, - Spec: networkingv1.IngressSpec{IngressClassName: ptr.To(tailscaleIngressClassName)}, + Spec: networkingv1.IngressSpec{IngressClassName: new(tailscaleIngressClassName)}, }) svc1 := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -1396,7 +1617,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, svc1) wantReqs := []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-1", Name: "ing-1"}}} - gotReqs := serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), svc1) + gotReqs := serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), svc1) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1412,7 +1633,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { DefaultBackend: &networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{Name: "def-backend"}, }, - IngressClassName: ptr.To(tailscaleIngressClassName), + IngressClassName: new(tailscaleIngressClassName), }, }) backendSvc := &corev1.Service{ @@ -1423,7 +1644,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, backendSvc) wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-2", Name: "ing-2"}}} - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), backendSvc) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1436,10 +1657,11 @@ func Test_serviceHandlerForIngress(t *testing.T) { Namespace: "ns-3", }, Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To(tailscaleIngressClassName), + IngressClassName: new(tailscaleIngressClassName), Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ - {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}}}, + {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}}, + }, }}}}, }, }) @@ -1451,7 +1673,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, backendSvc2) wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-3", Name: "ing-3"}}} - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), backendSvc2) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc2) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1466,7 +1688,8 @@ func Test_serviceHandlerForIngress(t *testing.T) { Spec: networkingv1.IngressSpec{ Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ - {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}}}, + {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}}, + }, }}}}, }, }) @@ -1477,7 +1700,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { }, } mustCreate(t, fc, nonTSBackend) - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), nonTSBackend) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), nonTSBackend) if len(gotReqs) > 0 { t.Errorf("unexpected reconcile request for a Service that does not belong to a Tailscale Ingress: %#+v\n", gotReqs) } @@ -1491,17 +1714,47 @@ func Test_serviceHandlerForIngress(t *testing.T) { }, } mustCreate(t, fc, someSvc) - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), someSvc) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), someSvc) if len(gotReqs) > 0 { t.Errorf("unexpected reconcile request for a Service that does not belong to any Ingress: %#+v\n", gotReqs) } } -func Test_clusterDomainFromResolverConf(t *testing.T) { - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) +func Test_serviceHandlerForIngress_multipleIngressClasses(t *testing.T) { + fc := fake.NewFakeClient() + zl := zap.Must(zap.NewDevelopment()) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: "default"}, + } + mustCreate(t, fc, svc) + + mustCreate(t, fc, &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{Name: "nginx-ing", Namespace: "default"}, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("nginx"), + DefaultBackend: &networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}, + }, + }) + + mustCreate(t, fc, &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{Name: "ts-ing", Namespace: "default"}, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}, + }, + }) + + got := serviceHandlerForIngress(fc, zl.Sugar(), "tailscale")(context.Background(), svc) + want := []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "default", Name: "ts-ing"}}} + + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } +} + +func Test_clusterDomainFromResolverConf(t *testing.T) { + zl := zap.Must(zap.NewDevelopment()) tests := []struct { name string conf *resolvconffile.Config @@ -1509,7 +1762,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { want string }{ { - name: "success- custom domain", + name: "success-custom-domain", conf: &resolvconffile.Config{ SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")}, }, @@ -1517,7 +1770,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { want: "department.org.io", }, { - name: "success- default domain", + name: "success-default-domain", conf: &resolvconffile.Config{ SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.cluster.local."), toFQDN(t, "svc.cluster.local."), toFQDN(t, "cluster.local.")}, }, @@ -1525,7 +1778,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { want: "cluster.local", }, { - name: "only two search domains found", + name: "only-two-search-domains", conf: &resolvconffile.Config{ SearchDomains: []dnsname.FQDN{toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")}, }, @@ -1533,7 +1786,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { want: "cluster.local", }, { - name: "first search domain does not match the expected structure", + name: "first-search-domain-mismatch", conf: &resolvconffile.Config{ SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.bar.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")}, }, @@ -1541,7 +1794,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { want: "cluster.local", }, { - name: "second search domain does not match the expected structure", + name: "second-search-domain-mismatch", conf: &resolvconffile.Config{ SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "foo.department.org.io"), toFQDN(t, "some.other.fqdn")}, }, @@ -1549,7 +1802,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { want: "cluster.local", }, { - name: "third search domain does not match the expected structure", + name: "third-search-domain-mismatch", conf: &resolvconffile.Config{ SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")}, }, @@ -1565,13 +1818,11 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { }) } } + func Test_authKeyRemoval(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) // 1. A new Service that should be exposed via Tailscale gets created, a Secret with a config that contains auth // key is generated. @@ -1580,7 +1831,7 @@ func Test_authKeyRemoval(t *testing.T) { Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -1593,12 +1844,12 @@ func Test_authKeyRemoval(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", - UID: types.UID("1234-UID"), + UID: "1234-UID", }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, }) @@ -1613,11 +1864,12 @@ func Test_authKeyRemoval(t *testing.T) { hostname: "default-test", clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, + replicas: new(int32(1)), } expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Apply update to the Secret that imitates the proxy setting device_id. s := expectedSecret(t, fc, opts) @@ -1635,10 +1887,7 @@ func Test_authKeyRemoval(t *testing.T) { func Test_externalNameService(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) // 1. A External name Service that should be exposed via Tailscale gets // created. @@ -1647,7 +1896,7 @@ func Test_externalNameService(t *testing.T) { Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -1665,7 +1914,7 @@ func Test_externalNameService(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ AnnotationExpose: "true", }, @@ -1680,6 +1929,7 @@ func Test_externalNameService(t *testing.T) { fullName, shortName := findGenName(t, fc, "default", "test", "svc") opts := configOpts{ + replicas: new(int32(1)), stsName: shortName, secretName: fullName, namespace: "default", @@ -1691,7 +1941,7 @@ func Test_externalNameService(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Change the ExternalName and verify that changes get propagated. mustUpdate(t, sr, "default", "test", func(s *corev1.Service) { @@ -1699,7 +1949,7 @@ func Test_externalNameService(t *testing.T) { }) expectReconciled(t, sr, "default", "test") opts.clusterTargetDNS = "bar.com" - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func Test_metricsResourceCreation(t *testing.T) { @@ -1711,19 +1961,20 @@ func Test_metricsResourceCreation(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: 1, - }}}, + }}, + }, } svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", UID: types.UID("1234-UID"), - Labels: map[string]string{LabelProxyClass: "metrics"}, + Labels: map[string]string{LabelAnnotationProxyClass: "metrics"}, }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, } crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} @@ -1733,16 +1984,13 @@ func Test_metricsResourceCreation(t *testing.T) { WithStatusSubresource(pc). Build() ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), operatorNamespace: "operator-ns", }, logger: zl.Sugar(), @@ -1807,16 +2055,13 @@ func TestIgnorePGService(t *testing.T) { _, _, fc, _, _ := setupServiceTest(t) ft := &fakeTSClient{} - zl, err := zap.NewDevelopment() - if err != nil { - t.Fatal(err) - } + zl := zap.Must(zap.NewDevelopment()) clock := tstest.NewClock(tstest.ClockOpts{}) sr := &ServiceReconciler{ Client: fc, ssr: &tailscaleSTSReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), defaultTags: []string{"tag:k8s"}, operatorNamespace: "operator-ns", proxyImage: "tailscale/tailscale", @@ -1834,7 +2079,7 @@ func TestIgnorePGService(t *testing.T) { // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. - UID: types.UID("1234-UID"), + UID: "1234-UID", Annotations: map[string]string{ "tailscale.com/proxygroup": "test-pg", }, diff --git a/cmd/k8s-operator/proxyclass.go b/cmd/k8s-operator/proxyclass.go index 5ec9897d0a8b7..1484cf44827f5 100644 --- a/cmd/k8s-operator/proxyclass.go +++ b/cmd/k8s-operator/proxyclass.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstime" @@ -44,22 +45,24 @@ const ( type ProxyClassReconciler struct { client.Client - recorder record.EventRecorder - logger *zap.SugaredLogger - clock tstime.Clock + recorder record.EventRecorder + logger *zap.SugaredLogger + clock tstime.Clock + tsNamespace string mu sync.Mutex // protects following // managedProxyClasses is a set of all ProxyClass resources that we're currently // managing. This is only used for metrics. managedProxyClasses set.Slice[types.UID] + // nodePortRange is the NodePort range set for the Kubernetes Cluster. This is used + // when validating port ranges configured by users for spec.StaticEndpoints + nodePortRange *tsapi.PortRange } -var ( - // gaugeProxyClassResources tracks the number of ProxyClass resources - // that we're currently managing. - gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources") -) +// gaugeProxyClassResources tracks the number of ProxyClass resources +// that we're currently managing. +var gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources") func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { logger := pcr.logger.With("ProxyClass", req.Name) @@ -96,7 +99,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re pcr.mu.Unlock() oldPCStatus := pc.Status.DeepCopy() - if errs := pcr.validate(ctx, pc); errs != nil { + if errs := pcr.validate(ctx, pc, logger); errs != nil { msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error()) pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg) tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger) @@ -112,7 +115,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re return reconcile.Result{}, nil } -func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) { +func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) (violations field.ErrorList) { if sts := pc.Spec.StatefulSet; sts != nil { if len(sts.Labels) > 0 { if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil { @@ -168,10 +171,11 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl } } } + if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable { found, err := hasServiceMonitorCRD(ctx, pcr.Client) if err != nil { - pcr.logger.Infof("[unexpected]: error retrieving %q CRD: %v", serviceMonitorCRD, err) + pcr.logger.Errorf("error retrieving %q CRD: %v", serviceMonitorCRD, err) // best effort validation - don't error out here } else if !found { msg := fmt.Sprintf("ProxyClass defines that a ServiceMonitor custom resource should be created, but %q CRD was not found", serviceMonitorCRD) @@ -183,6 +187,17 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl violations = append(violations, errs...) } } + + if stat := pc.Spec.StaticEndpoints; stat != nil { + if err := validateNodePortRanges(ctx, pcr.Client, pcr.nodePortRange, pc); err != nil { + var prs tsapi.PortRanges = stat.NodePort.Ports + violations = append(violations, field.TypeInvalid(field.NewPath("spec", "staticEndpoints", "nodePort", "ports"), prs.String(), err.Error())) + } + + if len(stat.NodePort.Selector) < 1 { + logger.Debug("no Selectors specified on `spec.staticEndpoints.nodePort.selectors` field") + } + } // We do not validate embedded fields (security context, resource // requirements etc) as we inherit upstream validation for those fields. // Invalid values would get rejected by upstream validations at apply diff --git a/cmd/k8s-operator/proxyclass_test.go b/cmd/k8s-operator/proxyclass_test.go index 48290eea782b5..171cfc5904cd3 100644 --- a/cmd/k8s-operator/proxyclass_test.go +++ b/cmd/k8s-operator/proxyclass_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -131,9 +131,11 @@ func TestProxyClass(t *testing.T) { proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}} }) - expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", + expectedEvents := []string{ + "Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_TS_CONFIGFILE_PATH env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", - "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."} + "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", + } expectReconciled(t, pcr, "", "test") expectEvents(t, fr, expectedEvents) @@ -176,6 +178,110 @@ func TestProxyClass(t *testing.T) { expectEqual(t, fc, pc) } +func TestValidateProxyClassStaticEndpoints(t *testing.T) { + for name, tc := range map[string]struct { + staticEndpointConfig *tsapi.StaticEndpointsConfig + valid bool + }{ + "no_static_endpoints": { + staticEndpointConfig: nil, + valid: true, + }, + "valid_specific_ports": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: true, + }, + "valid_port_ranges": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3002}, + {Port: 3005, EndPort: 3007}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: true, + }, + "overlapping_port_ranges": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 1000, EndPort: 2000}, + {Port: 1500, EndPort: 1800}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "clashing_port_and_range": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3005}, + {Port: 3001, EndPort: 3010}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "malformed_port_range": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001, EndPort: 3000}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "empty_selector": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{{Port: 3000}}, + Selector: map[string]string{}, + }, + }, + valid: true, + }, + } { + t.Run(name, func(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + Build() + zl, _ := zap.NewDevelopment() + pcr := &ProxyClassReconciler{ + logger: zl.Sugar(), + Client: fc, + } + + pc := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + StaticEndpoints: tc.staticEndpointConfig, + }, + } + + logger := pcr.logger.With("ProxyClass", pc) + err := pcr.validate(context.Background(), pc, logger) + valid := err == nil + if valid != tc.valid { + t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err) + } + }) + } +} + func TestValidateProxyClass(t *testing.T) { for name, tc := range map[string]struct { pc *tsapi.ProxyClass @@ -219,8 +325,12 @@ func TestValidateProxyClass(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - pcr := &ProxyClassReconciler{} - err := pcr.validate(context.Background(), tc.pc) + zl, _ := zap.NewDevelopment() + pcr := &ProxyClassReconciler{ + logger: zl.Sugar(), + } + logger := pcr.logger.With("ProxyClass", tc.pc) + err := pcr.validate(context.Background(), tc.pc, logger) valid := err == nil if valid != tc.valid { t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index e7c0590b0dbb9..c4da525ddee1a 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,17 +7,20 @@ package main import ( "context" - "crypto/sha256" "encoding/json" "errors" "fmt" - "net/http" + "net/netip" "slices" + "sort" "strings" "sync" + "time" + dockerref "github.com/distribution/reference" "go.uber.org/zap" xslices "golang.org/x/exp/slices" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -25,60 +28,84 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "tailscale.com/client/tailscale" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/egressservices" + "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstime" - "tailscale.com/types/ptr" + "tailscale.com/types/opt" "tailscale.com/util/clientmetric" "tailscale.com/util/mak" "tailscale.com/util/set" ) const ( - reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" - reasonProxyGroupReady = "ProxyGroupReady" - reasonProxyGroupCreating = "ProxyGroupCreating" - reasonProxyGroupInvalid = "ProxyGroupInvalid" + reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" + reasonProxyGroupReady = "ProxyGroupReady" + reasonProxyGroupAvailable = "ProxyGroupAvailable" + reasonProxyGroupCreating = "ProxyGroupCreating" + reasonProxyGroupInvalid = "ProxyGroupInvalid" + reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + staticEndpointsMaxAddrs = 2 + + // The minimum tailcfg.CapabilityVersion that deployed clients are expected + // to support to be compatible with the current ProxyGroup controller. + // If the controller needs to depend on newer client behaviour, it should + // maintain backwards compatible logic for older capability versions for 3 + // stable releases, as per documentation on supported version drift: + // https://tailscale.com/kb/1236/kubernetes-operator#supported-versions + // + // tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was + // first introduced. + pgMinCapabilityVersion = 106 ) var ( - gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) - gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) + gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeAPIServerProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupAPIServerCount) ) // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition. type ProxyGroupReconciler struct { client.Client - l *zap.SugaredLogger + log *zap.SugaredLogger recorder record.EventRecorder clock tstime.Clock - tsClient tsClient + clients ClientProvider // User-specified defaults from the helm installation. tsNamespace string - proxyImage string + tsProxyImage string + k8sProxyImage string defaultTags []string tsFirewallMode string defaultProxyClass string - - mu sync.Mutex // protects following - egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge - ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + loginServer string + + mu sync.Mutex // protects following + egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge + ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge + authKeyRateLimits map[string]*rate.Limiter // per-ProxyGroup rate limiters for auth key re-issuance. + authKeyReissuing map[string]bool } func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger { - return r.l.With("ProxyGroup", name) + return r.log.With("ProxyGroup", name) } func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { @@ -94,6 +121,18 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } else if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyGroup: %w", err) } + + tsClient, err := r.clients.For(pg.Spec.Tailnet) + if err != nil { + oldPGStatus := pg.Status.DeepCopy() + nrr := ¬ReadyReason{ + reason: reasonProxyGroupTailnetUnavailable, + message: fmt.Errorf("failed to get tailscale client and loginUrl: %w", err).Error(), + } + + return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, make(map[string][]netip.AddrPort))) + } + if markedForDeletion(pg) { logger.Debugf("ProxyGroup is being deleted, cleaning up resources") ix := xslices.Index(pg.Finalizers, FinalizerName) @@ -102,7 +141,11 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ return reconcile.Result{}, nil } - if done, err := r.maybeCleanup(ctx, pg); err != nil { + if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil { + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return reconcile.Result{}, nil + } return reconcile.Result{}, err } else if !done { logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...") @@ -117,17 +160,15 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } oldPGStatus := pg.Status.DeepCopy() - setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) - if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { - // An error encountered here should get returned by the Reconcile function. - if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { - err = errors.Join(err, updateErr) - } - } - return reconcile.Result{}, err - } + staticEndpoints, nrr, err := r.reconcilePG(ctx, tsClient, pg, logger) + return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)) +} +// reconcilePG handles all reconciliation of a ProxyGroup that is not marked +// for deletion. It is separated out from Reconcile to make a clear separation +// between reconciling the ProxyGroup, and posting the status of its created +// resources onto the ProxyGroup status field. +func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, tsClient tsclient.Client, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (map[string][]netip.AddrPort, *notReadyReason, error) { if !slices.Contains(pg.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, @@ -135,18 +176,11 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ // operation is underway. logger.Infof("ensuring ProxyGroup is set up") pg.Finalizers = append(pg.Finalizers, FinalizerName) - if err = r.Update(ctx, pg); err != nil { - err = fmt.Errorf("error adding finalizer: %w", err) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, reasonProxyGroupCreationFailed) + if err := r.Update(ctx, pg); err != nil { + return r.notReadyErrf(pg, logger, "error adding finalizer: %w", err) } } - if err = r.validate(pg); err != nil { - message := fmt.Sprintf("ProxyGroup is invalid: %s", err) - r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupInvalid, message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupInvalid, message) - } - proxyClassName := r.defaultProxyClass if pg.Spec.ProxyClass != "" { proxyClassName = pg.Spec.ProxyClass @@ -157,60 +191,33 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ proxyClass = new(tsapi.ProxyClass) err := r.Get(ctx, types.NamespacedName{Name: proxyClassName}, proxyClass) if apierrors.IsNotFound(err) { - err = nil - message := fmt.Sprintf("the ProxyGroup's ProxyClass %s does not (yet) exist", proxyClassName) - logger.Info(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q does not (yet) exist", proxyClassName) + logger.Info(msg) + return notReady(reasonProxyGroupCreating, msg) } if err != nil { - err = fmt.Errorf("error getting ProxyGroup's ProxyClass %s: %s", proxyClassName, err) - r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error()) + return r.notReadyErrf(pg, logger, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) } - validateProxyClassForPG(logger, pg, proxyClass) if !tsoperator.ProxyClassIsReady(proxyClass) { - message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName) - logger.Info(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) - } - } - - if err = r.maybeProvision(ctx, pg, proxyClass); err != nil { - reason := reasonProxyGroupCreationFailed - msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { - reason = reasonProxyGroupCreating - msg = fmt.Sprintf("optimistic lock error, retrying: %s", err) - err = nil + msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName) logger.Info(msg) - } else { - r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg) + return notReady(reasonProxyGroupCreating, msg) } - return setStatusReady(pg, metav1.ConditionFalse, reason, msg) } - desiredReplicas := int(pgReplicas(pg)) - if len(pg.Status.Devices) < desiredReplicas { - message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas) - logger.Debug(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + if err := r.validate(ctx, pg, proxyClass, logger); err != nil { + return notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err)) } - if len(pg.Status.Devices) > desiredReplicas { - message := fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas) - logger.Debug(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + staticEndpoints, nrr, err := r.maybeProvision(ctx, tsClient, pg, proxyClass) + if err != nil { + return nil, nrr, err } - logger.Info("ProxyGroup resources synced") - return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady) + return staticEndpoints, nrr, nil } -// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup. -func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) { - if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { - return - } +func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) error { // Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check // beig accessible on the replica Pod IP:9002. This address can also be modified by users, via // TS_LOCAL_ADDR_PORT env var. @@ -222,25 +229,105 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc // shouldn't need to set their own). // // TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later). - if hasLocalAddrPortSet(pc) { + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && hasLocalAddrPortSet(pc) { msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+ "This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+ "In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+ "Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name) logger.Warn(msg) } + + // image is the value of pc.Spec.StatefulSet.Pod.TailscaleContainer.Image or "" + // imagePath is a slash-delimited path ending with the image name, e.g. + // "tailscale/tailscale" or maybe "k8s-proxy" if hosted at example.com/k8s-proxy. + var image, imagePath string + if pc != nil && + pc.Spec.StatefulSet != nil && + pc.Spec.StatefulSet.Pod != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer.Image != "" { + image, err := dockerref.ParseNormalizedNamed(pc.Spec.StatefulSet.Pod.TailscaleContainer.Image) + if err != nil { + // Shouldn't be possible as the ProxyClass won't be marked ready + // without successfully parsing the image. + return fmt.Errorf("error parsing %q as a container image reference: %w", pc.Spec.StatefulSet.Pod.TailscaleContainer.Image, err) + } + imagePath = dockerref.Path(image) + } + + var errs []error + if isAuthAPIServerProxy(pg) { + // Validate that the static ServiceAccount already exists. + sa := &corev1.ServiceAccount{} + if err := r.Get(ctx, types.NamespacedName{Namespace: r.tsNamespace, Name: authAPIServerProxySAName}, sa); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error validating that ServiceAccount %q exists: %w", authAPIServerProxySAName, err) + } + + errs = append(errs, fmt.Errorf("the ServiceAccount %q used for the API server proxy in auth mode does not exist but "+ + "should have been created during operator installation; use apiServerProxyConfig.allowImpersonation=true "+ + "in the helm chart, or authproxy-rbac.yaml from the static manifests", authAPIServerProxySAName)) + } + } else { + // Validate that the ServiceAccount we create won't overwrite the static one. + // TODO(tomhjp): This doesn't cover other controllers that could create a + // ServiceAccount. Perhaps should have some guards to ensure that an update + // would never change the ownership of a resource we expect to already be owned. + if pgServiceAccountName(pg) == authAPIServerProxySAName { + errs = append(errs, fmt.Errorf("the name of the ProxyGroup %q conflicts with the static ServiceAccount used for the API server proxy in auth mode", pg.Name)) + } + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + if strings.HasSuffix(imagePath, "tailscale") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "k8s-proxy", pg.Spec.Type)) + } + + if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleInitContainer != nil { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies Tailscale init container config, but ProxyGroups of type %q do not use init containers", pc.Name, pg.Spec.Type)) + } + } else { + if strings.HasSuffix(imagePath, "k8s-proxy") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "tailscale", pg.Spec.Type)) + } + } + + return errors.Join(errs...) } -func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error { +func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tsclient.Client, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { logger := r.logger(pg.Name) r.mu.Lock() - r.ensureAddedToGaugeForProxyGroup(pg) + r.ensureStateAddedForProxyGroup(pg) r.mu.Unlock() - cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass) + svcToNodePorts := make(map[string]uint16) + var tailscaledPort *uint16 + if proxyClass != nil && proxyClass.Spec.StaticEndpoints != nil { + var err error + svcToNodePorts, tailscaledPort, err = r.ensureNodePortServiceCreated(ctx, pg, proxyClass) + if err != nil { + if _, ok := errors.AsType[*allocatePortsErr](err); ok { + reason := reasonProxyGroupCreationFailed + msg := fmt.Sprintf("error provisioning NodePort Services for static endpoints: %v", err) + r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) + return notReady(reason, msg) + } + return r.notReadyErrf(pg, logger, "error provisioning NodePort Services for static endpoints: %w", err) + } + } + + staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, tsClient, pg, proxyClass, svcToNodePorts) if err != nil { - return fmt.Errorf("error provisioning config Secrets: %w", err) + if _, ok := errors.AsType[*FindStaticEndpointErr](err); ok { + reason := reasonProxyGroupCreationFailed + msg := fmt.Sprintf("error provisioning config Secrets: %v", err) + r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) + return notReady(reason, msg) + } + return r.notReadyErrf(pg, logger, "error provisioning config Secrets: %w", err) } + // State secrets are precreated so we can use the ProxyGroup CR as their owner ref. stateSecrets := pgStateSecrets(pg, r.tsNamespace) for _, sec := range stateSecrets { @@ -249,17 +336,24 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning state Secrets: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning state Secrets: %w", err) } } - sa := pgServiceAccount(pg, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { - s.ObjectMeta.Labels = sa.ObjectMeta.Labels - s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations - s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences - }); err != nil { - return fmt.Errorf("error provisioning ServiceAccount: %w", err) + + // auth mode kube-apiserver ProxyGroups use a statically created + // ServiceAccount to keep ClusterRole creation permissions limited to the + // helm chart installer. + if !isAuthAPIServerProxy(pg) { + sa := pgServiceAccount(pg, r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { + s.ObjectMeta.Labels = sa.ObjectMeta.Labels + s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations + s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences + }); err != nil { + return r.notReadyErrf(pg, logger, "error provisioning ServiceAccount: %w", err) + } } + role := pgRole(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels @@ -267,8 +361,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences r.Rules = role.Rules }); err != nil { - return fmt.Errorf("error provisioning Role: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning Role: %w", err) } + roleBinding := pgRoleBinding(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels @@ -277,8 +372,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.RoleRef = roleBinding.RoleRef r.Subjects = roleBinding.Subjects }); err != nil { - return fmt.Errorf("error provisioning RoleBinding: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning RoleBinding: %w", err) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { cm, hp := pgEgressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { @@ -286,61 +382,42 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp) }); err != nil { - return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, logger, "error provisioning egress ConfigMap %q: %w", cm.Name, err) } } + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { cm := pgIngressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { existing.ObjectMeta.Labels = cm.ObjectMeta.Labels existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } - ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, proxyClass) + + defaultImage := r.tsProxyImage + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + defaultImage = r.k8sProxyImage + } + ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { - return fmt.Errorf("error generating StatefulSet spec: %w", err) + return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err) } cfg := &tailscaleSTSConfig{ proxyType: string(pg.Spec.Type), } ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger) - capver, err := r.capVerForPG(ctx, pg, logger) - if err != nil { - return fmt.Errorf("error getting device info: %w", err) - } - - updateSS := func(s *appsv1.StatefulSet) { - - // This is a temporary workaround to ensure that egress ProxyGroup proxies with capver older than 110 - // are restarted when tailscaled configfile contents have changed. - // This workaround ensures that: - // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. - // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. - // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid - // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a - // restarting Pod and the hash is re-added again. - // Note that this workaround is only applied to egress ProxyGroups, because ingress ProxyGroup was added after capver 110. - // Note also that the hash annotation is only set on updates, not creation, because if the StatefulSet is - // being created, there is no need for a restart. - // TODO(irbekrm): remove this in 1.84. - hash := cfgHash - if capver >= 110 { - hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] - } - s.Spec = ss.Spec - if hash != "" && pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) - } + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { + s.Spec = ss.Spec s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences + }); err != nil { + return r.notReadyErrf(pg, logger, "error provisioning StatefulSet: %w", err) } - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil { - return fmt.Errorf("error provisioning StatefulSet: %w", err) - } + mo := &metricsOpts{ tsNamespace: r.tsNamespace, proxyStsName: pg.Name, @@ -348,72 +425,273 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro proxyType: "proxygroup", } if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil { - return fmt.Errorf("error reconciling metrics resources: %w", err) + return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err) } - if err := r.cleanupDanglingResources(ctx, pg); err != nil { - return fmt.Errorf("error cleaning up dangling resources: %w", err) + if err := r.cleanupDanglingResources(ctx, tsClient, pg, proxyClass); err != nil { + return r.notReadyErrf(pg, logger, "error cleaning up dangling resources: %w", err) } - devices, err := r.getDeviceInfo(ctx, pg) + logger.Info("ProxyGroup resources synced") + + return staticEndpoints, nil, nil +} + +func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, oldPGStatus *tsapi.ProxyGroupStatus, nrr *notReadyReason, endpoints map[string][]netip.AddrPort) (err error) { + defer func() { + if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { + if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { + if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error updating status, retrying: %s", updateErr) + updateErr = nil + } + err = errors.Join(err, updateErr) + } + } + }() + + devices, err := r.getRunningProxies(ctx, pg, endpoints) if err != nil { - return fmt.Errorf("failed to get device info: %w", err) + return fmt.Errorf("failed to list running proxies: %w", err) } pg.Status.Devices = devices + desiredReplicas := int(pgReplicas(pg)) + + // Set ProxyGroupAvailable condition. + status := metav1.ConditionFalse + reason := reasonProxyGroupCreating + message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(devices), desiredReplicas) + if len(devices) > 0 { + status = metav1.ConditionTrue + if len(devices) == desiredReplicas { + reason = reasonProxyGroupAvailable + } + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) + + // Set ProxyGroupReady condition. + tsSvcValid, tsSvcSet := tsoperator.KubeAPIServerProxyValid(pg) + status = metav1.ConditionFalse + reason = reasonProxyGroupCreating + switch { + case nrr != nil: + // If we failed earlier, that reason takes precedence. + reason = nrr.reason + message = nrr.message + case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && tsSvcSet && !tsSvcValid: + reason = reasonProxyGroupInvalid + message = "waiting for config in spec.kubeAPIServer to be marked valid" + case len(devices) < desiredReplicas: + case len(devices) > desiredReplicas: + message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas) + case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !tsoperator.KubeAPIServerProxyConfigured(pg): + reason = reasonProxyGroupCreating + message = "waiting for proxies to start advertising the kube-apiserver proxy's hostname" + default: + status = metav1.ConditionTrue + reason = reasonProxyGroupReady + message = reasonProxyGroupReady + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) + return nil } +// getServicePortsForProxyGroups returns a map of ProxyGroup Service names to their NodePorts, +// and a set of all allocated NodePorts for quick occupancy checking. +func getServicePortsForProxyGroups(ctx context.Context, c client.Client, namespace string, portRanges tsapi.PortRanges) (map[string]uint16, set.Set[uint16], error) { + svcs := new(corev1.ServiceList) + matchingLabels := client.MatchingLabels(map[string]string{ + LabelParentType: "proxygroup", + }) + + err := c.List(ctx, svcs, matchingLabels, client.InNamespace(namespace)) + if err != nil { + return nil, nil, fmt.Errorf("failed to list ProxyGroup Services: %w", err) + } + + svcToNodePorts := map[string]uint16{} + usedPorts := set.Set[uint16]{} + for _, svc := range svcs.Items { + if len(svc.Spec.Ports) == 1 && svc.Spec.Ports[0].NodePort != 0 { + p := uint16(svc.Spec.Ports[0].NodePort) + if portRanges.Contains(p) { + svcToNodePorts[svc.Name] = p + usedPorts.Add(p) + } + } + } + + return svcToNodePorts, usedPorts, nil +} + +type allocatePortsErr struct { + msg string +} + +func (e *allocatePortsErr) Error() string { + return e.msg +} + +func (r *ProxyGroupReconciler) allocatePorts(ctx context.Context, pg *tsapi.ProxyGroup, proxyClassName string, portRanges tsapi.PortRanges) (map[string]uint16, error) { + replicaCount := int(pgReplicas(pg)) + svcToNodePorts, usedPorts, err := getServicePortsForProxyGroups(ctx, r.Client, r.tsNamespace, portRanges) + if err != nil { + return nil, &allocatePortsErr{msg: fmt.Sprintf("failed to find ports for existing ProxyGroup NodePort Services: %s", err.Error())} + } + + replicasAllocated := 0 + for i := range pgReplicas(pg) { + if _, ok := svcToNodePorts[pgNodePortServiceName(pg.Name, i)]; !ok { + svcToNodePorts[pgNodePortServiceName(pg.Name, i)] = 0 + } else { + replicasAllocated++ + } + } + + for replica, port := range svcToNodePorts { + if port == 0 { + for p := range portRanges.All() { + if !usedPorts.Contains(p) { + svcToNodePorts[replica] = p + usedPorts.Add(p) + replicasAllocated++ + break + } + } + } + } + + if replicasAllocated < replicaCount { + return nil, &allocatePortsErr{msg: fmt.Sprintf("not enough available ports to allocate all replicas (needed %d, got %d). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass %q must have bigger range allocated", replicaCount, usedPorts.Len(), proxyClassName)} + } + + return svcToNodePorts, nil +} + +func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) { + // NOTE: (ChaosInTheCRD) we want the same TargetPort for every static endpoint NodePort Service for the ProxyGroup + tailscaledPort := getRandomPort() + svcs := []*corev1.Service{} + for i := range pgReplicas(pg) { + nodePortSvcName := pgNodePortServiceName(pg.Name, i) + + svc := &corev1.Service{} + err := r.Get(ctx, types.NamespacedName{Name: nodePortSvcName, Namespace: r.tsNamespace}, svc) + if err != nil && !apierrors.IsNotFound(err) { + return nil, nil, fmt.Errorf("error getting Kubernetes Service %q: %w", nodePortSvcName, err) + } + if apierrors.IsNotFound(err) { + svcs = append(svcs, pgNodePortService(pg, nodePortSvcName, r.tsNamespace)) + } else { + // NOTE: if we can we want to recover the random port used for tailscaled, + // as well as the NodePort previously used for that Service + if len(svc.Spec.Ports) == 1 { + if svc.Spec.Ports[0].Port != 0 { + tailscaledPort = uint16(svc.Spec.Ports[0].Port) + } + } + svcs = append(svcs, svc) + } + } + + svcToNodePorts, err := r.allocatePorts(ctx, pg, pc.Name, pc.Spec.StaticEndpoints.NodePort.Ports) + if err != nil { + return nil, nil, fmt.Errorf("failed to allocate NodePorts to ProxyGroup Services: %w", err) + } + + for _, svc := range svcs { + // NOTE: we know that every service is going to have 1 port here + svc.Spec.Ports[0].Port = int32(tailscaledPort) + svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(tailscaledPort)) + svc.Spec.Ports[0].NodePort = int32(svcToNodePorts[svc.Name]) + + _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, svc, func(s *corev1.Service) { + s.ObjectMeta.Labels = svc.ObjectMeta.Labels + s.ObjectMeta.Annotations = svc.ObjectMeta.Annotations + s.ObjectMeta.OwnerReferences = svc.ObjectMeta.OwnerReferences + s.Spec.Selector = svc.Spec.Selector + s.Spec.Ports = svc.Spec.Ports + }) + if err != nil { + return nil, nil, fmt.Errorf("error creating/updating Kubernetes NodePort Service %q: %w", svc.Name, err) + } + } + + return svcToNodePorts, new(tailscaledPort), nil +} + // cleanupDanglingResources ensures we don't leak config secrets, state secrets, and // tailnet devices when the number of replicas specified is reduced. -func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error { +func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, tsClient tsclient.Client, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error { logger := r.logger(pg.Name) - metadata, err := r.getNodeMetadata(ctx, pg) + metadata, err := getNodeMetadata(ctx, pg, r.Client, r.tsNamespace) if err != nil { return err } for _, m := range metadata { - if m.ordinal+1 <= int(pgReplicas(pg)) { + if m.ordinal+1 <= pgReplicas(pg) { continue } // Dangling resource, delete the config + state Secrets, as well as // deleting the device from the tailnet. - if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil { + if err := r.ensureDeviceDeleted(ctx, tsClient, m.tsID, logger); err != nil { return err } - if err := r.Delete(ctx, m.stateSecret); err != nil { - if !apierrors.IsNotFound(err) { - return fmt.Errorf("error deleting state Secret %s: %w", m.stateSecret.Name, err) - } + if err := r.Delete(ctx, m.stateSecret); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting state Secret %q: %w", m.stateSecret.Name, err) } configSecret := m.stateSecret.DeepCopy() configSecret.Name += "-config" - if err := r.Delete(ctx, configSecret); err != nil { + if err := r.Delete(ctx, configSecret); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting config Secret %q: %w", configSecret.Name, err) + } + // NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-nodeport", m.stateSecret.Name), + Namespace: m.stateSecret.Namespace, + }, + } + if err := r.Delete(ctx, svc); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err) + return fmt.Errorf("error deleting static endpoints Kubernetes Service %q: %w", svc.Name, err) } } } + // If the ProxyClass has its StaticEndpoints config removed, we want to remove all of the NodePort Services + if pc != nil && pc.Spec.StaticEndpoints == nil { + labels := map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentType: proxyTypeProxyGroup, + LabelParentName: pg.Name, + } + if err := r.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("error deleting Kubernetes Services for static endpoints: %w", err) + } + } + return nil } // maybeCleanup just deletes the device from the tailnet. All the kubernetes // resources linked to a ProxyGroup will get cleaned up via owner references // (which we can use because they are all in the same namespace). -func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.ProxyGroup) (bool, error) { +func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, tsClient tsclient.Client, pg *tsapi.ProxyGroup) (bool, error) { logger := r.logger(pg.Name) - metadata, err := r.getNodeMetadata(ctx, pg) + metadata, err := getNodeMetadata(ctx, pg, r.Client, r.tsNamespace) if err != nil { return false, err } for _, m := range metadata { - if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil { + if err := r.ensureDeviceDeleted(ctx, tsClient, m.tsID, logger); err != nil { return false, err } } @@ -421,251 +699,577 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy mo := &metricsOpts{ proxyLabels: pgLabels(pg.Name, nil), tsNamespace: r.tsNamespace, - proxyType: "proxygroup"} + proxyType: "proxygroup", + } if err := maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil { return false, fmt.Errorf("error cleaning up metrics resources: %w", err) } logger.Infof("cleaned up ProxyGroup resources") r.mu.Lock() - r.ensureRemovedFromGaugeForProxyGroup(pg) + r.ensureStateRemovedForProxyGroup(pg) r.mu.Unlock() return true, nil } -func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { +func (r *ProxyGroupReconciler) ensureDeviceDeleted(ctx context.Context, tsClient tsclient.Client, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { logger.Debugf("deleting device %s from control", string(id)) - if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil { - errResp := &tailscale.ErrResponse{} - if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound { - logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id)) - } else { - return fmt.Errorf("error deleting device: %w", err) - } - } else { - logger.Debugf("device %s deleted from control", string(id)) + err := tsClient.Devices().Delete(ctx, string(id)) + switch { + case tailscale.IsNotFound(err): + logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id)) + case err != nil: + return fmt.Errorf("error deleting device: %w", err) } + logger.Debugf("device %s deleted from control", string(id)) return nil } -func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) { +func (r *ProxyGroupReconciler) ensureConfigSecretsCreated( + ctx context.Context, + tsClient tsclient.Client, + pg *tsapi.ProxyGroup, + proxyClass *tsapi.ProxyClass, + svcToNodePorts map[string]uint16, +) (endpoints map[string][]netip.AddrPort, err error) { logger := r.logger(pg.Name) - var configSHA256Sum string + endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name. for i := range pgReplicas(pg) { + logger = logger.With("Pod", fmt.Sprintf("%s-%d", pg.Name, i)) cfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pg.Name, i), Namespace: r.tsNamespace, - Labels: pgSecretLabels(pg.Name, "config"), + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig), OwnerReferences: pgOwnerReference(pg), }, } var existingCfgSecret *corev1.Secret // unmodified copy of secret - if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil { + if err = r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil { logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName()) existingCfgSecret = cfgSecret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return "", err + return nil, err } - var authKey string - if existingCfgSecret == nil { - logger.Debugf("Creating authkey for new ProxyGroup proxy") - tags := pg.Spec.Tags.Stringify() - if len(tags) == 0 { - tags = r.defaultTags + authKey, err := r.getAuthKey(ctx, tsClient, pg, existingCfgSecret, i, logger) + if err != nil { + return nil, err + } + + nodePortSvcName := pgNodePortServiceName(pg.Name, i) + if len(svcToNodePorts) > 0 { + replicaName := fmt.Sprintf("%s-%d", pg.Name, i) + port, ok := svcToNodePorts[nodePortSvcName] + if !ok { + return nil, fmt.Errorf("could not find configured NodePort for ProxyGroup replica %q", replicaName) } - authKey, err = newAuthKey(ctx, r.tsClient, tags) + + endpoints[nodePortSvcName], err = r.findStaticEndpoints(ctx, existingCfgSecret, proxyClass, port, logger) if err != nil { - return "", err + return nil, fmt.Errorf("could not find static endpoints for replica %q: %w", replicaName, err) } } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret) - if err != nil { - return "", fmt.Errorf("error creating tailscaled config: %w", err) - } + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + hostname := pgHostname(pg, i) + + if authKey == nil && existingCfgSecret != nil { + deviceAuthed := false + for _, d := range pg.Status.Devices { + if d.Hostname == hostname { + deviceAuthed = true + break + } + } + if !deviceAuthed { + existingCfg := conf.ConfigV1Alpha1{} + if err := json.Unmarshal(existingCfgSecret.Data[kubetypes.KubeAPIServerConfigFile], &existingCfg); err != nil { + return nil, fmt.Errorf("error unmarshalling existing config: %w", err) + } + if existingCfg.AuthKey != nil { + authKey = existingCfg.AuthKey + } + } + } - for cap, cfg := range configs { - cfgJSON, err := json.Marshal(cfg) + mode := kubetypes.APIServerProxyModeAuth + if !isAuthAPIServerProxy(pg) { + mode = kubetypes.APIServerProxyModeNoAuth + } + cfg := conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: authKey, + State: new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))), + App: new(kubetypes.AppProxyGroupKubeAPIServer), + LogLevel: new(logger.Level().String()), + + // Reloadable fields. + Hostname: &hostname, + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + Mode: &mode, + // The first replica is elected as the cert issuer, same + // as containerboot does for ingress-pg-reconciler. + IssueCerts: opt.NewBool(i == 0), + }, + LocalPort: new(uint16(9002)), + HealthCheckEnabled: opt.NewBool(true), + }, + } + + // Copy over config that the apiserver-proxy-service-reconciler sets. + if existingCfgSecret != nil { + if k8sProxyCfg, ok := cfgSecret.Data[kubetypes.KubeAPIServerConfigFile]; ok { + k8sCfg := &conf.ConfigV1Alpha1{} + if err := json.Unmarshal(k8sProxyCfg, k8sCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal kube-apiserver config: %w", err) + } + + cfg.AdvertiseServices = k8sCfg.AdvertiseServices + if k8sCfg.APIServerProxy != nil { + cfg.APIServerProxy.ServiceName = k8sCfg.APIServerProxy.ServiceName + } + } + } + + if tsClient.LoginURL() != "" { + cfg.ServerURL = new(tsClient.LoginURL()) + } + + if proxyClass != nil && proxyClass.Spec.TailscaleConfig != nil { + cfg.AcceptRoutes = opt.NewBool(proxyClass.Spec.TailscaleConfig.AcceptRoutes) + } + + if proxyClass != nil && proxyClass.Spec.Metrics != nil { + cfg.MetricsEnabled = opt.NewBool(proxyClass.Spec.Metrics.Enable) + } + + if len(endpoints[nodePortSvcName]) > 0 { + cfg.StaticEndpoints = endpoints[nodePortSvcName] + } + + cfgB, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) + } + mak.Set(&cfgSecret.Data, kubetypes.KubeAPIServerConfigFile, cfgB) + } else { + // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we + // don't overwrite it if already set. + existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret) + if err != nil { + return nil, err + } + + configs, err := pgTailscaledConfig(pg, tsClient.LoginURL(), proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices) if err != nil { - return "", fmt.Errorf("error marshalling tailscaled config: %w", err) + return nil, fmt.Errorf("error creating tailscaled config: %w", err) } - mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) - } - - // The config sha256 sum is a value for a hash annotation used to trigger - // pod restarts when tailscaled config changes. Any config changes apply - // to all replicas, so it is sufficient to only hash the config for the - // first replica. - // - // In future, we're aiming to eliminate restarts altogether and have - // pods dynamically reload their config when it changes. - if i == 0 { - sum := sha256.New() - for _, cfg := range configs { - // Zero out the auth key so it doesn't affect the sha256 hash when we - // remove it from the config after the pods have all authed. Otherwise - // all the pods will need to restart immediately after authing. - cfg.AuthKey = nil - b, err := json.Marshal(cfg) + + for cap, cfg := range configs { + cfgJSON, err := json.Marshal(cfg) if err != nil { - return "", err - } - if _, err := sum.Write(b); err != nil { - return "", err + return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) } + mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) } - - configSHA256Sum = fmt.Sprintf("%x", sum.Sum(nil)) } if existingCfgSecret != nil { if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) { logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name) if err := r.Update(ctx, cfgSecret); err != nil { - return "", err + return nil, err } } } else { logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name) if err := r.Create(ctx, cfgSecret); err != nil { - return "", err + return nil, err + } + } + + } + + return endpoints, nil +} + +// getAuthKey returns an auth key for the proxy, or nil if none is needed. +// A new key is created if the config Secret doesn't exist yet, or if the +// proxy has requested a reissue via its state Secret. An existing key is +// retained while the device hasn't authed or a reissue is in progress. +func (r *ProxyGroupReconciler) getAuthKey(ctx context.Context, tsClient tsclient.Client, pg *tsapi.ProxyGroup, existingCfgSecret *corev1.Secret, ordinal int32, logger *zap.SugaredLogger) (*string, error) { + // Get state Secret to check if it's already authed or has requested + // a fresh auth key. + stateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, ordinal), + Namespace: r.tsNamespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + + var createAuthKey bool + var cfgAuthKey *string + if existingCfgSecret == nil { + createAuthKey = true + } else { + var err error + cfgAuthKey, err = authKeyFromSecret(existingCfgSecret) + if err != nil { + return nil, fmt.Errorf("error retrieving auth key from existing config Secret: %w", err) + } + } + + if !createAuthKey { + var err error + createAuthKey, err = r.shouldReissueAuthKey(ctx, tsClient, pg, stateSecret, cfgAuthKey) + if err != nil { + return nil, err + } + } + + var authKey *string + if createAuthKey { + logger.Debugf("creating auth key for ProxyGroup proxy %q", stateSecret.Name) + + tags := pg.Spec.Tags.Stringify() + if len(tags) == 0 { + tags = r.defaultTags + } + key, err := newAuthKey(ctx, tsClient, tags) + if err != nil { + return nil, err + } + authKey = &key + } else { + // Retain auth key if the device hasn't authed yet, or if a + // reissue is in progress (device_id is stale during reissue). + _, reissueRequested := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !deviceAuthed(stateSecret) || reissueRequested { + authKey = cfgAuthKey + } + } + + return authKey, nil +} + +// shouldReissueAuthKey returns true if the proxy needs a new auth key. It +// tracks in-flight reissues via authKeyReissuing to avoid duplicate API calls +// across reconciles. +func (r *ProxyGroupReconciler) shouldReissueAuthKey(ctx context.Context, tsClient tsclient.Client, pg *tsapi.ProxyGroup, stateSecret *corev1.Secret, cfgAuthKey *string) (shouldReissue bool, err error) { + r.mu.Lock() + reissuing := r.authKeyReissuing[stateSecret.Name] + r.mu.Unlock() + + if reissuing { + // Check if reissue is complete by seeing if request was cleared + _, requestStillPresent := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !requestStillPresent { + // Containerboot cleared the request, reissue is complete + r.mu.Lock() + r.authKeyReissuing[stateSecret.Name] = false + r.mu.Unlock() + r.log.Debugf("auth key reissue completed for %q", stateSecret.Name) + return false, nil + } + + // Reissue still in-flight; waiting for containerboot to pick up new key + r.log.Debugf("auth key already in process of re-issuance, waiting for secret to be updated") + return false, nil + } + + defer func() { + r.mu.Lock() + r.authKeyReissuing[stateSecret.Name] = shouldReissue + r.mu.Unlock() + }() + + brokenAuthkey, ok := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !ok { + // reissue hasn't been requested since the key in the secret hasn't been populated + return false, nil + } + + empty := cfgAuthKey == nil || *cfgAuthKey == "" + broken := cfgAuthKey != nil && *cfgAuthKey == string(brokenAuthkey) + + // A new key has been written but the proxy hasn't picked it up yet. + if !empty && !broken { + return false, nil + } + + lim := r.authKeyRateLimits[pg.Name] + if !lim.Allow() { + r.log.Debugf("auth key re-issuance rate limit exceeded, limit: %.2f, burst: %d, tokens: %.2f", + lim.Limit(), lim.Burst(), lim.Tokens()) + return false, fmt.Errorf("auth key re-issuance rate limit exceeded for ProxyGroup %q, will retry with backoff", pg.Name) + } + + r.log.Infof("Proxy failing to auth; attempting cleanup and new key") + if tsID := stateSecret.Data[kubetypes.KeyDeviceID]; len(tsID) > 0 { + id := tailcfg.StableNodeID(tsID) + if err = r.ensureDeviceDeleted(ctx, tsClient, id, r.log); err != nil { + return false, err + } + } + + return true, nil +} + +type FindStaticEndpointErr struct { + msg string +} + +func (e *FindStaticEndpointErr) Error() string { + return e.msg +} + +// findStaticEndpoints returns up to two `netip.AddrPort` entries, derived from the ExternalIPs of Nodes that +// match the `proxyClass`'s selector within the StaticEndpoints configuration. The port is set to the replica's NodePort Service Port. +func (r *ProxyGroupReconciler) findStaticEndpoints(ctx context.Context, existingCfgSecret *corev1.Secret, proxyClass *tsapi.ProxyClass, port uint16, logger *zap.SugaredLogger) ([]netip.AddrPort, error) { + var currAddrs []netip.AddrPort + if existingCfgSecret != nil { + oldConfB := existingCfgSecret.Data[tsoperator.TailscaledConfigFileName(106)] + if len(oldConfB) > 0 { + var oldConf ipn.ConfigVAlpha + if err := json.Unmarshal(oldConfB, &oldConf); err == nil { + currAddrs = oldConf.StaticEndpoints + } else { + logger.Debugf("failed to unmarshal tailscaled config from secret %q: %v", existingCfgSecret.Name, err) + } + } else { + logger.Debugf("failed to get tailscaled config from secret %q: empty data", existingCfgSecret.Name) + } + } + + nodes := new(corev1.NodeList) + selectors := client.MatchingLabels(proxyClass.Spec.StaticEndpoints.NodePort.Selector) + + err := r.List(ctx, nodes, selectors) + if err != nil { + return nil, fmt.Errorf("failed to list nodes: %w", err) + } + + if len(nodes.Items) == 0 { + return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass %q", proxyClass.Name)} + } + + endpoints := []netip.AddrPort{} + + // NOTE(ChaosInTheCRD): Setting a hard limit of two static endpoints. + newAddrs := []netip.AddrPort{} + for _, n := range nodes.Items { + for _, a := range n.Status.Addresses { + if a.Type == corev1.NodeExternalIP { + addr := getStaticEndpointAddress(&a, port) + if addr == nil { + logger.Debugf("failed to parse %q address on node %q: %q", corev1.NodeExternalIP, n.Name, a.Address) + continue + } + + // we want to add the currently used IPs first before + // adding new ones. + if currAddrs != nil && slices.Contains(currAddrs, *addr) { + endpoints = append(endpoints, *addr) + } else { + newAddrs = append(newAddrs, *addr) + } + } + + if len(endpoints) == 2 { + break + } + } + } + + // if the 2 endpoints limit hasn't been reached, we + // can start adding newIPs. + if len(endpoints) < 2 { + for _, a := range newAddrs { + endpoints = append(endpoints, a) + if len(endpoints) == 2 { + break } } } - return configSHA256Sum, nil + if len(endpoints) == 0 { + return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to find any `status.addresses` of type %q on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass %q", corev1.NodeExternalIP, proxyClass.Name)} + } + + // If we ended up selecting the same set of addresses already in use, keep + // the existing order. nodes.Items from r.List is not guaranteed to be in + // a stable order across calls, so without this the slice can permute on + // each reconcile, making the marshalled config Secret differ byte-for-byte + // even though nothing has effectively changed. That trips the DeepEqual + // check on the config Secret, which writes the Secret, which fires a + // watch event, which re-enqueues the ProxyGroup, and so on. + if len(currAddrs) > 0 && sameAddrPortSet(endpoints, currAddrs) { + return currAddrs, nil + } + + return endpoints, nil } -// ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup -// is created. r.mu must be held. -func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGroup) { +// sameAddrPortSet reports whether a and b contain the same AddrPorts, +// ignoring order. Both slices are assumed to be free of duplicates, which +// holds for callers in this package. +func sameAddrPortSet(a, b []netip.AddrPort) bool { + if len(a) != len(b) { + return false + } + for _, x := range a { + if !slices.Contains(b, x) { + return false + } + } + return true +} + +func getStaticEndpointAddress(a *corev1.NodeAddress, port uint16) *netip.AddrPort { + addr, err := netip.ParseAddr(a.Address) + if err != nil { + return nil + } + + return new(netip.AddrPortFrom(addr, port)) +} + +// ensureStateAddedForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup +// is created, and initialises per-ProxyGroup rate limits on re-issuing auth keys. r.mu must be held. +func (r *ProxyGroupReconciler) ensureStateAddedForProxyGroup(pg *tsapi.ProxyGroup) { switch pg.Spec.Type { case tsapi.ProxyGroupTypeEgress: r.egressProxyGroups.Add(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Add(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Add(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) + + if _, ok := r.authKeyRateLimits[pg.Name]; !ok { + // Allow every replica to have its auth key re-issued quickly the first + // time, but with an overall limit of 1 every 30s after a burst. + r.authKeyRateLimits[pg.Name] = rate.NewLimiter(rate.Every(30*time.Second), int(pgReplicas(pg))) + } + + for i := range pgReplicas(pg) { + rep := pgStateSecretName(pg.Name, i) + if _, ok := r.authKeyReissuing[rep]; !ok { + r.authKeyReissuing[rep] = false + } + } } -// ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the -// ProxyGroup is deleted. r.mu must be held. -func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.ProxyGroup) { +// ensureStateRemovedForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the +// ProxyGroup is deleted, and deletes the per-ProxyGroup rate limiter to free memory. r.mu must be held. +func (r *ProxyGroupReconciler) ensureStateRemovedForProxyGroup(pg *tsapi.ProxyGroup) { switch pg.Spec.Type { case tsapi.ProxyGroupTypeEgress: r.egressProxyGroups.Remove(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Remove(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Remove(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) + delete(r.authKeyRateLimits, pg.Name) + for i := range pgReplicas(pg) { + delete(r.authKeyReissuing, pgStateSecretName(pg.Name, i)) + } } -func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { +func pgTailscaledConfig(pg *tsapi.ProxyGroup, loginServer string, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ - Version: "alpha0", - AcceptDNS: "false", - AcceptRoutes: "false", // AcceptRoutes defaults to true - Locked: "false", - Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)), + Version: "alpha0", + AcceptDNS: "false", + AcceptRoutes: "false", // AcceptRoutes defaults to true + Locked: "false", + Hostname: new(pgHostname(pg, idx)), + AdvertiseServices: oldAdvertiseServices, + AuthKey: authKey, } - if pg.Spec.HostnamePrefix != "" { - conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) + if loginServer != "" { + conf.ServerURL = &loginServer } - if shouldAcceptRoutes(class) { + if shouldAcceptRoutes(pc) { conf.AcceptRoutes = "true" } - deviceAuthed := false - for _, d := range pg.Status.Devices { - if d.Hostname == *conf.Hostname { - deviceAuthed = true - break - } - } - - if authKey != "" { - conf.AuthKey = &authKey - } else if !deviceAuthed { - key, err := authKeyFromSecret(oldSecret) - if err != nil { - return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err) - } - conf.AuthKey = key + if len(staticEndpoints) > 0 { + conf.StaticEndpoints = staticEndpoints } - capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) - // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we - // don't overwrite it here. - if err := copyAdvertiseServicesConfig(conf, oldSecret, 106); err != nil { - return nil, err - } - capVerConfigs[106] = *conf - return capVerConfigs, nil + return map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha{ + pgMinCapabilityVersion: *conf, + }, nil } -func copyAdvertiseServicesConfig(conf *ipn.ConfigVAlpha, oldSecret *corev1.Secret, capVer tailcfg.CapabilityVersion) error { - if oldSecret == nil { - return nil +func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error) { + if cfgSecret == nil { + return nil, nil } - oldConfB := oldSecret.Data[tsoperator.TailscaledConfigFileName(capVer)] - if len(oldConfB) == 0 { - return nil + cfg, err := latestConfigFromSecret(cfgSecret) + if err != nil { + return nil, err } - var oldConf ipn.ConfigVAlpha - if err := json.Unmarshal(oldConfB, &oldConf); err != nil { - return fmt.Errorf("error unmarshalling existing config: %w", err) + if cfg == nil { + return nil, nil } - conf.AdvertiseServices = oldConf.AdvertiseServices - return nil -} - -func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error { - return nil + return cfg.AdvertiseServices, nil } // getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by // querying their state Secrets. It may not return the same number of items as // specified in the ProxyGroup spec if e.g. it is getting scaled up or down, or // some pods have failed to write state. -func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) { - // List all state secrets owned by this ProxyGroup. +// +// The returned metadata will contain an entry for each state Secret that exists. +func getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup, cl client.Client, tsNamespace string) (metadata []nodeMetadata, _ error) { + // List all state Secrets owned by this ProxyGroup. secrets := &corev1.SecretList{} - if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil { + if err := cl.List(ctx, secrets, client.InNamespace(tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState))); err != nil { return nil, fmt.Errorf("failed to list state Secrets: %w", err) } for _, secret := range secrets.Items { - var ordinal int + var ordinal int32 if _, err := fmt.Sscanf(secret.Name, pg.Name+"-%d", &ordinal); err != nil { return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err) } + nm := nodeMetadata{ + ordinal: ordinal, + stateSecret: &secret, + } + prefs, ok, err := getDevicePrefs(&secret) if err != nil { return nil, err } - if !ok { - continue + if ok { + nm.tsID = prefs.Config.NodeID + nm.dnsName = prefs.Config.UserProfile.LoginName } - nm := nodeMetadata{ - ordinal: ordinal, - stateSecret: &secret, - tsID: prefs.Config.NodeID, - dnsName: prefs.Config.UserProfile.LoginName, - } pod := &corev1.Pod{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) { + if err := cl.Get(ctx, client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d", pg.Name, ordinal)}, pod); err != nil && !apierrors.IsNotFound(err) { return nil, err } else if err == nil { nm.podUID = string(pod.UID) @@ -673,58 +1277,92 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr metadata = append(metadata, nm) } + // Sort for predictable ordering and status. + sort.Slice(metadata, func(i, j int) bool { + return metadata[i].ordinal < metadata[j].ordinal + }) + return metadata, nil } -func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) { - metadata, err := r.getNodeMetadata(ctx, pg) +// getRunningProxies will return status for all proxy Pods whose state Secret +// has an up to date Pod UID and at least a hostname. +func (r *ProxyGroupReconciler) getRunningProxies(ctx context.Context, pg *tsapi.ProxyGroup, staticEndpoints map[string][]netip.AddrPort) (devices []tsapi.TailnetDevice, _ error) { + metadata, err := getNodeMetadata(ctx, pg, r.Client, r.tsNamespace) if err != nil { return nil, err } - for _, m := range metadata { - device, ok, err := getDeviceInfo(ctx, r.tsClient, m.stateSecret) - if err != nil { - return nil, err + for i, m := range metadata { + if m.podUID == "" || !strings.EqualFold(string(m.stateSecret.Data[kubetypes.KeyPodUID]), m.podUID) { + // Current Pod has not yet written its UID to the state Secret, data may + // be stale. + continue } - if !ok { + + device := tsapi.TailnetDevice{} + if hostname, _, ok := strings.Cut(string(m.stateSecret.Data[kubetypes.KeyDeviceFQDN]), "."); ok { + device.Hostname = hostname + } else { continue } - devices = append(devices, tsapi.TailnetDevice{ - Hostname: device.Hostname, - TailnetIPs: device.TailnetIPs, - }) + + if ipsB := m.stateSecret.Data[kubetypes.KeyDeviceIPs]; len(ipsB) > 0 { + ips := []string{} + if err := json.Unmarshal(ipsB, &ips); err != nil { + return nil, fmt.Errorf("failed to extract device IPs from state Secret %q: %w", m.stateSecret.Name, err) + } + device.TailnetIPs = ips + } + + // TODO(tomhjp): This is our input to the proxy, but we should instead + // read this back from the proxy's state in some way to more accurately + // reflect its status. + if ep, ok := staticEndpoints[pgNodePortServiceName(pg.Name, int32(i))]; ok && len(ep) > 0 { + eps := make([]string, 0, len(ep)) + for _, e := range ep { + eps = append(eps, e.String()) + } + device.StaticEndpoints = eps + } + + devices = append(devices, device) } return devices, nil } type nodeMetadata struct { - ordinal int + ordinal int32 stateSecret *corev1.Secret - // podUID is the UID of the current Pod or empty if the Pod does not exist. - podUID string - tsID tailcfg.StableNodeID - dnsName string + podUID string // or empty if the Pod no longer exists. + tsID tailcfg.StableNodeID + dnsName string } -// capVerForPG returns best effort capability version for the given ProxyGroup. It attempts to find it by looking at the -// Secret + Pod for the replica with ordinal 0. Returns -1 if it is not possible to determine the capability version -// (i.e there is no Pod yet). -func (r *ProxyGroupReconciler) capVerForPG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (tailcfg.CapabilityVersion, error) { - metas, err := r.getNodeMetadata(ctx, pg) - if err != nil { - return -1, fmt.Errorf("error getting node metadata: %w", err) - } - if len(metas) == 0 { - return -1, nil - } - dev, err := deviceInfo(metas[0].stateSecret, metas[0].podUID, logger) - if err != nil { - return -1, fmt.Errorf("error getting device info: %w", err) - } - if dev == nil { - return -1, nil +func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) { + return nil, ¬ReadyReason{ + reason: reason, + message: msg, + }, nil +} + +func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { + err := fmt.Errorf(format, a...) + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error()) + logger.Info(msg) + return notReady(reasonProxyGroupCreating, msg) } - return dev.capver, nil + + r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) + return nil, ¬ReadyReason{ + reason: reasonProxyGroupCreationFailed, + message: err.Error(), + }, err +} + +type notReadyReason struct { + reason string + message string } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 1d12c39e0241e..cc54656bbfd59 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,38 +7,78 @@ package main import ( "fmt" + "maps" "slices" "strconv" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/yaml" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubetypes" - "tailscale.com/types/ptr" ) -// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. -const deletionGracePeriodSeconds int64 = 360 +const ( + // deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. + deletionGracePeriodSeconds int64 = 360 + staticEndpointPortName = "static-endpoint-port" + // authAPIServerProxySAName is the ServiceAccount deployed by the helm chart + // if apiServerProxy.authEnabled is true. + authAPIServerProxySAName = "kube-apiserver-auth-proxy" +) + +func pgNodePortServiceName(proxyGroupName string, replica int32) string { + return fmt.Sprintf("%s-%d-nodeport", proxyGroupName, replica) +} + +func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + // NOTE(ChaosInTheCRD): we set the ports once we've iterated over every svc and found any old configuration we want to persist. + { + Name: staticEndpointPortName, + Protocol: corev1.ProtocolUDP, + }, + }, + Selector: map[string]string{ + appsv1.StatefulSetPodNameLabel: strings.TrimSuffix(name, "-nodeport"), + }, + }, + } +} // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + return kubeAPIServerStatefulSet(pg, namespace, image, port) + } ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) } // Validate some base assumptions. if len(ss.Spec.Template.Spec.InitContainers) != 1 { - return nil, fmt.Errorf("[unexpected] base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers)) + return nil, fmt.Errorf("base proxy config had %d init containers instead of 1", len(ss.Spec.Template.Spec.InitContainers)) } if len(ss.Spec.Template.Spec.Containers) != 1 { - return nil, fmt.Errorf("[unexpected] base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers)) + return nil, fmt.Errorf("base proxy config had %d containers instead of 1", len(ss.Spec.Template.Spec.Containers)) } // StatefulSet config. @@ -48,7 +88,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Labels: pgLabels(pg.Name, nil), OwnerReferences: pgOwnerReference(pg), } - ss.Spec.Replicas = ptr.To(pgReplicas(pg)) + ss.Spec.Replicas = new(pgReplicas(pg)) ss.Spec.Selector = &metav1.LabelSelector{ MatchLabels: pgLabels(pg.Name, nil), } @@ -59,7 +99,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Name: pg.Name, Namespace: namespace, Labels: pgLabels(pg.Name, nil), - DeletionGracePeriodSeconds: ptr.To[int64](10), + DeletionGracePeriodSeconds: new(int64(10)), } tmpl.Spec.ServiceAccountName = pg.Name tmpl.Spec.InitContainers[0].Image = image @@ -135,6 +175,11 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Value: "$(POD_NAME)", }, { + Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", + Value: "false", + }, + { + // TODO(tomhjp): This is tsrecorder-specific and does nothing. Delete. Name: "TS_STATE", Value: "kube:$(POD_NAME)", }, @@ -142,6 +187,21 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)", }, + { + // This ensures that cert renewals can succeed if ACME account + // keys have changed since issuance. We cannot guarantee or + // validate that the account key has not changed, see + // https://github.com/tailscale/tailscale/issues/18251 + Name: "TS_DEBUG_ACME_FORCE_RENEWAL", + Value: "true", + }, + } + + if port != nil { + envs = append(envs, corev1.EnvVar{ + Name: "PORT", + Value: strconv.Itoa(int(*port)), + }) } if tsFirewallMode != "" { @@ -223,11 +283,119 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string } // Set the deletion grace period to 6 minutes to ensure that the pre-stop hook has enough time to terminate // gracefully. - ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds) + ss.Spec.Template.DeletionGracePeriodSeconds = new(deletionGracePeriodSeconds) } + return ss, nil } +func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, port *uint16) (*appsv1.StatefulSet, error) { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: new(pgReplicas(pg)), + Selector: &metav1.LabelSelector{ + MatchLabels: pgLabels(pg.Name, nil), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + DeletionGracePeriodSeconds: new(int64(10)), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: pgServiceAccountName(pg), + Containers: []corev1.Container{ + { + Name: mainContainerName, + Image: image, + Env: func() []corev1.EnvVar { + envs := []corev1.EnvVar{ + { + // Used as default hostname and in Secret names. + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + // Used by kubeclient to post Events about the Pod's lifecycle. + Name: "POD_UID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + { + // Used in an interpolated env var if metrics enabled. + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + // Included for completeness with POD_IP and easier backwards compatibility in future. + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, + { + Name: "TS_K8S_PROXY_CONFIG", + Value: "kube:" + types.NamespacedName{ + Namespace: namespace, + Name: "$(POD_NAME)-config", + }.String(), + }, + { + // This ensures that cert renewals can succeed if ACME account + // keys have changed since issuance. We cannot guarantee or + // validate that the account key has not changed, see + // https://github.com/tailscale/tailscale/issues/18251 + Name: "TS_DEBUG_ACME_FORCE_RENEWAL", + Value: "true", + }, + } + + if port != nil { + envs = append(envs, corev1.EnvVar{ + Name: "PORT", + Value: strconv.Itoa(int(*port)), + }) + } + + return envs + }(), + Ports: []corev1.ContainerPort{ + { + Name: "k8s-proxy", + ContainerPort: 443, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + }, + } + + return sts, nil +} + func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount { return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -253,6 +421,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { Resources: []string{"secrets"}, Verbs: []string{ "list", + "watch", // For k8s-proxy. }, }, { @@ -266,8 +435,8 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { ResourceNames: func() (secrets []string) { for i := range pgReplicas(pg) { secrets = append(secrets, - pgConfigSecretName(pg.Name, i), // Config with auth key. - fmt.Sprintf("%s-%d", pg.Name, i), // State. + pgConfigSecretName(pg.Name, i), // Config with auth key. + pgPodName(pg.Name, i), // State. ) } return secrets @@ -297,7 +466,7 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: pg.Name, + Name: pgServiceAccountName(pg), Namespace: namespace, }, }, @@ -308,13 +477,34 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { } } +// kube-apiserver proxies in auth mode use a static ServiceAccount. Everything +// else uses a per-ProxyGroup ServiceAccount. +func pgServiceAccountName(pg *tsapi.ProxyGroup) string { + if isAuthAPIServerProxy(pg) { + return authAPIServerProxySAName + } + + return pg.Name +} + +func isAuthAPIServerProxy(pg *tsapi.ProxyGroup) bool { + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + return false + } + + // The default is auth mode. + return pg.Spec.KubeAPIServer == nil || + pg.Spec.KubeAPIServer.Mode == nil || + *pg.Spec.KubeAPIServer.Mode == tsapi.APIServerProxyModeAuth +} + func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) { for i := range pgReplicas(pg) { secrets = append(secrets, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", pg.Name, i), + Name: pgStateSecretName(pg.Name, i), Namespace: namespace, - Labels: pgSecretLabels(pg.Name, "state"), + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState), OwnerReferences: pgOwnerReference(pg), }, }) @@ -355,16 +545,14 @@ func pgSecretLabels(pgName, secretType string) map[string]string { } func pgLabels(pgName string, customLabels map[string]string) map[string]string { - l := make(map[string]string, len(customLabels)+3) - for k, v := range customLabels { - l[k] = v - } + labels := make(map[string]string, len(customLabels)+3) + maps.Copy(labels, customLabels) - l[kubetypes.LabelManaged] = "true" - l[LabelParentType] = "proxygroup" - l[LabelParentName] = pgName + labels[kubetypes.LabelManaged] = "true" + labels[LabelParentType] = "proxygroup" + labels[LabelParentName] = pgName - return l + return labels } func pgOwnerReference(owner *tsapi.ProxyGroup) []metav1.OwnerReference { @@ -379,10 +567,26 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 { return 2 } +func pgPodName(pgName string, i int32) string { + return fmt.Sprintf("%s-%d", pgName, i) +} + +func pgHostname(pg *tsapi.ProxyGroup, i int32) string { + if pg.Spec.HostnamePrefix != "" { + return fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, i) + } + + return fmt.Sprintf("%s-%d", pg.Name, i) +} + func pgConfigSecretName(pgName string, i int32) string { return fmt.Sprintf("%s-%d-config", pgName, i) } +func pgStateSecretName(pgName string, i int32) string { + return fmt.Sprintf("%s-%d", pgName, i) +} + func pgEgressCMName(pg string) string { return fmt.Sprintf("%s-egress-config", pg) } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index f3f87aaacf663..5432df940a4b3 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -9,39 +9,893 @@ import ( "context" "encoding/json" "fmt" + "net/netip" + "reflect" + "slices" + "strings" "testing" "time" "github.com/google/go-cmp/cmp" "go.uber.org/zap" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "tailscale.com/client/tailscale" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" + "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" "tailscale.com/tstest" - "tailscale.com/types/ptr" - "tailscale.com/util/mak" + "tailscale.com/types/opt" +) + +const ( + testProxyImage = "tailscale/tailscale:test" +) + +var ( + defaultProxyClassAnnotations = map[string]string{ + "some-annotation": "from-the-proxy-class", + } + + defaultReplicas = new(int32(2)) + defaultStaticEndpointConfig = &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 30001}, {Port: 30002}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + } ) -const testProxyImage = "tailscale/tailscale:test" +func TestProxyGroupWithStaticEndpoints(t *testing.T) { + type testNodeAddr struct { + ip string + addrType corev1.NodeAddressType + } + + type testNode struct { + name string + addresses []testNodeAddr + labels map[string]string + } -var defaultProxyClassAnnotations = map[string]string{ - "some-annotation": "from-the-proxy-class", + type reconcile struct { + staticEndpointConfig *tsapi.StaticEndpointsConfig + replicas *int32 + nodes []testNode + expectedIPs []netip.Addr + expectedEvents []string + expectedErr string + expectStatefulSet bool + } + + testCases := []struct { + name string + description string + reconciles []reconcile + }{ + { + // the reconciler should manage to create static endpoints when Nodes have IPv6 addresses. + name: "IPv6", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: new(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "2001:0db8::1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "2001:0db8::2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "2001:0db8::3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("2001:0db8::1"), netip.MustParseAddr("2001:0db8::2"), netip.MustParseAddr("2001:0db8::3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // declaring specific ports (with no `endPort`s) in the `spec.staticEndpoints.nodePort` should work. + name: "SpecificPorts", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: new(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("192.168.0.1"), netip.MustParseAddr("192.168.0.2"), netip.MustParseAddr("192.168.0.3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // if too narrow a range of `spec.staticEndpoints.nodePort.Ports` on the proxyClass should result in no StatefulSet being created. + name: "NotEnoughPorts", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: new(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning NodePort Services for static endpoints: failed to allocate NodePorts to ProxyGroup Services: not enough available ports to allocate all replicas (needed 4, got 3). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass \"default-pc\" must have bigger range allocated"}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // when supplying a variety of ranges that are not clashing, the reconciler should manage to create a StatefulSet. + name: "NonClashingRanges", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3002}, + {Port: 3003, EndPort: 3005}, + {Port: 3006}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: new(int32(3)), + nodes: []testNode{ + {name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + {name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + {name: "node3", addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // when there isn't a node that matches the selector, the ProxyGroup enters a failed state as there are no valid Static Endpoints. + // while it does create an event on the resource, It does not return an error + name: "NoMatchingNodes", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3005}, + }, + Selector: map[string]string{ + "zone": "us-west", + }, + }, + }, + replicas: defaultReplicas, + nodes: []testNode{ + {name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"zone": "eu-central"}}, + {name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, labels: map[string]string{"zone": "eu-central"}}, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // when all the nodes have only have addresses of type InternalIP populated in their status, the ProxyGroup enters a failed state as there are no valid Static Endpoints. + // while it does create an event on the resource, It does not return an error + name: "AllInternalIPAddresses", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: new(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to find any `status.addresses` of type \"ExternalIP\" on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass \"default-pc\""}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // When the node's (and some of their addresses) change between reconciles, the reconciler should first pick addresses that + // have been used previously (provided that they are still populated on a node that matches the selector) + name: "NodeIPChangesAndPersists", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.10", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectStatefulSet: true, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + }, + }, + }, + { + // given a new node being created with a new IP, and a node previously used for Static Endpoints being removed, the Static Endpoints should be updated + // correctly + name: "NodeIPChangesWithNewNode", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.3")}, + expectStatefulSet: true, + }, + }, + }, + { + // when all the node IPs change, they should all update + name: "AllNodeIPsChange", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.100", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.200", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.100"), netip.MustParseAddr("10.0.0.200")}, + expectStatefulSet: true, + }, + }, + }, + { + // if there are less ExternalIPs after changes to the nodes between reconciles, the reconciler should complete without issues + name: "LessExternalIPsAfterChange", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, + expectStatefulSet: true, + }, + }, + }, + { + // if node address parsing fails (given an invalid address), the reconciler should continue without failure and find other + // valid addresses + name: "NodeAddressParsingFails", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + }, + }, + { + // if the node's become unlabeled, the ProxyGroup should enter a ProxyGroupInvalid state, but the reconciler should not fail + name: "NodesBecomeUnlabeled", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{}, + }, + { + name: "node4", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectStatefulSet: true, + }, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + tsClient := &fakeTSClient{} + zl, _ := zap.NewDevelopment() + fr := record.NewFakeRecorder(10) + cl := tstest.NewClock(tstest.ClockOpts{}) + + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + ProxyClass: pc.Name, + }, + } + + fc := fake.NewClientBuilder(). + WithObjects(pc, pg). + WithStatusSubresource(pc, pg). + WithScheme(tsapi.GlobalScheme). + Build() + + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + defaultProxyClass: "default-pc", + + Client: fc, + clients: tsclient.NewProvider(tsClient), + recorder: fr, + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + } + + for i, r := range tt.reconciles { + var createdNodes []corev1.Node + t.Run(tt.name, func(t *testing.T) { + for _, n := range r.nodes { + no := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.name, + Labels: n.labels, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{}, + }, + } + for _, addr := range n.addresses { + no.Status.Addresses = append(no.Status.Addresses, corev1.NodeAddress{ + Type: addr.addrType, + Address: addr.ip, + }) + } + if err := fc.Create(t.Context(), no); err != nil { + t.Fatalf("failed to create node %q: %v", n.name, err) + } + createdNodes = append(createdNodes, *no) + t.Logf("created node %q with data", n.name) + } + + reconciler.log = zl.Sugar().With("TestName", tt.name).With("Reconcile", i) + pg.Spec.Replicas = r.replicas + pc.Spec.StaticEndpoints = r.staticEndpointConfig + + createOrUpdate(t.Context(), fc, "", pg, func(o *tsapi.ProxyGroup) { + o.Spec.Replicas = pg.Spec.Replicas + }) + + createOrUpdate(t.Context(), fc, "", pc, func(o *tsapi.ProxyClass) { + o.Spec.StaticEndpoints = pc.Spec.StaticEndpoints + }) + + if r.expectedErr != "" { + expectError(t, reconciler, "", pg.Name) + } else { + expectReconciled(t, reconciler, "", pg.Name) + } + expectEvents(t, fr, r.expectedEvents) + + sts := &appsv1.StatefulSet{} + err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts) + if r.expectStatefulSet { + if err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + for j := range 2 { + sec := &corev1.Secret{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d-config", pg.Name, j)}, sec); err != nil { + t.Fatalf("failed to get state Secret for replica %d: %v", j, err) + } + + config := &ipn.ConfigVAlpha{} + foundConfig := false + for _, d := range sec.Data { + if err := json.Unmarshal(d, config); err == nil { + foundConfig = true + break + } + } + if !foundConfig { + t.Fatalf("could not unmarshal config from secret data for replica %d", j) + } + + if len(config.StaticEndpoints) > staticEndpointsMaxAddrs { + t.Fatalf("expected %d StaticEndpoints in config Secret, but got %d for replica %d. Found Static Endpoints: %v", staticEndpointsMaxAddrs, len(config.StaticEndpoints), j, config.StaticEndpoints) + } + + for _, e := range config.StaticEndpoints { + if !slices.Contains(r.expectedIPs, e.Addr()) { + t.Fatalf("found unexpected static endpoint IP %q for replica %d. Expected one of %v", e.Addr().String(), j, r.expectedIPs) + } + if c := r.staticEndpointConfig; c != nil && c.NodePort.Ports != nil { + var ports tsapi.PortRanges = c.NodePort.Ports + found := false + for port := range ports.All() { + if port == e.Port() { + found = true + break + } + } + + if !found { + t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected one of %v .", e.Port(), j, ports.All()) + } + } else { + if e.Port() != 3001 && e.Port() != 3002 { + t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected 3001 or 3002.", e.Port(), j) + } + } + } + } + + pgroup := &tsapi.ProxyGroup{} + err = fc.Get(t.Context(), client.ObjectKey{Name: pg.Name}, pgroup) + if err != nil { + t.Fatalf("failed to get ProxyGroup %q: %v", pg.Name, err) + } + + t.Logf("getting proxygroup after reconcile") + for _, d := range pgroup.Status.Devices { + t.Logf("found device %q", d.Hostname) + for _, e := range d.StaticEndpoints { + t.Logf("found static endpoint %q", e) + } + } + } else { + if err == nil { + t.Fatal("expected error when getting Statefulset") + } + } + }) + + // node cleanup between reconciles + // we created a new set of nodes for each + for _, n := range createdNodes { + err := fc.Delete(t.Context(), &n) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("failed to delete node: %v", err) + } + } + } + + t.Run("delete_and_cleanup", func(t *testing.T) { + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + defaultProxyClass: "default-pc", + + Client: fc, + clients: tsclient.NewProvider(tsClient), + recorder: fr, + log: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + } + + if err := fc.Delete(t.Context(), pg); err != nil { + t.Fatalf("error deleting ProxyGroup: %v", err) + } + + expectReconciled(t, reconciler, "", pg.Name) + expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name) + + if err := fc.Delete(t.Context(), pc); err != nil { + t.Fatalf("error deleting ProxyClass: %v", err) + } + expectMissing[tsapi.ProxyClass](t, fc, "", pc.Name) + }) + }) + } } -func TestProxyGroup(t *testing.T) { - const initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196" +// TestFindStaticEndpointsStableOrder verifies that findStaticEndpoints returns +// the existing endpoint order from the config Secret when the resulting set of +// addresses is unchanged. nodes.Items from r.List is not order-stable across +// calls, so without this guarantee the slice can permute on each reconcile, +// triggering a spurious config Secret rewrite which fires a watch event that +// re-enqueues the ProxyGroup, looping forever (issue #19700). +func TestFindStaticEndpointsStableOrder(t *testing.T) { + const ( + addrA = "10.0.0.1" + addrB = "10.0.0.2" + port = uint16(30001) + ) + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pc"}, + Spec: tsapi.ProxyClassSpec{ + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{{Port: port}}, + Selector: map[string]string{"foo/bar": "baz"}, + }, + }, + }, + } + + // Existing config Secret already pins the order [B, A]. The fake client + // lists nodes in name order ([node-a, node-b]) so without the stable-order + // guard findStaticEndpoints would return [A, B], differing from currAddrs + // and causing a spurious Secret rewrite. + currAddrs := []netip.AddrPort{ + netip.MustParseAddrPort(addrB + ":30001"), + netip.MustParseAddrPort(addrA + ":30001"), + } + cfg := ipn.ConfigVAlpha{StaticEndpoints: currAddrs} + cfgJSON, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + existingSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-0-config", Namespace: tsNamespace}, + Data: map[string][]byte{tsoperator.TailscaledConfigFileName(106): cfgJSON}, + } + + nodes := []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: "node-a", Labels: map[string]string{"foo/bar": "baz"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeExternalIP, Address: addrA}, + }}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "node-b", Labels: map[string]string{"foo/bar": "baz"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{ + {Type: corev1.NodeExternalIP, Address: addrB}, + }}, + }, + } + + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc, nodes[0], nodes[1], existingSecret). + Build() + + zl, _ := zap.NewDevelopment() + r := &ProxyGroupReconciler{Client: fc} + + got, err := r.findStaticEndpoints(t.Context(), existingSecret, pc, port, zl.Sugar()) + if err != nil { + t.Fatalf("findStaticEndpoints: %v", err) + } + if !slices.Equal(got, currAddrs) { + t.Errorf("findStaticEndpoints returned %v, want %v (order must match currAddrs to avoid reconcile churn)", got, currAddrs) + } + + // Repeat to confirm the result is stable across calls. + got2, err := r.findStaticEndpoints(t.Context(), existingSecret, pc, port, zl.Sugar()) + if err != nil { + t.Fatalf("findStaticEndpoints (2nd call): %v", err) + } + if !slices.Equal(got, got2) { + t.Errorf("findStaticEndpoints not stable across calls: first=%v second=%v", got, got2) + } +} + +func TestProxyGroup(t *testing.T) { pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ Name: "default-pc", @@ -56,6 +910,7 @@ func TestProxyGroup(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test", Finalizers: []string{"tailscale.com/finalizer"}, + Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeEgress, @@ -73,17 +928,20 @@ func TestProxyGroup(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", - Client: fc, - tsClient: tsClient, - recorder: fr, - l: zl.Sugar(), - clock: cl, + Client: fc, + clients: tsclient.NewProvider(tsClient), + recorder: fr, + log: zl.Sugar(), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } + crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} opts := configOpts{ proxyType: "proxygroup", @@ -96,9 +954,13 @@ func TestProxyGroup(t *testing.T) { t.Run("proxyclass_not_ready", func(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, false, "", pc) + expectProxyGroupResources(t, fc, pg, false, pc) + if tsoperator.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to not be available") + } }) t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) { @@ -111,36 +973,43 @@ func TestProxyGroup(t *testing.T) { LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, }}, } - if err := fc.Status().Update(context.Background(), pc); err != nil { + if err := fc.Status().Update(t.Context(), pc); err != nil { t.Fatal(err) } - + pg.ObjectMeta.Generation = 2 + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.ObjectMeta.Generation = pg.ObjectMeta.Generation + }) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 2, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "", pc) + expectProxyGroupResources(t, fc, pg, true, pc) + if tsoperator.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to not be available") + } if expected := 1; reconciler.egressProxyGroups.Len() != expected { t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } - expectProxyGroupResources(t, fc, pg, true, "", pc) - keyReq := tailscale.KeyCapabilities{ - Devices: tailscale.KeyDeviceCapabilities{ - Create: tailscale.KeyDeviceCreateCapabilities{ - Reusable: false, - Ephemeral: false, - Preauthorized: true, - Tags: []string{"tag:test-tag"}, - }, - }, - } - if diff := cmp.Diff(tsClient.KeyRequests(), []tailscale.KeyCapabilities{keyReq, keyReq}); diff != "" { + expectProxyGroupResources(t, fc, pg, true, pc) + var keyReq tailscale.KeyCapabilities + keyReq.Devices.Create.Reusable = false + keyReq.Devices.Create.Ephemeral = false + keyReq.Devices.Create.Preauthorized = true + keyReq.Devices.Create.Tags = []string{"tag:test-tag"} + + if diff := cmp.Diff(tsClient.keyRequests, []tailscale.KeyCapabilities{keyReq, keyReq}); diff != "" { t.Fatalf("unexpected secrets (-got +want):\n%s", diff) } }) t.Run("simulate_successful_device_auth", func(t *testing.T) { addNodeIDToStateSecrets(t, fc, pg) + pg.ObjectMeta.Generation = 3 + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.ObjectMeta.Generation = pg.ObjectMeta.Generation + }) expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = []tsapi.TailnetDevice{ @@ -153,34 +1022,40 @@ func TestProxyGroup(t *testing.T) { TailnetIPs: []string{"1.2.3.4", "::1"}, }, } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) + if !tsoperator.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to be available") + } }) t.Run("scale_up_to_3", func(t *testing.T) { - pg.Spec.Replicas = ptr.To[int32](3) + pg.Spec.Replicas = new(int32(3)) mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { p.Spec = pg.Spec }) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 3, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ Hostname: "hostname-nodeid-2", TailnetIPs: []string{"1.2.3.4", "::1"}, }) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) }) t.Run("scale_down_to_1", func(t *testing.T) { - pg.Spec.Replicas = ptr.To[int32](1) + pg.Spec.Replicas = new(int32(1)) mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { p.Spec = pg.Spec }) @@ -188,22 +1063,9 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) - }) - - t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) { - pc.Spec.TailscaleConfig = &tsapi.TailscaleConfig{ - AcceptRoutes: true, - } - mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { - p.Spec = pc.Spec - }) - - expectReconciled(t, reconciler, "", pg.Name) - - expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74", pc) + expectProxyGroupResources(t, fc, pg, true, pc) }) t.Run("enable_metrics", func(t *testing.T) { @@ -228,7 +1090,7 @@ func TestProxyGroup(t *testing.T) { }) t.Run("delete_and_cleanup", func(t *testing.T) { - if err := fc.Delete(context.Background(), pg); err != nil { + if err := fc.Delete(t.Context(), pg); err != nil { t.Fatal(err) } @@ -274,12 +1136,14 @@ func TestProxyGroupTypes(t *testing.T) { zl, _ := zap.NewDevelopment() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zl.Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zl.Sugar(), + clients: tsclient.NewProvider(&fakeTSClient{}), + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } t.Run("egress_type", func(t *testing.T) { @@ -290,16 +1154,16 @@ func TestProxyGroupTypes(t *testing.T) { }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeEgress, - Replicas: ptr.To[int32](0), + Replicas: new(int32(0)), }, } mustCreate(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 0, 1) + verifyProxyGroupCounts(t, reconciler, 0, 1, 0) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress) @@ -309,7 +1173,7 @@ func TestProxyGroupTypes(t *testing.T) { // Verify that egress configuration has been set up. cm := &corev1.ConfigMap{} cmName := fmt.Sprintf("%s-egress-config", pg.Name) - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil { t.Fatalf("failed to get ConfigMap: %v", err) } @@ -365,7 +1229,7 @@ func TestProxyGroupTypes(t *testing.T) { }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeEgress, - Replicas: ptr.To[int32](0), + Replicas: new(int32(0)), ProxyClass: "test", }, } @@ -385,7 +1249,7 @@ func TestProxyGroupTypes(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } @@ -402,18 +1266,18 @@ func TestProxyGroupTypes(t *testing.T) { }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, - Replicas: ptr.To[int32](0), + Replicas: new(int32(0)), }, } - if err := fc.Create(context.Background(), pg); err != nil { + if err := fc.Create(t.Context(), pg); err != nil { t.Fatal(err) } expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 1, 2) + verifyProxyGroupCounts(t, reconciler, 1, 2, 0) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) @@ -447,6 +1311,207 @@ func TestProxyGroupTypes(t *testing.T) { t.Errorf("unexpected volume mounts (-want +got):\n%s", diff) } }) + + t.Run("kubernetes_api_server_type", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: new(int32(2)), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: new(tsapi.APIServerProxyModeNoAuth), + }, + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + + expectReconciled(t, reconciler, "", pg.Name) + verifyProxyGroupCounts(t, reconciler, 1, 2, 1) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + // Verify the StatefulSet configuration for KubernetesAPIServer type. + if sts.Spec.Template.Spec.Containers[0].Name != mainContainerName { + t.Errorf("unexpected container name %s, want %s", sts.Spec.Template.Spec.Containers[0].Name, mainContainerName) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort != 443 { + t.Errorf("unexpected container port %d, want 443", sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].Name != "k8s-proxy" { + t.Errorf("unexpected port name %s, want k8s-proxy", sts.Spec.Template.Spec.Containers[0].Ports[0].Name) + } + }) +} + +func TestKubeAPIServerStatusConditionFlow(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + Generation: 1, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: new(int32(1)), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: new(tsapi.APIServerProxyModeNoAuth), + }, + }, + } + stateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, 0), + Namespace: tsNamespace, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, stateSecret). + WithStatusSubresource(pg). + Build() + r := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zap.Must(zap.NewDevelopment()).Sugar(), + clients: tsclient.NewProvider(&fakeTSClient{}), + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + } + + expectReconciled(t, r, "", pg.Name) + pg.ObjectMeta.Finalizers = append(pg.ObjectMeta.Finalizers, FinalizerName) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.log) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set kube-apiserver valid. + mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log) + }) + expectReconciled(t, r, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set available. + addNodeIDToStateSecrets(t, fc, pg) + expectReconciled(t, r, "", pg.Name) + pg.Status.Devices = []tsapi.TailnetDevice{ + { + Hostname: "hostname-nodeid-0", + TailnetIPs: []string{"1.2.3.4", "::1"}, + }, + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.log) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set kube-apiserver configured. + mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log) + }) + expectReconciled(t, r, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.log) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) +} + +func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.ProxyGroup{}). + Build() + + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zap.Must(zap.NewDevelopment()).Sugar(), + clients: tsclient.NewProvider(&fakeTSClient{}), + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + } + + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: new(int32(1)), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: new(tsapi.APIServerProxyModeNoAuth), // Avoid needing to pre-create the static ServiceAccount. + }, + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + expectReconciled(t, reconciler, "", pg.Name) + + cfg := conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: new("new-authkey"), + State: new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))), + App: new(kubetypes.AppProxyGroupKubeAPIServer), + LogLevel: new("debug"), + + Hostname: new("test-k8s-apiserver-0"), + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + Mode: new(kubetypes.APIServerProxyModeNoAuth), + IssueCerts: opt.NewBool(true), + }, + LocalPort: new(uint16(9002)), + HealthCheckEnabled: opt.NewBool(true), + }, + } + cfgB, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("failed to marshal config: %v", err) + } + + cfgSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pg.Name, 0), + Namespace: tsNamespace, + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig), + OwnerReferences: pgOwnerReference(pg), + }, + Data: map[string][]byte{ + kubetypes.KubeAPIServerConfigFile: cfgB, + }, + } + expectEqual(t, fc, cfgSecret) + + // Now simulate the kube-apiserver services reconciler updating config, + // then check the proxygroup reconciler doesn't overwrite it. + cfg.APIServerProxy.ServiceName = new(tailcfg.ServiceName("svc:some-svc-name")) + cfg.AdvertiseServices = []string{"svc:should-not-be-overwritten"} + cfgB, err = json.Marshal(cfg) + if err != nil { + t.Fatalf("failed to marshal config: %v", err) + } + mustUpdate(t, fc, tsNamespace, cfgSecret.Name, func(s *corev1.Secret) { + s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + }) + expectReconciled(t, reconciler, "", pg.Name) + + cfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + expectEqual(t, fc, cfgSecret) } func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { @@ -455,12 +1520,14 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { WithStatusSubresource(&tsapi.ProxyGroup{}). Build() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zap.Must(zap.NewDevelopment()).Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + log: zap.Must(zap.NewDevelopment()).Sugar(), + clients: tsclient.NewProvider(&fakeTSClient{}), + clock: tstest.NewClock(tstest.ClockOpts{}), + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } existingServices := []string{"svc1", "svc2"} @@ -479,7 +1546,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { Namespace: tsNamespace, }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): existingConfigBytes, + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): existingConfigBytes, }, }) @@ -490,7 +1557,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, - Replicas: ptr.To[int32](1), + Replicas: new(int32(1)), }, }) expectReconciled(t, reconciler, "", pgName) @@ -504,7 +1571,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { AcceptDNS: "false", AcceptRoutes: "false", Locked: "false", - Hostname: ptr.To(fmt.Sprintf("%s-%d", pgName, 0)), + Hostname: new(fmt.Sprintf("%s-%d", pgName, 0)), }) if err != nil { t.Fatal(err) @@ -516,11 +1583,366 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { ResourceVersion: "2", }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): expectedConfigBytes, + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): expectedConfigBytes, }, }) } +func TestValidateProxyGroup(t *testing.T) { + type testCase struct { + typ tsapi.ProxyGroupType + pgName string + image string + noauth bool + initContainer bool + staticSAExists bool + expectedErrs int + } + + for name, tc := range map[string]testCase{ + "default_ingress": { + typ: tsapi.ProxyGroupTypeIngress, + }, + "default_kube": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + }, + "default_kube_noauth": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + noauth: true, + // Does not require the static ServiceAccount to exist. + }, + "kube_static_sa_missing": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: false, + expectedErrs: 1, + }, + "kube_noauth_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + noauth: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "ingress_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeIngress, + staticSAExists: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_1": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_2": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_3": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale:latest", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_4": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "tailscale/tailscale", + expectedErrs: 1, + }, + "k8s_proxy_image_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + image: "example.com/k8s-proxy", + expectedErrs: 1, + }, + "init_container_for_kube_pg": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + initContainer: true, + expectedErrs: 1, + }, + "init_container_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + initContainer: true, + }, + "init_container_for_egress_pg": { + typ: tsapi.ProxyGroupTypeEgress, + initContainer: true, + }, + } { + t.Run(name, func(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{}, + }, + }, + } + if tc.image != "" { + pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{ + Image: tc.image, + } + } + if tc.initContainer { + pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{} + } + pgName := "some-pg" + if tc.pgName != "" { + pgName = tc.pgName + } + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgName, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tc.typ, + }, + } + if tc.noauth { + pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{ + Mode: new(tsapi.APIServerProxyModeNoAuth), + } + } + + var objs []client.Object + if tc.staticSAExists { + objs = append(objs, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: authAPIServerProxySAName, + Namespace: tsNamespace, + }, + }) + } + r := ProxyGroupReconciler{ + tsNamespace: tsNamespace, + Client: fake.NewClientBuilder(). + WithObjects(objs...). + Build(), + } + + logger, _ := zap.NewDevelopment() + err := r.validate(t.Context(), pg, pc, logger.Sugar()) + if tc.expectedErrs == 0 { + if err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + // Test finished. + return + } + + if err == nil { + t.Fatalf("expected %d errors, got none", tc.expectedErrs) + } + + type unwrapper interface { + Unwrap() []error + } + errs := err.(unwrapper) + if len(errs.Unwrap()) != tc.expectedErrs { + t.Fatalf("expected %d errors, got %d: %v", tc.expectedErrs, len(errs.Unwrap()), err) + } + }) + } +} + +func TestProxyGroupGetAuthKey(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + Replicas: new(int32(1)), + }, + } + tsClient := &fakeTSClient{} + + // Variables to reference in test cases. + existingAuthKey := new("existing-auth-key") + newAuthKey := new("new-authkey") + configWith := func(authKey *string) map[string][]byte { + value := []byte("{}") + if authKey != nil { + value = fmt.Appendf(nil, `{"AuthKey": "%s"}`, *authKey) + } + return map[string][]byte{ + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): value, + } + } + + initTest := func() (*ProxyGroupReconciler, client.WithWatch) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg). + WithStatusSubresource(pg). + Build() + zl, _ := zap.NewDevelopment() + fr := record.NewFakeRecorder(1) + cl := tstest.NewClock(tstest.ClockOpts{}) + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + + Client: fc, + clients: tsclient.NewProvider(tsClient), + recorder: fr, + log: zl.Sugar(), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), + } + reconciler.ensureStateAddedForProxyGroup(pg) + + return reconciler, fc + } + + // Config Secret: exists or not, has key or not. + // State Secret: has device ID or not, requested reissue or not. + for name, tc := range map[string]struct { + configData map[string][]byte + stateData map[string][]byte + expectedAuthKey *string + expectReissue bool + }{ + "no_secrets_needs_new": { + expectedAuthKey: newAuthKey, // New ProxyGroup or manually cleared Pod. + }, + "no_config_secret_state_authed_ok": { + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + }, + expectedAuthKey: newAuthKey, // Always create an auth key if we're creating the config Secret. + }, + "config_secret_without_key_state_authed_with_reissue_needs_new": { + configData: configWith(nil), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + kubetypes.KeyReissueAuthkey: []byte(""), + }, + expectedAuthKey: newAuthKey, + expectReissue: true, // Device is authed but reissue was requested. + }, + "config_secret_with_key_state_with_reissue_stale_ok": { + configData: configWith(existingAuthKey), + stateData: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("some-older-authkey"), + }, + expectedAuthKey: existingAuthKey, // Config's auth key is different from the one marked for reissue. + }, + "config_secret_with_key_state_with_reissue_existing_key_needs_new": { + configData: configWith(existingAuthKey), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + kubetypes.KeyReissueAuthkey: []byte(*existingAuthKey), + }, + expectedAuthKey: newAuthKey, + expectReissue: true, // Current config's auth key is marked for reissue. + }, + "config_secret_without_key_no_state_ok": { + configData: configWith(nil), + expectedAuthKey: nil, // Proxy will set reissue_authkey and then next reconcile will reissue. + }, + "config_secret_without_key_state_authed_ok": { + configData: configWith(nil), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + }, + expectedAuthKey: nil, // Device is already authed. + }, + "config_secret_with_key_state_authed_ok": { + configData: configWith(existingAuthKey), + stateData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("nodeid-0"), + }, + expectedAuthKey: nil, // Auth key getting removed because device is authed. + }, + "config_secret_with_key_no_state_keeps_existing": { + configData: configWith(existingAuthKey), + expectedAuthKey: existingAuthKey, // No state, waiting for containerboot to try the auth key. + }, + } { + t.Run(name, func(t *testing.T) { + tsClient.deleted = tsClient.deleted[:0] // Reset deleted devices for each test case. + reconciler, fc := initTest() + var cfgSecret *corev1.Secret + if tc.configData != nil { + cfgSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pg.Name, 0), + Namespace: tsNamespace, + }, + Data: tc.configData, + } + } + if tc.stateData != nil { + mustCreate(t, fc, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, 0), + Namespace: tsNamespace, + }, + Data: tc.stateData, + }) + } + + authKey, err := reconciler.getAuthKey(t.Context(), tsClient, pg, cfgSecret, 0, reconciler.log.With("TestName", t.Name())) + if err != nil { + t.Fatalf("unexpected error getting auth key: %v", err) + } + if !reflect.DeepEqual(authKey, tc.expectedAuthKey) { + deref := func(s *string) string { + if s == nil { + return "" + } + return *s + } + t.Errorf("expected auth key %v, got %v", deref(tc.expectedAuthKey), deref(authKey)) + } + + // Use the device deletion as a proxy for the fact the new auth key + // was due to a reissue. + switch { + case tc.expectReissue && len(tsClient.deleted) != 1: + t.Errorf("expected 1 deleted device, got %v", tsClient.deleted) + case !tc.expectReissue && len(tsClient.deleted) != 0: + t.Errorf("expected no deleted devices, got %v", tsClient.deleted) + } + + if tc.expectReissue { + // Trigger the rate limit in a tight loop. Up to 100 iterations + // to allow for CI that is extremely slow, but should happen on + // first try for any reasonable machine. + stateSecretName := pgStateSecretName(pg.Name, 0) + for range 100 { + //NOTE: (ChaosInTheCRD) we added some protection here to avoid + // trying to reissue when already reissung. This overrides it. + reconciler.mu.Lock() + reconciler.authKeyReissuing[stateSecretName] = false + reconciler.mu.Unlock() + _, err := reconciler.getAuthKey(context.Background(), tsClient, pg, cfgSecret, 0, + reconciler.log.With("TestName", t.Name())) + if err != nil { + if !strings.Contains(err.Error(), "rate limit exceeded") { + t.Fatalf("unexpected error getting auth key: %v", err) + } + return // Expected rate limit error. + } + } + t.Fatal("expected rate limit error, but got none") + } + }) + } +} + func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) { pcLEStaging := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ @@ -556,7 +1978,7 @@ func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsap func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name string) *tsapi.ProxyClass { t.Helper() pc := &tsapi.ProxyClass{} - if err := fc.Get(context.Background(), client.ObjectKey{Name: name}, pc); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Name: name}, pc); err != nil { t.Fatal(err) } pc.Status = tsapi.ProxyClassStatus{ @@ -569,13 +1991,13 @@ func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name s ObservedGeneration: pc.Generation, }}, } - if err := fc.Status().Update(context.Background(), pc); err != nil { + if err := fc.Status().Update(t.Context(), pc); err != nil { t.Fatal(err) } return pc } -func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) { +func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress, wantAPIServer int) { t.Helper() if r.ingressProxyGroups.Len() != wantIngress { t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len()) @@ -583,6 +2005,9 @@ func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, if r.egressProxyGroups.Len() != wantEgress { t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len()) } + if r.apiServerProxyGroups.Len() != wantAPIServer { + t.Errorf("expected %d kube-apiserver proxy groups, got %d", wantAPIServer, r.apiServerProxyGroups.Len()) + } } func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) { @@ -608,20 +2033,17 @@ func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string) } } -func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string, proxyClass *tsapi.ProxyClass) { +func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, proxyClass *tsapi.ProxyClass) { t.Helper() role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", proxyClass) + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass) if err != nil { t.Fatal(err) } statefulSet.Annotations = defaultProxyClassAnnotations - if cfgHash != "" { - mak.Set(&statefulSet.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, cfgHash) - } if shouldExist { expectEqual(t, fc, role) @@ -651,7 +2073,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { t.Helper() secrets := &corev1.SecretList{} - if err := fc.List(context.Background(), secrets); err != nil { + if err := fc.List(t.Context(), secrets); err != nil { t.Fatal(err) } @@ -666,6 +2088,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { } func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) { + t.Helper() const key = "profile-abc" for i := range pgReplicas(pg) { bytes, err := json.Marshal(map[string]any{ @@ -677,10 +2100,27 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG t.Fatal(err) } - mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) { + podUID := fmt.Sprintf("pod-uid-%d", i) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", pg.Name, i), + Namespace: "tailscale", + UID: types.UID(podUID), + }, + } + if _, err := createOrUpdate(t.Context(), fc, "tailscale", pod, nil); err != nil { + t.Fatalf("failed to create or update Pod %s: %v", pod.Name, err) + } + mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) { s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, + currentProfileKey: []byte(key), + key: bytes, + kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`), + kubetypes.KeyDeviceFQDN: fmt.Appendf(nil, "hostname-nodeid-%d.tails-scales.ts.net", i), + // TODO(tomhjp): We have two different mechanisms to retrieve device IDs. + // Consolidate on this one. + kubetypes.KeyDeviceID: fmt.Appendf(nil, "nodeid-%d", i), + kubetypes.KeyPodUID: []byte(podUID), } }) } @@ -724,7 +2164,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { }, Spec: tsapi.ProxyGroupSpec{ Type: tt.pgType, - Replicas: ptr.To[int32](1), + Replicas: new(int32(1)), ProxyClass: tt.proxyClassPerResource, }, } @@ -746,13 +2186,15 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test"}, defaultProxyClass: tt.defaultProxyClass, Client: fc, - tsClient: &fakeTSClient{}, - l: zl.Sugar(), + clients: tsclient.NewProvider(&fakeTSClient{}), + log: zl.Sugar(), clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } expectReconciled(t, reconciler, "", pg.Name) @@ -760,7 +2202,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { // Verify that the StatefulSet created for ProxyGrup has // the expected setting for the staging endpoint. sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 70b25f2d28784..daad35aadf293 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,13 +7,13 @@ package main import ( "context" - "crypto/sha256" _ "embed" "encoding/json" "errors" "fmt" - "net/http" + "maps" "os" + "path" "slices" "strconv" "strings" @@ -21,6 +21,7 @@ import ( "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -28,15 +29,16 @@ import ( "k8s.io/apiserver/pkg/storage/names" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" - "tailscale.com/client/tailscale" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/net/netutil" "tailscale.com/tailcfg" "tailscale.com/types/opt" - "tailscale.com/types/ptr" "tailscale.com/util/mak" ) @@ -51,7 +53,7 @@ const ( // LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or // cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for // the Ingress or Service. - LabelProxyClass = "tailscale.com/proxy-class" + LabelAnnotationProxyClass = "tailscale.com/proxy-class" FinalizerName = "tailscale.com/finalizer" @@ -61,13 +63,14 @@ const ( AnnotationHostname = "tailscale.com/hostname" annotationTailnetTargetIPOld = "tailscale.com/ts-tailnet-target-ip" AnnotationTailnetTargetIP = "tailscale.com/tailnet-ip" - //MagicDNS name of tailnet node. + // MagicDNS name of tailnet node. AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" AnnotationProxyGroup = "tailscale.com/proxy-group" // Annotations settable by users on ingresses. - AnnotationFunnel = "tailscale.com/funnel" + AnnotationFunnel = "tailscale.com/funnel" + AnnotationHTTPRedirect = "tailscale.com/http-redirect" // If set to true, set up iptables/nftables rules in the proxy forward // cluster traffic to the tailnet IP of that proxy. This can only be set @@ -91,8 +94,6 @@ const ( podAnnotationLastSetClusterDNSName = "tailscale.com/operator-last-set-cluster-dns-name" podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip" podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn" - // podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents. - podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash" proxyTypeEgress = "egress_service" proxyTypeIngressService = "ingress_service" @@ -104,16 +105,20 @@ const ( defaultLocalAddrPort = 9002 // metrics and health check port letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory" + + mainContainerName = "tailscale" + operatorTailnet = "" ) var ( // tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods. tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"} // tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods. - tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash} + tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN} ) type tailscaleSTSConfig struct { + Replicas int32 ParentResourceName string ParentResourceUID string ChildResourceLabels map[string]string @@ -141,6 +146,16 @@ type tailscaleSTSConfig struct { ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one) + + // LoginServer denotes the URL of the control plane that should be used by the proxy. + LoginServer string + + // HostnamePrefix specifies the desired prefix for the device's hostname. The hostname will be suffixed with the + // ordinal number generated by the StatefulSet. + HostnamePrefix string + + // Tailnet specifies the Tailnet resource to use for producing auth keys. + Tailnet string } type connector struct { @@ -159,17 +174,18 @@ type tsnetServer interface { type tailscaleSTSReconciler struct { client.Client tsnetServer tsnetServer - tsClient tsClient + clients ClientProvider defaultTags []string operatorNamespace string proxyImage string proxyPriorityClassName string tsFirewallMode string + loginServer string } -func (sts tailscaleSTSReconciler) validate() error { - if sts.tsFirewallMode != "" && !isValidFirewallMode(sts.tsFirewallMode) { - return fmt.Errorf("invalid proxy firewall mode %s, valid modes are iptables, nftables or unset", sts.tsFirewallMode) +func (r *tailscaleSTSReconciler) validate() error { + if r.tsFirewallMode != "" && !isValidFirewallMode(r.tsFirewallMode) { + return fmt.Errorf("invalid proxy firewall mode %s, valid modes are iptables, nftables or unset", r.tsFirewallMode) } return nil } @@ -181,17 +197,17 @@ func IsHTTPSEnabledOnTailnet(tsnetServer tsnetServer) bool { // Provision ensures that the StatefulSet for the given service is running and // up to date. -func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) { +func (r *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) { // Do full reconcile. // TODO (don't create Service for the Connector) - hsvc, err := a.reconcileHeadlessService(ctx, logger, sts) + hsvc, err := r.reconcileHeadlessService(ctx, logger, sts) if err != nil { return nil, fmt.Errorf("failed to reconcile headless service: %w", err) } proxyClass := new(tsapi.ProxyClass) if sts.ProxyClassName != "" { - if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil { + if err := r.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil { return nil, fmt.Errorf("failed to get ProxyClass: %w", err) } if !tsoperator.ProxyClassIsReady(proxyClass) { @@ -201,11 +217,17 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga } sts.ProxyClass = proxyClass - secretName, tsConfigHash, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc) + tsClient, err := r.clients.For(sts.Tailnet) + if err != nil { + return nil, fmt.Errorf("failed to get tailscale client: %w", err) + } + + secretNames, err := r.provisionSecrets(ctx, tsClient, sts, hsvc, logger) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } - _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash) + + _, err = r.reconcileSTS(ctx, logger, sts, hsvc, secretNames) if err != nil { return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } @@ -215,7 +237,7 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga proxyLabels: hsvc.Labels, proxyType: sts.proxyType, } - if err = reconcileMetricsResources(ctx, logger, mo, sts.ProxyClass, a.Client); err != nil { + if err = reconcileMetricsResources(ctx, logger, mo, sts.ProxyClass, r.Client); err != nil { return nil, fmt.Errorf("failed to ensure metrics resources: %w", err) } return hsvc, nil @@ -224,17 +246,24 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga // Cleanup removes all resources associated that were created by Provision with // the given labels. It returns true when all resources have been removed, // otherwise it returns false and the caller should retry later. -func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) { +func (r *tailscaleSTSReconciler) Cleanup(ctx context.Context, tailnet string, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) { + tsClient, err := r.clients.For(tailnet) + if err != nil { + logger.Errorf("failed to get tailscale client: %v", err) + return false, nil + } + // Need to delete the StatefulSet first, and delete it with foreground // cascading deletion. That way, the pod that's writing to the Secret will // stop running before we start looking at the Secret's contents, and // assuming k8s ordering semantics don't mess with us, that should avoid // tailscale device deletion races where we fail to notice a device that // should be removed. - sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, labels) + sts, err := getSingleObject[appsv1.StatefulSet](ctx, r.Client, r.operatorNamespace, labels) if err != nil { return false, fmt.Errorf("getting statefulset: %w", err) } + if sts != nil { if !sts.GetDeletionTimestamp().IsZero() { // Deletion in progress, check again later. We'll get another @@ -242,49 +271,62 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare logger.Debugf("waiting for statefulset %s/%s deletion", sts.GetNamespace(), sts.GetName()) return false, nil } - err := a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, client.InNamespace(a.operatorNamespace), client.MatchingLabels(labels), client.PropagationPolicy(metav1.DeletePropagationForeground)) - if err != nil { + + options := []client.DeleteAllOfOption{ + client.InNamespace(r.operatorNamespace), + client.MatchingLabels(labels), + client.PropagationPolicy(metav1.DeletePropagationForeground), + } + + if err = r.DeleteAllOf(ctx, &appsv1.StatefulSet{}, options...); err != nil { return false, fmt.Errorf("deleting statefulset: %w", err) } + logger.Debugf("started deletion of statefulset %s/%s", sts.GetNamespace(), sts.GetName()) return false, nil } - dev, err := a.DeviceInfo(ctx, labels, logger) + devices, err := r.DeviceInfo(ctx, labels, logger) if err != nil { return false, fmt.Errorf("getting device info: %w", err) } - if dev != nil && dev.id != "" { - logger.Debugf("deleting device %s from control", string(dev.id)) - if err := a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil { - errResp := &tailscale.ErrResponse{} - if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound { + + for _, dev := range devices { + if dev.id != "" { + logger.Debugf("deleting device %s from control", string(dev.id)) + err = tsClient.Devices().Delete(ctx, string(dev.id)) + switch { + case tailscale.IsNotFound(err): logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id)) - } else { + case err != nil: return false, fmt.Errorf("deleting device: %w", err) } - } else { + logger.Debugf("device %s deleted from control", string(dev.id)) } } - types := []client.Object{ + resourceTypes := []client.Object{ &corev1.Service{}, &corev1.Secret{}, } - for _, typ := range types { - if err := a.DeleteAllOf(ctx, typ, client.InNamespace(a.operatorNamespace), client.MatchingLabels(labels)); err != nil { + + for _, resourceType := range resourceTypes { + if err = r.DeleteAllOf(ctx, resourceType, client.InNamespace(r.operatorNamespace), client.MatchingLabels(labels)); err != nil { return false, err } } + mo := &metricsOpts{ proxyLabels: labels, - tsNamespace: a.operatorNamespace, + tsNamespace: r.operatorNamespace, proxyType: typ, } - if err := maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil { + + if err = maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil { return false, fmt.Errorf("error cleaning up metrics resources: %w", err) } + return true, nil } @@ -315,12 +357,12 @@ func statefulSetNameBase(parent string) string { } } -func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) { +func (r *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) { nameBase := statefulSetNameBase(sts.ParentResourceName) hsvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ GenerateName: nameBase, - Namespace: a.operatorNamespace, + Namespace: r.operatorNamespace, Labels: sts.ChildResourceLabels, }, Spec: corev1.ServiceSpec{ @@ -328,138 +370,193 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l Selector: map[string]string{ "app": sts.ParentResourceUID, }, - IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicyPreferDualStack), + IPFamilyPolicy: new(corev1.IPFamilyPolicyPreferDualStack), }, } logger.Debugf("reconciling headless service for StatefulSet") - return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) + return createOrUpdate(ctx, r.Client, r.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) } -func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaledConfigs, _ error) { - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - // Hardcode a -0 suffix so that in future, if we support - // multiple StatefulSet replicas, we can provision -N for - // those. - Name: hsvc.Name + "-0", - Namespace: a.operatorNamespace, - Labels: stsC.ChildResourceLabels, - }, - } - var orig *corev1.Secret // unmodified copy of secret - if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil { - logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName()) - orig = secret.DeepCopy() - } else if !apierrors.IsNotFound(err) { - return "", "", nil, err - } +func (r *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, tsClient tsclient.Client, stsC *tailscaleSTSConfig, hsvc *corev1.Service, logger *zap.SugaredLogger) ([]string, error) { + secretNames := make([]string, stsC.Replicas) + + // Start by ensuring we have Secrets for the desired number of replicas. This will handle both creating and scaling + // up a StatefulSet. + for i := range stsC.Replicas { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", hsvc.Name, i), + Namespace: r.operatorNamespace, + Labels: stsC.ChildResourceLabels, + }, + } - var authKey string - if orig == nil { - // Initially it contains only tailscaled config, but when the - // proxy starts, it will also store there the state, certs and - // ACME account key. - sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels) - if err != nil { - return "", "", nil, err + // If we only have a single replica, use the hostname verbatim. Otherwise, use the hostname prefix and add + // an ordinal suffix. + hostname := stsC.Hostname + if stsC.HostnamePrefix != "" { + hostname = fmt.Sprintf("%s-%d", stsC.HostnamePrefix, i) } - if sts != nil { - // StatefulSet exists, so we have already created the secret. - // If the secret is missing, they should delete the StatefulSet. - logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName()) - return "", "", nil, nil + + secretNames[i] = secret.Name + + var orig *corev1.Secret // unmodified copy of secret + if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil { + logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName()) + orig = secret.DeepCopy() + } else if !apierrors.IsNotFound(err) { + return nil, err } - // Create API Key secret which is going to be used by the statefulset - // to authenticate with Tailscale. - logger.Debugf("creating authkey for new tailscale proxy") - tags := stsC.Tags - if len(tags) == 0 { - tags = a.defaultTags + + var ( + authKey string + err error + ) + + if orig == nil { + // Create API Key secret which is going to be used by the statefulset + // to authenticate with Tailscale. + logger.Debugf("creating authkey for new tailscale proxy") + tags := stsC.Tags + if len(tags) == 0 { + tags = r.defaultTags + } + + authKey, err = newAuthKey(ctx, tsClient, tags) + if err != nil { + return nil, err + } } - authKey, err = newAuthKey(ctx, a.tsClient, tags) + + configs, err := tailscaledConfig(stsC, tsClient.LoginURL(), authKey, orig, hostname) if err != nil { - return "", "", nil, err + return nil, fmt.Errorf("error creating tailscaled config: %w", err) + } + + latest := tailcfg.CapabilityVersion(-1) + var latestConfig ipn.ConfigVAlpha + for key, val := range configs { + fn := tsoperator.TailscaledConfigFileName(key) + b, err := json.Marshal(val) + if err != nil { + return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + } + + mak.Set(&secret.StringData, fn, string(b)) + if key > latest { + latest = key + latestConfig = val + } + } + + if stsC.ServeConfig != nil { + j, err := json.Marshal(stsC.ServeConfig) + if err != nil { + return nil, err + } + + mak.Set(&secret.StringData, "serve-config", string(j)) + } + + if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) { + logger.With("config", sanitizeConfig(latestConfig)).Debugf("patching the existing proxy Secret") + if err = r.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { + return nil, err + } + } else { + logger.With("config", sanitizeConfig(latestConfig)).Debugf("creating a new Secret for the proxy") + if err = r.Create(ctx, secret); err != nil { + return nil, err + } } } - configs, err := tailscaledConfig(stsC, authKey, orig) - if err != nil { - return "", "", nil, fmt.Errorf("error creating tailscaled config: %w", err) - } - hash, err = tailscaledConfigHash(configs) - if err != nil { - return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err) + + // Next, we check if we have additional secrets and remove them and their associated device. This happens when we + // scale an StatefulSet down. + var secrets corev1.SecretList + if err := r.List(ctx, &secrets, client.InNamespace(r.operatorNamespace), client.MatchingLabels(stsC.ChildResourceLabels)); err != nil { + return nil, err } - latest := tailcfg.CapabilityVersion(-1) - var latestConfig ipn.ConfigVAlpha - for key, val := range configs { - fn := tsoperator.TailscaledConfigFileName(key) - b, err := json.Marshal(val) - if err != nil { - return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + for _, secret := range secrets.Items { + var ordinal int32 + if _, err := fmt.Sscanf(secret.Name, hsvc.Name+"-%d", &ordinal); err != nil { + return nil, err } - mak.Set(&secret.StringData, fn, string(b)) - if key > latest { - latest = key - latestConfig = val + + if ordinal < stsC.Replicas { + continue } - } - if stsC.ServeConfig != nil { - j, err := json.Marshal(stsC.ServeConfig) + dev, err := deviceInfo(&secret, "", logger) if err != nil { - return "", "", nil, err + return nil, err } - mak.Set(&secret.StringData, "serve-config", string(j)) - } - if orig != nil { - logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig)) - if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { - return "", "", nil, err + if dev != nil && dev.id != "" { + // If we get a not found error then this device has possibly already been deleted in the admin console. + // So we can ignore this and move on to removing the secret. + if err = tsClient.Devices().Delete(ctx, string(dev.id)); err != nil && !tailscale.IsNotFound(err) { + return nil, err + } } - } else { - logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig)) - if err := a.Create(ctx, secret); err != nil { - return "", "", nil, err + + if err = r.Delete(ctx, &secret); err != nil { + return nil, err } } - return secret.Name, hash, configs, nil + + return secretNames, nil } -// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted -// auth key. -func sanitizeConfigBytes(c ipn.ConfigVAlpha) string { +// sanitizeConfig returns an ipn.ConfigVAlpha with sensitive fields redacted. Since we pump everything +// into JSON-encoded logs it's easier to read this with a .With method than converting it to a string. +func sanitizeConfig(c ipn.ConfigVAlpha) ipn.ConfigVAlpha { + // Explicitly redact AuthKey because we never want it appearing in logs. Never populate this with the + // actual auth key. if c.AuthKey != nil { - c.AuthKey = ptr.To("**redacted**") + c.AuthKey = new("**redacted**") } - sanitizedBytes, err := json.Marshal(c) - if err != nil { - return "invalid config" - } - return string(sanitizedBytes) + + return c } // DeviceInfo returns the device ID, hostname, IPs and capver for the Tailscale device that acts as an operator proxy. // It retrieves info from a Kubernetes Secret labeled with the provided labels. Capver is cross-validated against the // Pod to ensure that it is the currently running Pod that set the capver. If the Pod or the Secret does not exist, the // returned capver is -1. Either of device ID, hostname and IPs can be empty string if not found in the Secret. -func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) (dev *device, err error) { - sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels) - if err != nil { - return dev, err - } - if sec == nil { - return dev, nil +func (r *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) ([]*device, error) { + var secrets corev1.SecretList + if err := r.List(ctx, &secrets, client.InNamespace(r.operatorNamespace), client.MatchingLabels(childLabels)); err != nil { + return nil, err } - podUID := "" - pod := new(corev1.Pod) - if err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod); err != nil && !apierrors.IsNotFound(err) { - return dev, err - } else if err == nil { - podUID = string(pod.ObjectMeta.UID) + + devices := make([]*device, 0) + for _, sec := range secrets.Items { + podUID := "" + pod := new(corev1.Pod) + err := r.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod) + switch { + case apierrors.IsNotFound(err): + // If the Pod is not found, we won't have its UID. We can still get the device information but the + // capability version will be unknown. + case err != nil: + return nil, err + default: + podUID = string(pod.ObjectMeta.UID) + } + + info, err := deviceInfo(&sec, podUID, logger) + if err != nil { + return nil, err + } + + if info != nil { + devices = append(devices, info) + } } - return deviceInfo(sec, podUID, logger) + + return devices, nil } // device contains tailscale state of a proxy device as gathered from its tailscale state Secret. @@ -511,22 +608,18 @@ func deviceInfo(sec *corev1.Secret, podUID string, log *zap.SugaredLogger) (dev return dev, nil } -func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string, error) { - caps := tailscale.KeyCapabilities{ - Devices: tailscale.KeyDeviceCapabilities{ - Create: tailscale.KeyDeviceCreateCapabilities{ - Reusable: false, - Preauthorized: true, - Tags: tags, - }, - }, - } +func newAuthKey(ctx context.Context, client tsclient.Client, tags []string) (string, error) { + var caps tailscale.KeyCapabilities + caps.Devices.Create.Reusable = false + caps.Devices.Create.Preauthorized = true + caps.Devices.Create.Tags = tags - key, _, err := tsClient.CreateKey(ctx, caps) + key, err := client.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{Capabilities: caps}) if err != nil { return "", err } - return key, nil + + return key.Key, nil } //go:embed deploy/manifests/proxy.yaml @@ -535,7 +628,7 @@ var proxyYaml []byte //go:embed deploy/manifests/userspace-proxy.yaml var userspaceProxyYaml []byte -func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { +func (r *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecrets []string) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { @@ -548,17 +641,17 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S for i := range ss.Spec.Template.Spec.InitContainers { c := &ss.Spec.Template.Spec.InitContainers[i] if c.Name == "sysctler" { - c.Image = a.proxyImage + c.Image = r.proxyImage break } } } pod := &ss.Spec.Template container := &pod.Spec.Containers[0] - container.Image = a.proxyImage + container.Image = r.proxyImage ss.ObjectMeta = metav1.ObjectMeta{ Name: headlessSvc.Name, - Namespace: a.operatorNamespace, + Namespace: r.operatorNamespace, } for key, val := range sts.ChildResourceLabels { mak.Set(&ss.ObjectMeta.Labels, key, val) @@ -570,22 +663,37 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }, } mak.Set(&pod.Labels, "app", sts.ParentResourceUID) - for key, val := range sts.ChildResourceLabels { - pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod + // sync StatefulSet labels to Pod to make it easier for users to select the Pod + maps.Copy(pod.Labels, sts.ChildResourceLabels) + + if sts.Replicas > 0 { + ss.Spec.Replicas = new(sts.Replicas) } // Generic containerboot configuration options. container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_KUBE_SECRET", - Value: proxySecret, + Value: "$(POD_NAME)", + }, + corev1.EnvVar{ + Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", + Value: "false", }, corev1.EnvVar{ - // New style is in the form of cap-.hujson. Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", - Value: "/etc/tsconfig", + Value: "/etc/tsconfig/$(POD_NAME)", + }, + corev1.EnvVar{ + // This ensures that cert renewals can succeed if ACME account + // keys have changed since issuance. We cannot guarantee or + // validate that the account key has not changed, see + // https://github.com/tailscale/tailscale/issues/18251 + Name: "TS_DEBUG_ACME_FORCE_RENEWAL", + Value: "true", }, ) + if sts.ForwardClusterTrafficViaL7IngressProxy { container.Env = append(container.Env, corev1.EnvVar{ Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", @@ -593,28 +701,31 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }) } - configVolume := corev1.Volume{ - Name: "tailscaledconfig", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: proxySecret, + for i, secret := range proxySecrets { + configVolume := corev1.Volume{ + Name: "tailscaledconfig-" + strconv.Itoa(i), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret, + }, }, - }, + } + + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: fmt.Sprintf("tailscaledconfig-%d", i), + ReadOnly: true, + MountPath: path.Join("/etc/tsconfig/", secret), + }) } - pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: "tailscaledconfig", - ReadOnly: true, - MountPath: "/etc/tsconfig", - }) - if a.tsFirewallMode != "" { + if r.tsFirewallMode != "" { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_DEBUG_FIREWALL_MODE", - Value: a.tsFirewallMode, + Value: r.tsFirewallMode, }) } - pod.Spec.PriorityClassName = a.proxyPriorityClassName + pod.Spec.PriorityClassName = r.proxyPriorityClassName // Ingress/egress proxy configuration options. if sts.ClusterTargetIP != "" { @@ -644,27 +755,27 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S } else if sts.ServeConfig != nil { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_SERVE_CONFIG", - Value: "/etc/tailscaled/serve-config", + Value: "/etc/tailscaled/$(POD_NAME)/serve-config", }) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: "serve-config", - ReadOnly: true, - MountPath: "/etc/tailscaled", - }) - pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "serve-config", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: proxySecret, - Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}, + + for i, secret := range proxySecrets { + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: "serve-config-" + strconv.Itoa(i), + ReadOnly: true, + MountPath: path.Join("/etc/tailscaled", secret), + }) + + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "serve-config-" + strconv.Itoa(i), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret, + Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}, + }, }, - }, - }) - } + }) + } - dev, err := a.DeviceInfo(ctx, sts.ChildResourceLabels, logger) - if err != nil { - return nil, fmt.Errorf("failed to get device info: %w", err) } app, err := appInfoForProxy(sts) @@ -672,7 +783,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S // No need to error out if now or in future we end up in a // situation where app info cannot be determined for one of the // many proxy configurations that the operator can produce. - logger.Error("[unexpected] unable to determine proxy type") + logger.Error("unable to determine proxy type") } else { container.Env = append(container.Env, corev1.EnvVar{ Name: "TS_INTERNAL_APP", @@ -685,29 +796,11 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { - // This is a temporary workaround to ensure that proxies with capver older than 110 - // are restarted when tailscaled configfile contents have changed. - // This workaround ensures that: - // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. - // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. - // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid - // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a - // restarting Pod and the hash is re-added again. - // Note that the hash annotation is only set on updates not creation, because if the StatefulSet is - // being created, there is no need for a restart. - // TODO(irbekrm): remove this in 1.84. - hash := tsConfigHash - if dev == nil || dev.capver >= 110 { - hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] - } s.Spec = ss.Spec - if hash != "" { - mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) - } s.ObjectMeta.Labels = ss.Labels s.ObjectMeta.Annotations = ss.Annotations } - return createOrUpdate(ctx, a.Client, a.operatorNamespace, ss, updateSS) + return createOrUpdate(ctx, r.Client, r.operatorNamespace, ss, updateSS) } func appInfoForProxy(cfg *tailscaleSTSConfig) (string, error) { @@ -785,14 +878,21 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, enableEndpoints(ss, metricsEnabled, debugEnabled) } } - if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) { - for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { - ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ - Name: "TS_DEBUG_ACME_DIRECTORY_URL", - Value: letsEncryptStagingEndpoint, - }) - break + + if stsCfg != nil { + usesLetsEncrypt := stsCfg.proxyType == proxyTypeIngressResource || + stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress) || + stsCfg.proxyType == string(tsapi.ProxyGroupTypeKubernetesAPIServer) + + if pc.Spec.UseLetsEncryptStagingEnvironment && usesLetsEncrypt { + for i, c := range ss.Spec.Template.Spec.Containers { + if isMainContainer(&c) { + ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ + Name: "TS_DEBUG_ACME_DIRECTORY_URL", + Value: letsEncryptStagingEndpoint, + }) + break + } } } } @@ -826,7 +926,14 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector ss.Spec.Template.Spec.Affinity = wantsPod.Affinity ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations + ss.Spec.Template.Spec.PriorityClassName = wantsPod.PriorityClassName ss.Spec.Template.Spec.TopologySpreadConstraints = wantsPod.TopologySpreadConstraints + if wantsPod.DNSPolicy != nil { + ss.Spec.Template.Spec.DNSPolicy = *wantsPod.DNSPolicy + } + if wantsPod.DNSConfig != nil { + ss.Spec.Template.Spec.DNSConfig = wantsPod.DNSConfig + } // Update containers. updateContainer := func(overlay *tsapi.Container, base corev1.Container) corev1.Container { @@ -836,7 +943,17 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, if overlay.SecurityContext != nil { base.SecurityContext = overlay.SecurityContext } - base.Resources = overlay.Resources + + if len(overlay.Resources.Requests) > 0 { + base.Resources.Requests = overlay.Resources.Requests + } + if len(overlay.Resources.Limits) > 0 { + base.Resources.Limits = overlay.Resources.Limits + } + if len(overlay.Resources.Claims) > 0 { + base.Resources.Limits = overlay.Resources.Limits + } + for _, e := range overlay.Env { // Env vars configured via ProxyClass might override env // vars that have been specified by the operator, i.e @@ -855,7 +972,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, return base } for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { ss.Spec.Template.Spec.Containers[i] = updateContainer(wantsPod.TailscaleContainer, ss.Spec.Template.Spec.Containers[i]) break } @@ -873,7 +990,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { if debug { ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, // Serve tailscaled's debug metrics on on @@ -928,28 +1045,27 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { } } -func readAuthKey(secret *corev1.Secret, key string) (*string, error) { - origConf := &ipn.ConfigVAlpha{} - if err := json.Unmarshal([]byte(secret.Data[key]), origConf); err != nil { - return nil, fmt.Errorf("error unmarshaling previous tailscaled config in %q: %w", key, err) - } - return origConf.AuthKey, nil +func isMainContainer(c *corev1.Container) bool { + return c.Name == mainContainerName } // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy -// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that -// configuration. -func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { +// state and auth key and returns tailscaled config files for currently supported proxy versions. +func tailscaledConfig(stsC *tailscaleSTSConfig, loginUrl string, newAuthkey string, oldSecret *corev1.Secret, hostname string) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ Version: "alpha0", AcceptDNS: "false", AcceptRoutes: "false", // AcceptRoutes defaults to true Locked: "false", - Hostname: &stsC.Hostname, + Hostname: &hostname, NoStatefulFiltering: "true", // Explicitly enforce default value, see #14216 AppConnector: &ipn.AppConnectorPrefs{Advertise: false}, } + if stsC.LoginServer != "" { + conf.ServerURL = &stsC.LoginServer + } + if stsC.Connector != nil { routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode) if err != nil { @@ -966,7 +1082,7 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co if newAuthkey != "" { conf.AuthKey = &newAuthkey - } else if shouldRetainAuthKey(oldSecret) { + } else if !deviceAuthed(oldSecret) { key, err := authKeyFromSecret(oldSecret) if err != nil { return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err) @@ -974,6 +1090,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co conf.AuthKey = key } + if loginUrl != "" { + conf.ServerURL = new(loginUrl) + } + capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) capVerConfigs[107] = *conf @@ -983,7 +1103,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co return capVerConfigs, nil } -func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { +// latestConfigFromSecret returns the ipn.ConfigVAlpha with the highest capver +// as found in the Secret's key names, e.g. "cap-107.hujson" has capver 107. +// If no config is found, it returns nil. +func latestConfigFromSecret(s *corev1.Secret) (*ipn.ConfigVAlpha, error) { latest := tailcfg.CapabilityVersion(-1) latestStr := "" for k, data := range s.Data { @@ -1000,22 +1123,43 @@ func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { latest = v } } + + var conf *ipn.ConfigVAlpha + if latestStr != "" { + conf = &ipn.ConfigVAlpha{} + if err := json.Unmarshal([]byte(s.Data[latestStr]), conf); err != nil { + return nil, fmt.Errorf("error unmarshaling tailscaled config from Secret %q in field %q: %w", s.Name, latestStr, err) + } + } + + return conf, nil +} + +// authKeyFromSecret returns the auth key from the latest config version if +// found, or else nil. +func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { + conf, err := latestConfigFromSecret(s) + if err != nil { + return nil, err + } + // Allow for configs that don't contain an auth key. Perhaps // users have some mechanisms to delete them. Auth key is // normally not needed after the initial login. - if latestStr != "" { - return readAuthKey(s, latestStr) + if conf != nil { + key = conf.AuthKey } + return key, nil } -// shouldRetainAuthKey returns true if the state stored in a proxy's state Secret suggests that auth key should be -// retained (because the proxy has not yet successfully authenticated). -func shouldRetainAuthKey(s *corev1.Secret) bool { +// deviceAuthed returns true if the state stored in a proxy's state Secret +// suggests that the proxy has successfully authenticated. +func deviceAuthed(s *corev1.Secret) bool { if s == nil { - return false // nothing to retain here + return false // No state Secret means no device state. } - return len(s.Data["device_id"]) == 0 // proxy has not authed yet + return len(s.Data["device_id"]) > 0 } func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool { @@ -1031,27 +1175,6 @@ type ptrObject[T any] interface { type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha -// hashBytes produces a hash for the provided tailscaled config that is the same across -// different invocations of this code. We do not use the -// tailscale.com/deephash.Hash here because that produces a different hash for -// the same value in different tailscale builds. The hash we are producing here -// is used to determine if the container running the Connector Tailscale node -// needs to be restarted. The container does not need restarting when the only -// thing that changed is operator version (the hash is also exposed to users via -// an annotation and might be confusing if it changes without the config having -// changed). -func tailscaledConfigHash(c tailscaledConfigs) (string, error) { - b, err := json.Marshal(c) - if err != nil { - return "", fmt.Errorf("error marshalling tailscaled configs: %w", err) - } - h := sha256.New() - if _, err = h.Write(b); err != nil { - return "", fmt.Errorf("error calculating hash: %w", err) - } - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - // createOrMaybeUpdate adds obj to the k8s cluster, unless the object already exists, // in which case update is called to make changes to it. If update is nil or returns // an error, the object is returned unmodified. @@ -1166,6 +1289,22 @@ func nameForService(svc *corev1.Service) string { return svc.Namespace + "-" + svc.Name } +// proxyClassForObject returns the proxy class for the given object. If the +// object does not have a proxy class label, it returns the default proxy class +func proxyClassForObject(o client.Object, proxyDefaultClass string) string { + proxyClass, exists := o.GetLabels()[LabelAnnotationProxyClass] + if exists { + return proxyClass + } + + proxyClass, exists = o.GetAnnotations()[LabelAnnotationProxyClass] + if exists { + return proxyClass + } + + return proxyDefaultClass +} + func isValidFirewallMode(m string) bool { return m == "auto" || m == "nftables" || m == "iptables" } @@ -1182,7 +1321,7 @@ func proxyCapVer(sec *corev1.Secret, podUID string, log *zap.SugaredLogger) tail } capVer, err := strconv.Atoi(string(sec.Data[kubetypes.KeyCapVer])) if err != nil { - log.Infof("[unexpected]: unexpected capability version in proxy's state Secret, expected an integer, got %q", string(sec.Data[kubetypes.KeyCapVer])) + log.Warnf("unexpected capability version in proxy's state Secret, expected an integer, got %q", string(sec.Data[kubetypes.KeyCapVer])) return tailcfg.CapabilityVersion(-1) } if !strings.EqualFold(podUID, string(sec.Data[kubetypes.KeyPodUID])) { diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 35c512c8cd05b..f55f582a6de42 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -8,6 +8,7 @@ package main import ( _ "embed" "fmt" + "maps" "reflect" "regexp" "strings" @@ -22,7 +23,6 @@ import ( "sigs.k8s.io/yaml" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" - "tailscale.com/types/ptr" ) // Test_statefulSetNameBase tests that parent name portion in a StatefulSet name @@ -61,6 +61,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { // Setup proxyClassAllOpts := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ + UseLetsEncryptStagingEnvironment: true, StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"foo.io/bar": "foo"}, @@ -68,13 +69,14 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { Labels: tsapi.Labels{"bar": "foo"}, Annotations: map[string]string{"bar.io/foo": "foo"}, SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(0)), + RunAsUser: new(int64(0)), }, - ImagePullSecrets: []corev1.LocalObjectReference{{Name: "docker-creds"}}, - NodeName: "some-node", - NodeSelector: map[string]string{"beta.kubernetes.io/os": "linux"}, - Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}}, - Tolerations: []corev1.Toleration{{Key: "", Operator: "Exists"}}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "docker-creds"}}, + NodeName: "some-node", + NodeSelector: map[string]string{"beta.kubernetes.io/os": "linux"}, + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{}}}, + Tolerations: []corev1.Toleration{{Key: "", Operator: "Exists"}}, + PriorityClassName: "high-priority", TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ { WhenUnsatisfiable: "DoNotSchedule", @@ -85,9 +87,18 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { }, }, }, + DNSPolicy: new(corev1.DNSClusterFirstWithHostNet), + DNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"1.1.1.1", "8.8.8.8"}, + Searches: []string{"example.com", "test.local"}, + Options: []corev1.PodDNSConfigOption{ + {Name: "ndots", Value: new("2")}, + {Name: "edns0"}, + }, + }, TailscaleContainer: &tsapi.Container{ SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), + Privileged: new(true), }, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, @@ -99,8 +110,8 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { }, TailscaleInitContainer: &tsapi.Container{ SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - RunAsUser: ptr.To(int64(0)), + Privileged: new(true), + RunAsUser: new(int64(0)), }, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, @@ -197,6 +208,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent" wantSS.Spec.Template.Spec.InitContainers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething" wantSS.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = "IfNotPresent" + wantSS.Spec.Template.Spec.PriorityClassName = proxyClassAllOpts.Spec.StatefulSet.Pod.PriorityClassName + wantSS.Spec.Template.Spec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + wantSS.Spec.Template.Spec.DNSConfig = proxyClassAllOpts.Spec.StatefulSet.Pod.DNSConfig gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { @@ -235,6 +249,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent" wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething" + wantSS.Spec.Template.Spec.PriorityClassName = proxyClassAllOpts.Spec.StatefulSet.Pod.PriorityClassName + wantSS.Spec.Template.Spec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + wantSS.Spec.Template.Spec.DNSConfig = proxyClassAllOpts.Spec.StatefulSet.Pod.DNSConfig gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Errorf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) @@ -276,7 +293,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"}, ) wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9002}} - gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, ptr.To(false)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) + gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, new(false)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) } @@ -288,10 +305,14 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"}, ) wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "debug", Protocol: "TCP", ContainerPort: 9001}} - gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(false, ptr.To(true)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) + gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(false, new(true)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) } + + // 8. A Kubernetes API proxy with letsencrypt staging enabled + gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), &tailscaleSTSConfig{proxyType: string(tsapi.ProxyGroupTypeKubernetesAPIServer)}, zl.Sugar()) + verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) } func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { @@ -303,76 +324,76 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { want map[string]string }{ { - name: "no custom labels specified and none present in current labels, return current labels", + name: "no-custom-labels-none-present", current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, want: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, managed: tailscaleManagedLabels, }, { - name: "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels", + name: "no-custom-labels-some-present", current: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, want: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, managed: tailscaleManagedLabels, }, { - name: "custom labels specified, current labels only contain tailscale managed labels, return a union of both", + name: "custom-labels-with-managed-only", current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, want: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, managed: tailscaleManagedLabels, }, { - name: "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels", + name: "custom-labels-stale-removed", current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, want: map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, managed: tailscaleManagedLabels, }, { - name: "no current labels present, return custom labels only", + name: "no-current-labels-return-custom", custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, want: map[string]string{"foo": "bar", "something.io/foo": "bar"}, managed: tailscaleManagedLabels, }, { - name: "no current labels present, no custom labels specified, return empty map", + name: "no-current-no-custom-return-empty", want: map[string]string{}, managed: tailscaleManagedLabels, }, { - name: "no custom annots specified and none present in current annots, return current annots", + name: "no-custom-annots-none-present", current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { - name: "no custom annots specified, but some present in current annots, return tailscale managed annots only from the current annots", + name: "no-custom-annots-some-present", current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { - name: "custom annots specified, current annots only contain tailscale managed annots, return a union of both", + name: "custom-annots-with-managed-only", current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4"}, custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, want: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { - name: "custom annots specified, current annots contain tailscale managed annots and custom annots, some of which are not present in the new custom annots, return a union of managed annots and the desired custom annots", + name: "custom-annots-stale-removed", current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, custom: map[string]string{"something.io/foo": "bar"}, want: map[string]string{"something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4"}, managed: tailscaleManagedAnnotations, }, { - name: "no current annots present, return custom annots only", + name: "no-current-annots-return-custom", custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, want: map[string]string{"foo": "bar", "something.io/foo": "bar"}, managed: tailscaleManagedAnnotations, }, { - name: "no current labels present, no custom labels specified, return empty map", + name: "no-current-annots-no-custom-return-empty", want: map[string]string{}, managed: tailscaleManagedAnnotations, }, @@ -388,7 +409,5 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { // updateMap updates map a with the values from map b. func updateMap(a, b map[string]string) { - for key, val := range b { - a[key] = val - } + maps.Copy(a, b) } diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index c9b5b8ae69a18..08d573f837869 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -10,7 +10,6 @@ import ( "encoding/json" "errors" "fmt" - "net/http" "net/netip" "reflect" "slices" @@ -27,10 +26,12 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "tailscale.com/internal/client/tailscale" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" @@ -41,13 +42,10 @@ import ( ) const ( - finalizerName = "tailscale.com/service-pg-finalizer" - + svcPGFinalizerName = "tailscale.com/service-pg-finalizer" reasonIngressSvcInvalid = "IngressSvcInvalid" - reasonIngressSvcValid = "IngressSvcValid" reasonIngressSvcConfigured = "IngressSvcConfigured" reasonIngressSvcNoBackendsConfigured = "IngressSvcNoBackendsConfigured" - reasonIngressSvcCreationFailed = "IngressSvcCreationFailed" ) var gaugePGServiceResources = clientmetric.NewGauge(kubetypes.MetricServicePGResourceCount) @@ -59,10 +57,8 @@ type HAServiceReconciler struct { isDefaultLoadBalancer bool recorder record.EventRecorder logger *zap.SugaredLogger - tsClient tsClient - tsnetServer tsnetServer + clients ClientProvider tsNamespace string - lc localClient defaultTags []string operatorID string // stableID of the operator's Tailscale device @@ -101,12 +97,40 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque return res, fmt.Errorf("failed to get Service: %w", err) } + pgName := svc.Annotations[AnnotationProxyGroup] + if pgName == "" { + return res, nil + } + + logger = logger.With("ProxyGroup", pgName) + + pg := &tsapi.ProxyGroup{} + err = r.Get(ctx, client.ObjectKey{Name: pgName}, pg) + switch { + case apierrors.IsNotFound(err): + logger.Infof("ProxyGroup %q does not exist, it may have been deleted. Reconciliation for service %q will be skipped until the ProxyGroup is found", pgName, svc.Name) + r.recorder.Event(svc, corev1.EventTypeWarning, "ProxyGroupNotFound", "ProxyGroup not found") + return res, nil + case err != nil: + return res, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) + } + + if !tsoperator.ProxyGroupAvailable(pg) { + logger.Infof("ProxyGroup is not (yet) ready") + return res, nil + } + + tsClient, err := r.clients.For(pg.Spec.Tailnet) + if err != nil { + return res, fmt.Errorf("failed to get tailscale client: %w", err) + } + hostname := nameForService(svc) logger = logger.With("hostname", hostname) if !svc.DeletionTimestamp.IsZero() || !r.isTailscaleService(svc) { logger.Debugf("Service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up") - _, err = r.maybeCleanup(ctx, hostname, svc, logger) + _, err = r.maybeCleanup(ctx, hostname, svc, logger, tsClient) return res, err } @@ -114,7 +138,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque // is the case, we reconcile the Ingress one more time to ensure that concurrent updates to the Tailscale Service in a // multi-cluster Ingress setup have not resulted in another actor overwriting our Tailscale Service update. needsRequeue := false - needsRequeue, err = r.maybeProvision(ctx, hostname, svc, logger) + needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient) if err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { logger.Infof("optimistic lock error, retrying: %s", err) @@ -137,7 +161,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque // If a Tailscale Service exists, but does not have an owner reference from any operator, we error // out assuming that this is an owner reference created by an unknown actor. // Returns true if the operation resulted in a Tailscale Service update. -func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname string, svc *corev1.Service, logger *zap.SugaredLogger) (svcsChanged bool, err error) { +func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname string, svc *corev1.Service, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, tsClient tsclient.Client) (svcsChanged bool, err error) { oldSvcStatus := svc.Status.DeepCopy() defer func() { if !apiequality.Semantic.DeepEqual(oldSvcStatus, &svc.Status) { @@ -146,42 +170,19 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin } }() - pgName := svc.Annotations[AnnotationProxyGroup] - if pgName == "" { - logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning") - return false, nil - } - - logger = logger.With("ProxyGroup", pgName) - - pg := &tsapi.ProxyGroup{} - if err := r.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil { - if apierrors.IsNotFound(err) { - msg := fmt.Sprintf("ProxyGroup %q does not exist", pgName) - logger.Warnf(msg) - r.recorder.Event(svc, corev1.EventTypeWarning, "ProxyGroupNotFound", msg) - return false, nil - } - return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) - } - if !tsoperator.ProxyGroupIsReady(pg) { - logger.Infof("ProxyGroup is not (yet) ready") - return false, nil - } - - if err := r.validateService(ctx, svc, pg); err != nil { + if err = r.validateService(ctx, svc, pg); err != nil { r.recorder.Event(svc, corev1.EventTypeWarning, reasonIngressSvcInvalid, err.Error()) tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, err.Error(), r.clock, logger) return false, nil } - if !slices.Contains(svc.Finalizers, finalizerName) { + if !slices.Contains(svc.Finalizers, svcPGFinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, // this is a nice place to tell the operator that the high level, // multi-reconcile operation is underway. logger.Infof("exposing Service over tailscale") - svc.Finalizers = append(svc.Finalizers, finalizerName) + svc.Finalizers = append(svc.Finalizers, svcPGFinalizerName) if err := r.Update(ctx, svc); err != nil { return false, fmt.Errorf("failed to add finalizer: %w", err) } @@ -199,7 +200,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // that in edge cases (a single update changed both hostname and removed // ProxyGroup annotation) the Tailscale Service is more likely to be // (eventually) removed. - svcsChanged, err = r.maybeCleanupProxyGroup(ctx, pgName, logger) + svcsChanged, err = r.maybeCleanupProxyGroup(ctx, pg.Name, logger, tsClient) if err != nil { return false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err) } @@ -207,13 +208,8 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // 2. Ensure that there isn't a Tailscale Service with the same hostname // already created and not owned by this Service. serviceName := tailcfg.ServiceName("svc:" + hostname) - existingTSSvc, err := r.tsClient.GetVIPService(ctx, serviceName) - if isErrorFeatureFlagNotEnabled(err) { - logger.Warn(msgFeatureFlagNotEnabled) - r.recorder.Event(svc, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msgFeatureFlagNotEnabled) - return false, nil - } - if err != nil && !isErrorTailscaleServiceNotFound(err) { + existingTSSvc, err := tsClient.VIPServices().Get(ctx, serviceName.String()) + if err != nil && !tailscale.IsNotFound(err) { return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) } @@ -221,7 +217,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Service and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { instr := fmt.Sprintf("To proceed, you can either manually delete the existing Tailscale Service or choose a different hostname with the '%s' annotaion", AnnotationHostname) msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -236,8 +232,8 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin tags = strings.Split(tstr, ",") } - tsSvc := &tailscale.VIPService{ - Name: serviceName, + tsSvc := tailscale.VIPService{ + Name: serviceName.String(), Tags: tags, Ports: []string{"do-not-validate"}, // we don't want to validate ports Comment: managedTSServiceComment, @@ -252,15 +248,16 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // with the same generation number has been reconciled ~more than N times and stop attempting to apply updates. if existingTSSvc == nil || !reflect.DeepEqual(tsSvc.Tags, existingTSSvc.Tags) || - !ownersAreSetAndEqual(tsSvc, existingTSSvc) { + !ownersAreSetAndEqual(tsSvc, *existingTSSvc) { logger.Infof("Ensuring Tailscale Service exists and is up to date") - if err := r.tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil { + if err = tsClient.VIPServices().CreateOrUpdate(ctx, tsSvc); err != nil { return false, fmt.Errorf("error creating Tailscale Service: %w", err) } - existingTSSvc = tsSvc + + existingTSSvc = &tsSvc } - cm, cfgs, err := ingressSvcsConfigs(ctx, r.Client, pgName, r.tsNamespace) + cm, cfgs, err := ingressSvcsConfigs(ctx, r.Client, pg.Name, r.tsNamespace) if err != nil { return false, fmt.Errorf("error retrieving ingress services configuration: %w", err) } @@ -269,12 +266,12 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin return false, nil } - if existingTSSvc.Addrs == nil { - existingTSSvc, err = r.tsClient.GetVIPService(ctx, tsSvc.Name) - if err != nil { + if len(existingTSSvc.Addrs) == 0 { + existingTSSvc, err = tsClient.VIPServices().Get(ctx, tsSvc.Name) + switch { + case err != nil: return false, fmt.Errorf("error getting Tailscale Service: %w", err) - } - if existingTSSvc.Addrs == nil { + case len(existingTSSvc.Addrs) == 0: // TODO(irbekrm): this should be a retry return false, fmt.Errorf("unexpected: Tailscale Service addresses not populated") } @@ -335,7 +332,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin return false, fmt.Errorf("failed to update tailscaled config: %w", err) } - count, err := r.numberPodsAdvertising(ctx, pgName, serviceName) + count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName) if err != nil { return false, fmt.Errorf("failed to get number of advertised Pods: %w", err) } @@ -351,7 +348,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin conditionReason := reasonIngressSvcNoBackendsConfigured conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", count, pgReplicas(pg)) if count != 0 { - dnsName, err := r.dnsNameForService(ctx, serviceName) + dnsName, err := dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) if err != nil { return false, fmt.Errorf("error getting DNS name for Service: %w", err) } @@ -377,9 +374,9 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // Service is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only // deleted if it does not contain any other owner references. If it does the cleanup only removes the owner reference // corresponding to this Service. -func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, svc *corev1.Service, logger *zap.SugaredLogger) (svcChanged bool, err error) { +func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, svc *corev1.Service, logger *zap.SugaredLogger, tsClient tsclient.Client) (svcChanged bool, err error) { logger.Debugf("Ensuring any resources for Service are cleaned up") - ix := slices.Index(svc.Finalizers, finalizerName) + ix := slices.Index(svc.Finalizers, svcPGFinalizerName) if ix < 0 { logger.Debugf("no finalizer, nothing to do") return false, nil @@ -395,7 +392,7 @@ func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, serviceName := tailcfg.ServiceName("svc:" + hostname) // 1. Clean up the Tailscale Service. - svcChanged, err = r.cleanupTailscaleService(ctx, serviceName, logger) + svcChanged, err = cleanupTailscaleService(ctx, tsClient, serviceName.String(), r.operatorID, logger) if err != nil { return false, fmt.Errorf("error deleting Tailscale Service: %w", err) } @@ -428,14 +425,14 @@ func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, // Tailscale Services that are associated with the provided ProxyGroup and no longer managed this operator's instance are deleted, if not owned by other operator instances, else the owner reference is cleaned up. // Returns true if the operation resulted in existing Tailscale Service updates (owner reference removal). -func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger) (svcsChanged bool, err error) { +func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger, tsClient tsclient.Client) (svcsChanged bool, err error) { cm, config, err := ingressSvcsConfigs(ctx, r.Client, proxyGroupName, r.tsNamespace) if err != nil { return false, fmt.Errorf("failed to get ingress service config: %s", err) } svcList := &corev1.ServiceList{} - if err := r.Client.List(ctx, svcList, client.MatchingFields{indexIngressProxyGroup: proxyGroupName}); err != nil { + if err = r.Client.List(ctx, svcList, client.MatchingFields{indexIngressProxyGroup: proxyGroupName}); err != nil { return false, fmt.Errorf("failed to find Services for ProxyGroup %q: %w", proxyGroupName, err) } @@ -456,7 +453,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } - svcsChanged, err = r.cleanupTailscaleService(ctx, tailcfg.ServiceName(tsSvcName), logger) + svcsChanged, err = cleanupTailscaleService(ctx, tsClient, tsSvcName, r.operatorID, logger) if err != nil { return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err) } @@ -486,12 +483,12 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG func (r *HAServiceReconciler) deleteFinalizer(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { svc.Finalizers = slices.DeleteFunc(svc.Finalizers, func(f string) bool { - return f == finalizerName + return f == svcPGFinalizerName }) - logger.Debugf("ensure %q finalizer is removed", finalizerName) + logger.Debugf("ensure %q finalizer is removed", svcPGFinalizerName) if err := r.Update(ctx, svc); err != nil { - return fmt.Errorf("failed to remove finalizer %q: %w", finalizerName, err) + return fmt.Errorf("failed to remove finalizer %q: %w", svcPGFinalizerName, err) } r.mu.Lock() defer r.mu.Unlock() @@ -516,76 +513,64 @@ func (r *HAServiceReconciler) shouldExposeClusterIP(svc *corev1.Service) bool { return isTailscaleLoadBalancerService(svc, r.isDefaultLoadBalancer) || hasExposeAnnotation(svc) } -// tailnetCertDomain returns the base domain (TCD) of the current tailnet. -func (r *HAServiceReconciler) tailnetCertDomain(ctx context.Context) (string, error) { - st, err := r.lc.StatusWithoutPeers(ctx) - if err != nil { - return "", fmt.Errorf("error getting tailscale status: %w", err) - } - return st.CurrentTailnet.MagicDNSSuffix, nil -} - // cleanupTailscaleService deletes any Tailscale Service by the provided name if it is not owned by operator instances other than this one. // If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference. // If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing. // It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred. -func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, err error) { - svc, err := r.tsClient.GetVIPService(ctx, name) - if isErrorFeatureFlagNotEnabled(err) { - msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled) - logger.Warn(msg) +func cleanupTailscaleService(ctx context.Context, tsClient tsclient.Client, name string, operatorID string, logger *zap.SugaredLogger) (updated bool, err error) { + svc, err := tsClient.VIPServices().Get(ctx, name) + switch { + case tailscale.IsNotFound(err): return false, nil + case err != nil: + return false, fmt.Errorf("unexpected error getting Tailscale Service %q: %w", name, err) } - if err != nil { - errResp := &tailscale.ErrResponse{} - ok := errors.As(err, errResp) - if ok && errResp.Status == http.StatusNotFound { - return false, nil - } - if !ok { - return false, fmt.Errorf("unexpected error getting Tailscale Service %q: %w", name.String(), err) - } - return false, fmt.Errorf("error getting Tailscale Service: %w", err) - } if svc == nil { return false, nil } + o, err := parseOwnerAnnotation(svc) if err != nil { return false, fmt.Errorf("error parsing Tailscale Service owner annotation: %w", err) } + if o == nil || len(o.OwnerRefs) == 0 { return false, nil } + // Comparing with the operatorID only means that we will not be able to // clean up Tailscale Services in cases where the operator was deleted from the // cluster before deleting the Ingress. Perhaps the comparison could be // 'if or.OperatorID == r.operatorID || or.ingressUID == r.ingressUID'. ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool { - return or.OperatorID == r.operatorID + return or.OperatorID == operatorID }) if ix == -1 { return false, nil } + if len(o.OwnerRefs) == 1 { logger.Infof("Deleting Tailscale Service %q", name) - return false, r.tsClient.DeleteVIPService(ctx, name) + return false, tsClient.VIPServices().Delete(ctx, name) } + o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1) logger.Infof("Updating Tailscale Service %q", name) - json, err := json.Marshal(o) + + data, err := json.Marshal(o) if err != nil { return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err) } - svc.Annotations[ownerAnnotation] = string(json) - return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc) + + svc.Annotations[ownerAnnotation] = string(data) + return true, tsClient.VIPServices().CreateOrUpdate(ctx, *svc) } -func (a *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceName, replicaName, pgName string, wantsCfg *ingressservices.Config, logger *zap.SugaredLogger) (bool, error) { +func (r *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceName, replicaName string, wantsCfg *ingressservices.Config, logger *zap.SugaredLogger) (bool, error) { logger.Debugf("checking backend routes for service '%s'", serviceName) pod := &corev1.Pod{} - err := a.Get(ctx, client.ObjectKey{Namespace: a.tsNamespace, Name: replicaName}, pod) + err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: replicaName}, pod) if apierrors.IsNotFound(err) { logger.Debugf("Pod %q not found", replicaName) return false, nil @@ -594,7 +579,7 @@ func (a *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceNam return false, fmt.Errorf("failed to get Pod: %w", err) } secret := &corev1.Secret{} - err = a.Get(ctx, client.ObjectKey{Namespace: a.tsNamespace, Name: replicaName}, secret) + err = r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: replicaName}, secret) if apierrors.IsNotFound(err) { logger.Debugf("Secret %q not found", replicaName) return false, nil @@ -649,17 +634,17 @@ func isCurrentStatus(gotCfgs ingressservices.Status, pod *corev1.Pod, logger *za return true, nil } -func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, svc *corev1.Service, pgName string, serviceName tailcfg.ServiceName, cfg *ingressservices.Config, shouldBeAdvertised bool, logger *zap.SugaredLogger) (err error) { +func (r *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, svc *corev1.Service, pgName string, serviceName tailcfg.ServiceName, cfg *ingressservices.Config, shouldBeAdvertised bool, logger *zap.SugaredLogger) (err error) { logger.Debugf("checking advertisement for service '%s'", serviceName) // Get all config Secrets for this ProxyGroup. // Get all Pods secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil { + if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil { return fmt.Errorf("failed to list config Secrets: %w", err) } if svc != nil && shouldBeAdvertised { - shouldBeAdvertised, err = a.checkEndpointsReady(ctx, svc, logger) + shouldBeAdvertised, err = r.checkEndpointsReady(ctx, svc, logger) if err != nil { return fmt.Errorf("failed to check readiness of Service '%s' endpoints: %w", svc.Name, err) } @@ -688,10 +673,10 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con case shouldBeAdvertised: replicaName, ok := strings.CutSuffix(secret.Name, "-config") if !ok { - logger.Infof("[unexpected] unable to determine replica name from config Secret name %q, unable to determine if backend routing has been configured", secret.Name) + logger.Warnf("unable to determine replica name from config Secret name %q, unable to determine if backend routing has been configured", secret.Name) return nil } - ready, err := a.backendRoutesSetup(ctx, serviceName.String(), replicaName, pgName, cfg, logger) + ready, err := r.backendRoutesSetup(ctx, serviceName.String(), replicaName, cfg, logger) if err != nil { return fmt.Errorf("error checking backend routes: %w", err) } @@ -710,7 +695,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con updated = true } if updated { - if err := a.Update(ctx, &secret); err != nil { + if err := r.Update(ctx, &secret); err != nil { return fmt.Errorf("error updating ProxyGroup config Secret: %w", err) } } @@ -718,10 +703,10 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con return nil } -func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) { +func (r *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) { // Get all state Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil { + if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil { return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err) } @@ -742,56 +727,29 @@ func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName return count, nil } -// ownerAnnotations returns the updated annotations required to ensure this -// instance of the operator is included as an owner. If the Tailscale Service is not -// nil, but does not contain an owner we return an error as this likely means -// that the Tailscale Service was created by something other than a Tailscale -// Kubernetes operator. -func (r *HAServiceReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { - ref := OwnerRef{ - OperatorID: r.operatorID, - } - if svc == nil { - c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} - json, err := json.Marshal(c) - if err != nil { - return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service owner annotation contents: %w, please report this", err) - } - return map[string]string{ - ownerAnnotation: string(json), - }, nil - } - o, err := parseOwnerAnnotation(svc) - if err != nil { - return nil, err - } - if o == nil || len(o.OwnerRefs) == 0 { - return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) - } - if slices.Contains(o.OwnerRefs, ref) { // up to date - return svc.Annotations, nil - } - o.OwnerRefs = append(o.OwnerRefs, ref) - json, err := json.Marshal(o) - if err != nil { - return nil, fmt.Errorf("error marshalling updated owner references: %w", err) - } +// dnsNameForService returns the DNS name for the given Tailscale Service name. +func dnsNameForService(ctx context.Context, cl client.Client, svc tailcfg.ServiceName, pg *tsapi.ProxyGroup, namespace string) (string, error) { + s := svc.WithoutPrefix() - newAnnots := make(map[string]string, len(svc.Annotations)+1) - for k, v := range svc.Annotations { - newAnnots[k] = v + md, err := getNodeMetadata(ctx, pg, cl, namespace) + switch { + case err != nil: + return "", fmt.Errorf("error getting node metadata: %w", err) + case len(md) == 0: + return "", fmt.Errorf("failed to find node metadata for ProxyGroup %q", pg.Name) } - newAnnots[ownerAnnotation] = string(json) - return newAnnots, nil -} -// dnsNameForService returns the DNS name for the given Tailscale Service name. -func (r *HAServiceReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) { - s := svc.WithoutPrefix() - tcd, err := r.tailnetCertDomain(ctx) - if err != nil { - return "", fmt.Errorf("error determining DNS name base: %w", err) + // To determine the appropriate magic DNS name we take the first dns name we can find that is not empty and + // contains a period. + idx := slices.IndexFunc(md, func(metadata nodeMetadata) bool { + return metadata.dnsName != "" && strings.ContainsRune(metadata.dnsName, '.') + }) + if idx == -1 { + return "", fmt.Errorf("failed to find dns name for ProxyGroup %q", pg.Name) } + + tcd := strings.SplitN(md[idx].dnsName, ".", 2)[1] + return s + "." + tcd, nil } @@ -866,7 +824,7 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S } svcList := &corev1.ServiceList{} if err := r.List(ctx, svcList); err != nil { - errs = append(errs, fmt.Errorf("[unexpected] error listing Services: %w", err)) + errs = append(errs, fmt.Errorf("error listing Services: %w", err)) return errors.Join(errs...) } svcName := nameForService(svc) diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index 5772cd5d64e7f..455d3363cb956 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -22,15 +22,15 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "tailscale.com/ipn/ipnstate" + "tailscale.com/client/tailscale/v2" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/ingressservices" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" - "tailscale.com/types/ptr" "tailscale.com/util/mak" - - "tailscale.com/tailcfg" ) func TestServicePGReconciler(t *testing.T) { @@ -102,11 +102,11 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) { verifyTailscaleService(t, ft, fmt.Sprintf("svc:%s", hostname), []string{"do-not-validate"}) verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:%s", hostname)}) - _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(fmt.Sprintf("svc:default-%s", svc.Name))) + _, err := ft.VIPServices().Get(context.Background(), fmt.Sprintf("svc:default-%s", svc.Name)) if err == nil { t.Fatalf("svc:default-%s not cleaned up", svc.Name) } - if !isErrorTailscaleServiceNotFound(err) { + if !tailscale.IsNotFound(err) { t.Fatalf("unexpected error: %v", err) } } @@ -139,10 +139,10 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName("test-pg", 0), Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "config"), + Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte(`{"Version":""}`), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`), }, } @@ -179,7 +179,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien // Set ProxyGroup status to ready pg.Status.Conditions = []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, @@ -187,33 +187,24 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } - fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} - ft := &fakeTSClient{} + ft := &fakeTSClient{ + vipServices: make(map[string]tailscale.VIPService), + } zl, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } - lc := &fakeLocalClient{ - status: &ipnstate.Status{ - CurrentTailnet: &ipnstate.TailnetStatus{ - MagicDNSSuffix: "ts.net", - }, - }, - } - cl := tstest.NewClock(tstest.ClockOpts{}) svcPGR := &HAServiceReconciler{ Client: fc, - tsClient: ft, + clients: tsclient.NewProvider(ft), clock: cl, defaultTags: []string{"tag:k8s"}, tsNamespace: "operator-ns", - tsnetServer: fakeTsnetServer, logger: zl.Sugar(), recorder: record.NewFakeRecorder(10), - lc: lc, } return svcPGR, pgStateSecret, fc, ft, cl @@ -236,7 +227,7 @@ func TestValidateService(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.4", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, } svc2 := &corev1.Service{ @@ -253,7 +244,7 @@ func TestValidateService(t *testing.T) { Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.5", Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), }, } wantSvc := &corev1.Service{ @@ -281,21 +272,18 @@ func TestValidateService(t *testing.T) { func TestServicePGReconciler_MultiCluster(t *testing.T) { var ft *fakeTSClient - var lc localClient for i := 0; i <= 10; i++ { pgr, stateSecret, fc, fti, _ := setupServiceTest(t) if i == 0 { ft = fti - lc = pgr.lc } else { - pgr.tsClient = ft - pgr.lc = lc + pgr.clients = tsclient.NewProvider(ft) } svc, _ := setupTestService(t, "test-multi-cluster", "", "4.3.2.1", fc, stateSecret) expectReconciled(t, pgr, "default", svc.Name) - tsSvcs, err := ft.ListVIPServices(context.Background()) + tsSvcs, err := ft.VIPServices().List(t.Context()) if err != nil { t.Fatalf("getting Tailscale Service: %v", err) } @@ -304,8 +292,8 @@ func TestServicePGReconciler_MultiCluster(t *testing.T) { t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs)) } - for name := range tsSvcs { - t.Logf("found Tailscale Service with name %q", name.String()) + for _, svc := range tsSvcs { + t.Logf("found Tailscale Service with name %q", svc.Name) } } } @@ -336,7 +324,7 @@ func TestIgnoreRegularService(t *testing.T) { verifyTailscaledConfig(t, fc, "test-pg", nil) - tsSvcs, err := ft.ListVIPServices(context.Background()) + tsSvcs, err := ft.VIPServices().List(t.Context()) if err == nil { if len(tsSvcs) > 0 { t.Fatal("unexpected Tailscale Services found") @@ -393,7 +381,7 @@ func setupTestService(t *testing.T, svcName string, hostname string, clusterIP s }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, - LoadBalancerClass: ptr.To("tailscale"), + LoadBalancerClass: new("tailscale"), ClusterIP: clusterIP, ClusterIPs: []string{clusterIP}, }, @@ -413,7 +401,7 @@ func setupTestService(t *testing.T, svcName string, hostname string, clusterIP s { Addresses: []string{"4.3.2.1"}, Conditions: discoveryv1.EndpointConditions{ - Ready: ptr.To(true), + Ready: new(true), }, }, }, diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index c880f59f5012a..198a0c943eb2f 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -94,7 +95,7 @@ func childResourceLabels(name, ns, typ string) map[string]string { func (a *ServiceReconciler) isTailscaleService(svc *corev1.Service) bool { targetIP := tailnetTargetAnnotation(svc) targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN] - return a.shouldExpose(svc) || targetIP != "" || targetFQDN != "" + return shouldExpose(svc, a.isDefaultLoadBalancer) || targetIP != "" || targetFQDN != "" } func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { @@ -161,11 +162,11 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare } proxyTyp := proxyTypeEgress - if a.shouldExpose(svc) { + if shouldExpose(svc, a.isDefaultLoadBalancer) { proxyTyp = proxyTypeIngressService } - if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil { + if done, err := a.ssr.Cleanup(ctx, operatorTailnet, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil { return fmt.Errorf("failed to cleanup: %w", err) } else if !done { logger.Debugf("cleanup not done yet, waiting for next reconcile") @@ -262,24 +263,26 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga } sts := &tailscaleSTSConfig{ + Replicas: 1, ParentResourceName: svc.Name, ParentResourceUID: string(svc.UID), Hostname: nameForService(svc), Tags: tags, ChildResourceLabels: crl, ProxyClassName: proxyClass, + LoginServer: a.ssr.loginServer, } sts.proxyType = proxyTypeEgress - if a.shouldExpose(svc) { + if shouldExpose(svc, a.isDefaultLoadBalancer) { sts.proxyType = proxyTypeIngressService } a.mu.Lock() - if a.shouldExposeClusterIP(svc) { + if shouldExposeClusterIP(svc, a.isDefaultLoadBalancer) { sts.ClusterTargetIP = svc.Spec.ClusterIP a.managedIngressProxies.Add(svc.UID) gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len())) - } else if a.shouldExposeDNSName(svc) { + } else if shouldExposeDNSName(svc) { sts.ClusterTargetDNSName = svc.Spec.ExternalName a.managedIngressProxies.Add(svc.UID) gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len())) @@ -328,11 +331,12 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga return nil } - dev, err := a.ssr.DeviceInfo(ctx, crl, logger) + devices, err := a.ssr.DeviceInfo(ctx, crl, logger) if err != nil { return fmt.Errorf("failed to get device ID: %w", err) } - if dev == nil || dev.hostname == "" { + + if len(devices) == 0 || devices[0].hostname == "" { msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth" logger.Debug(msg) // No hostname yet. Wait for the proxy pod to auth. @@ -341,16 +345,20 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga return nil } + dev := devices[0] logger.Debugf("setting Service LoadBalancer status to %q, %s", dev.hostname, strings.Join(dev.ips, ", ")) + ingress := []corev1.LoadBalancerIngress{ {Hostname: dev.hostname}, } + clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP) if err != nil { msg := fmt.Sprintf("failed to parse cluster IP: %v", err) tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger) return errors.New(msg) } + for _, ip := range dev.ips { addr, err := netip.ParseAddr(ip) if err != nil { @@ -360,6 +368,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip}) } } + svc.Status.LoadBalancer.Ingress = ingress tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger) return nil @@ -367,6 +376,9 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga func validateService(svc *corev1.Service) []string { violations := make([]string, 0) + if svc.Spec.ClusterIP == "None" { + violations = append(violations, "headless Services are not supported.") + } if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" { violations = append(violations, fmt.Sprintf("only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN)) } @@ -396,19 +408,19 @@ func validateService(svc *corev1.Service) []string { return violations } -func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool { - return a.shouldExposeClusterIP(svc) || a.shouldExposeDNSName(svc) +func shouldExpose(svc *corev1.Service, isDefaultLoadBalancer bool) bool { + return shouldExposeClusterIP(svc, isDefaultLoadBalancer) || shouldExposeDNSName(svc) } -func (a *ServiceReconciler) shouldExposeDNSName(svc *corev1.Service) bool { +func shouldExposeDNSName(svc *corev1.Service) bool { return hasExposeAnnotation(svc) && svc.Spec.Type == corev1.ServiceTypeExternalName && svc.Spec.ExternalName != "" } -func (a *ServiceReconciler) shouldExposeClusterIP(svc *corev1.Service) bool { - if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" { +func shouldExposeClusterIP(svc *corev1.Service, isDefaultLoadBalancer bool) bool { + if svc.Spec.ClusterIP == "" { return false } - return isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) || hasExposeAnnotation(svc) + return isTailscaleLoadBalancerService(svc, isDefaultLoadBalancer) || hasExposeAnnotation(svc) } func isTailscaleLoadBalancerService(svc *corev1.Service, isDefaultLoadBalancer bool) bool { @@ -438,16 +450,6 @@ func tailnetTargetAnnotation(svc *corev1.Service) string { return svc.Annotations[annotationTailnetTargetIPOld] } -// proxyClassForObject returns the proxy class for the given object. If the -// object does not have a proxy class label, it returns the default proxy class -func proxyClassForObject(o client.Object, proxyDefaultClass string) string { - proxyClass, exists := o.GetLabels()[LabelProxyClass] - if !exists { - proxyClass = proxyDefaultClass - } - return proxyClass -} - func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) { proxyClass := new(tsapi.ProxyClass) if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil { @@ -466,7 +468,7 @@ func retrieveClusterDomain(namespace string, logger *zap.SugaredLogger) string { if err != nil { // Vast majority of clusters use the cluster.local domain, so it // is probably better to fall back to that than error out. - logger.Infof("[unexpected] error parsing /etc/resolv.conf to determine cluster domain, defaulting to 'cluster.local'.") + logger.Warn("error parsing /etc/resolv.conf to determine cluster domain, defaulting to 'cluster.local'.") return defaultClusterDomain } return clusterDomainFromResolverConf(conf, namespace, logger) @@ -478,17 +480,17 @@ func retrieveClusterDomain(namespace string, logger *zap.SugaredLogger) string { // If the domains don't match the expected structure or an error is encountered, it defaults to 'cluster.local' domain. func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string, logger *zap.SugaredLogger) string { if len(conf.SearchDomains) < 3 { - logger.Infof("[unexpected] resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.") + logger.Warnf(" resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.", len(conf.SearchDomains)) return defaultClusterDomain } first := conf.SearchDomains[0] if !strings.HasPrefix(string(first), namespace+".svc") { - logger.Infof("[unexpected] first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.") + logger.Warnf("first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.") return defaultClusterDomain } second := conf.SearchDomains[1] if !strings.HasPrefix(string(second), "svc") { - logger.Infof("[unexpected] second search domain in resolver config is %s; expected 'svc.'.\nDefaulting cluster domain to 'cluster.local'.", second) + logger.Warnf("second search domain in resolver config is %s; expected 'svc.'.\nDefaulting cluster domain to 'cluster.local'.", second) return defaultClusterDomain } // Trim the trailing dot for backwards compatibility purposes as the @@ -497,7 +499,7 @@ func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string probablyClusterDomain := strings.TrimPrefix(second.WithoutTrailingDot(), "svc.") third := conf.SearchDomains[2] if !strings.EqualFold(third.WithoutTrailingDot(), probablyClusterDomain) { - logger.Infof("[unexpected] expected resolver config to contain serch domains .svc., svc., ; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third) + logger.Warnf("expected resolver config to contain serch domains .svc., svc., ; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third) return defaultClusterDomain } logger.Infof("Cluster domain %q extracted from resolver config", probablyClusterDomain) diff --git a/cmd/k8s-operator/svc_test.go b/cmd/k8s-operator/svc_test.go new file mode 100644 index 0000000000000..677e9db10d40d --- /dev/null +++ b/cmd/k8s-operator/svc_test.go @@ -0,0 +1,223 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "slices" + "testing" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tstest" +) + +func TestService_DefaultProxyClassInitiallyNotReady(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, + Spec: tsapi.ProxyClassSpec{ + TailscaleConfig: &tsapi.TailscaleConfig{ + AcceptRoutes: true, + }, + StatefulSet: &tsapi.StatefulSet{ + Labels: tsapi.Labels{"foo": "bar"}, + Annotations: map[string]string{"bar.io/foo": "some-val"}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}, + }, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc). + WithStatusSubresource(pc). + Build() + ft := &fakeTSClient{} + zl := zap.Must(zap.NewDevelopment()) + clock := tstest.NewClock(tstest.ClockOpts{}) + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(ft), + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + defaultProxyClass: "custom-metadata", + logger: zl.Sugar(), + clock: clock, + } + + // 1. A new tailscale LoadBalancer Service is created but the default + // ProxyClass is not ready yet. + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + }, + }) + expectReconciled(t, sr, "default", "test") + labels := map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentName: "test", + LabelParentNamespace: "operator-ns", + LabelParentType: "svc", + } + s, err := getSingleObject[corev1.Secret](context.Background(), fc, "operator-ns", labels) + if err != nil { + t.Fatalf("finding Secret for %q: %v", "test", err) + } + if s != nil { + t.Fatalf("expected no Secret to be created when default ProxyClass is not ready, but found one: %v", s) + } + + // 2. ProxyClass is set to Ready, the Service can become ready now. + mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) { + pc.Status = tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Status: metav1.ConditionTrue, + Type: string(tsapi.ProxyClassReady), + ObservedGeneration: pc.Generation, + }}, + } + }) + expectReconciled(t, sr, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") + opts := configOpts{ + replicas: new(int32(1)), + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "svc", + hostname: "default-test", + clusterTargetIP: "10.20.30.40", + app: kubetypes.AppIngressProxy, + proxyClass: pc.Name, + } + expectEqual(t, fc, expectedSecret(t, fc, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) +} + +func TestProxyClassHandlerForSvc(t *testing.T) { + svc := func(name string, annotations, labels map[string]string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + }, + } + } + lbSvc := func(name string, annotations map[string]string, class *string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "foo", + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: class, + ClusterIP: "1.2.3.4", + }, + } + } + + const ( + defaultPCName = "default-proxyclass" + otherPCName = "other-proxyclass" + unreferencedPCName = "unreferenced-proxyclass" + ) + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithIndex(&corev1.Service{}, indexServiceProxyClass, indexProxyClass). + WithIndex(&corev1.Service{}, indexServiceExposed, indexExposed). + WithIndex(&corev1.Service{}, indexServiceType, indexType). + WithObjects( + svc("not-exposed", nil, nil), + svc("exposed-default", map[string]string{AnnotationExpose: "true"}, nil), + svc("exposed-other", map[string]string{AnnotationExpose: "true", LabelAnnotationProxyClass: otherPCName}, nil), + svc("annotated", map[string]string{LabelAnnotationProxyClass: defaultPCName}, nil), + svc("labelled", nil, map[string]string{LabelAnnotationProxyClass: defaultPCName}), + lbSvc("lb-svc", nil, new("tailscale")), + lbSvc("lb-svc-no-class", nil, nil), + lbSvc("lb-svc-other-class", nil, new("other")), + lbSvc("lb-svc-other-pc", map[string]string{LabelAnnotationProxyClass: otherPCName}, nil), + ). + Build() + + zl := zap.Must(zap.NewDevelopment()) + mapFunc := proxyClassHandlerForSvc(fc, zl.Sugar(), defaultPCName, true) + + for _, tc := range []struct { + name string + proxyClassName string + expected []reconcile.Request + }{ + { + name: "default_ProxyClass", + proxyClassName: defaultPCName, + expected: []reconcile.Request{ + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "exposed-default"}}, + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "annotated"}}, + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "labelled"}}, + {NamespacedName: types.NamespacedName{Namespace: "foo", Name: "lb-svc"}}, + {NamespacedName: types.NamespacedName{Namespace: "foo", Name: "lb-svc-no-class"}}, + }, + }, + { + name: "other_ProxyClass", + proxyClassName: otherPCName, + expected: []reconcile.Request{ + {NamespacedName: types.NamespacedName{Namespace: "default", Name: "exposed-other"}}, + {NamespacedName: types.NamespacedName{Namespace: "foo", Name: "lb-svc-other-pc"}}, + }, + }, + { + name: "unreferenced_ProxyClass", + proxyClassName: unreferencedPCName, + expected: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + reqs := mapFunc(t.Context(), &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.proxyClassName, + }, + }) + if len(reqs) != len(tc.expected) { + t.Fatalf("expected %d requests, got %d: %v", len(tc.expected), len(reqs), reqs) + } + for _, expected := range tc.expected { + if !slices.Contains(reqs, expected) { + t.Errorf("expected request for Service %q not found in results: %v", expected.Name, reqs) + } + } + }) + } +} diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 619aecc56816e..074d920940cf4 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -9,9 +9,12 @@ import ( "context" "encoding/json" "fmt" + "maps" "net/http" "net/netip" + "path" "reflect" + "slices" "strings" "sync" "testing" @@ -22,19 +25,19 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "tailscale.com/internal/client/tailscale" + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn" - "tailscale.com/ipn/ipnstate" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" - "tailscale.com/tailcfg" - "tailscale.com/types/ptr" "tailscale.com/util/mak" ) @@ -62,7 +65,6 @@ type configOpts struct { subnetRoutes string isExitNode bool isAppConnector bool - confFileHash string serveConfig *ipn.ServeConfig shouldEnableForwardingClusterTrafficViaIngress bool proxyClass string // configuration from the named ProxyClass should be applied to proxy resources @@ -70,9 +72,9 @@ type configOpts struct { shouldRemoveAuthKey bool secretExtraData map[string][]byte resourceVersion string - - enableMetrics bool - serviceMonitorLabels tsapi.Labels + replicas *int32 + enableMetrics bool + serviceMonitorLabels tsapi.Labels } func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { @@ -89,11 +91,19 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, - {Name: "TS_KUBE_SECRET", Value: opts.secretName}, - {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"}, + {Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"}, + {Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", Value: "false"}, + {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"}, + {Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"}, }, SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), + Privileged: new(true), + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("1Mi"), + }, }, ImagePullPolicy: "Always", } @@ -107,7 +117,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef var volumes []corev1.Volume volumes = []corev1.Volume{ { - Name: "tailscaledconfig", + Name: "tailscaledconfig-0", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: opts.secretName, @@ -116,13 +126,10 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef }, } tsContainer.VolumeMounts = []corev1.VolumeMount{{ - Name: "tailscaledconfig", + Name: "tailscaledconfig-0", ReadOnly: true, - MountPath: "/etc/tsconfig", + MountPath: "/etc/tsconfig/" + opts.secretName, }} - if opts.confFileHash != "" { - mak.Set(&annots, "tailscale.com/operator-last-set-config-file-hash", opts.confFileHash) - } if opts.firewallMode != "" { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_DEBUG_FIREWALL_MODE", @@ -158,10 +165,21 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef if opts.serveConfig != nil { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_SERVE_CONFIG", - Value: "/etc/tailscaled/serve-config", + Value: "/etc/tailscaled/$(POD_NAME)/serve-config", }) - volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}) - tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}) + volumes = append(volumes, corev1.Volume{ + Name: "serve-config-0", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: opts.secretName, + Items: []corev1.KeyToPath{{ + Key: "serve-config", + Path: "serve-config", + }}, + }, + }, + }) + tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)}) } tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_INTERNAL_APP", @@ -206,7 +224,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef }, }, Spec: appsv1.StatefulSetSpec{ - Replicas: ptr.To[int32](1), + Replicas: opts.replicas, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": "1234-UID"}, }, @@ -214,7 +232,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annots, - DeletionGracePeriodSeconds: ptr.To[int64](10), + DeletionGracePeriodSeconds: new(int64(10)), Labels: map[string]string{ "tailscale.com/managed": "true", "tailscale.com/parent-resource": "test", @@ -233,7 +251,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef Command: []string{"/bin/sh", "-c"}, Args: []string{"sysctl -w net.ipv4.ip_forward=1 && if sysctl net.ipv6.conf.all.forwarding; then sysctl -w net.ipv6.conf.all.forwarding=1; fi"}, SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), + Privileged: new(true), }, }, }, @@ -270,15 +288,23 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, - {Name: "TS_KUBE_SECRET", Value: opts.secretName}, - {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"}, - {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, + {Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"}, + {Name: "TS_EXPERIMENTAL_SERVICE_AUTO_ADVERTISEMENT", Value: "false"}, + {Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"}, + {Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"}, + {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"}, {Name: "TS_INTERNAL_APP", Value: opts.app}, }, ImagePullPolicy: "Always", VolumeMounts: []corev1.VolumeMount{ - {Name: "tailscaledconfig", ReadOnly: true, MountPath: "/etc/tsconfig"}, - {Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}, + {Name: "tailscaledconfig-0", ReadOnly: true, MountPath: path.Join("/etc/tsconfig", opts.secretName)}, + {Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("1Mi"), + }, }, } if opts.enableMetrics { @@ -306,16 +332,22 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps } volumes := []corev1.Volume{ { - Name: "tailscaledconfig", + Name: "tailscaledconfig-0", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: opts.secretName, }, }, }, - {Name: "serve-config", + { + Name: "serve-config-0", VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}, + Secret: &corev1.SecretVolumeSource{ + SecretName: opts.secretName, + Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}, + }, + }, + }, } ss := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ @@ -333,14 +365,14 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps }, }, Spec: appsv1.StatefulSetSpec{ - Replicas: ptr.To[int32](1), + Replicas: new(int32(1)), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": "1234-UID"}, }, ServiceName: opts.stsName, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - DeletionGracePeriodSeconds: ptr.To[int64](10), + DeletionGracePeriodSeconds: new(int64(10)), Labels: map[string]string{ "tailscale.com/managed": "true", "tailscale.com/parent-resource": "test", @@ -358,10 +390,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps }, }, } - ss.Spec.Template.Annotations = map[string]string{} - if opts.confFileHash != "" { - ss.Spec.Template.Annotations["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash - } // If opts.proxyClass is set, retrieve the ProxyClass and apply // configuration from that to the StatefulSet. if opts.proxyClass != "" { @@ -393,7 +421,7 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service { "app": "1234-UID", }, ClusterIP: "None", - IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicyPreferDualStack), + IPFamilyPolicy: new(corev1.IPFamilyPolicyPreferDualStack), }, } } @@ -453,7 +481,7 @@ func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstruc Namespace: opts.tailscaleNamespace, Labels: smLabels, ResourceVersion: opts.resourceVersion, - OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: ptr.To(true), Controller: ptr.To(true)}}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: new(true), Controller: new(true)}}, }, TypeMeta: metav1.TypeMeta{ Kind: "ServiceMonitor", @@ -502,7 +530,7 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec AcceptDNS: "false", Hostname: &opts.hostname, Locked: "false", - AuthKey: ptr.To("secret-authkey"), + AuthKey: new("new-authkey"), AcceptRoutes: "false", AppConnector: &ipn.AppConnectorPrefs{Advertise: false}, NoStatefulFiltering: "true", @@ -529,7 +557,7 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec if opts.isExitNode { r = "0.0.0.0/0,::/0," + r } - for _, rr := range strings.Split(r, ",") { + for rr := range strings.SplitSeq(r, ",") { prefix, err := netip.ParsePrefix(rr) if err != nil { t.Fatal(err) @@ -600,6 +628,32 @@ func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full return s.GetName(), strings.TrimSuffix(s.GetName(), "-0") } +func findGenNames(t *testing.T, cl client.Client, ns, name, typ string) []string { + t.Helper() + labels := map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentName: name, + LabelParentNamespace: ns, + LabelParentType: typ, + } + + var list corev1.SecretList + if err := cl.List(t.Context(), &list, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil { + t.Fatalf("finding secrets for %q: %v", name, err) + } + + if len(list.Items) == 0 { + t.Fatalf("no secrets found for %q %s %+#v", name, ns, labels) + } + + names := make([]string, len(list.Items)) + for i, secret := range list.Items { + names[i] = secret.GetName() + } + + return names +} + func mustCreate(t *testing.T, client client.Client, obj client.Object) { t.Helper() if err := client.Create(context.Background(), obj); err != nil { @@ -770,12 +824,9 @@ func expectEvents(t *testing.T, rec *record.FakeRecorder, wantsEvents []string) select { case gotEvent := <-rec.Events: found := false - for _, wantEvent := range wantsEvents { - if wantEvent == gotEvent { - found = true - seenEvents = append(seenEvents, gotEvent) - break - } + if slices.Contains(wantsEvents, gotEvent) { + found = true + seenEvents = append(seenEvents, gotEvent) } if !found { t.Errorf("got unexpected event %q, expected events: %+#v", gotEvent, wantsEvents) @@ -786,73 +837,139 @@ func expectEvents(t *testing.T, rec *record.FakeRecorder, wantsEvents []string) } } -type fakeTSClient struct { - sync.Mutex - keyRequests []tailscale.KeyCapabilities - deleted []string - vipServices map[tailcfg.ServiceName]*tailscale.VIPService -} -type fakeTSNetServer struct { - certDomains []string +type ( + fakeTSClient struct { + sync.Mutex + loginURL string + keyRequests []tailscale.KeyCapabilities + deleted []string + devices []tailscale.Device + vipServices map[string]tailscale.VIPService + } + + fakeVIPServices struct { + mu sync.RWMutex + vipServices map[string]tailscale.VIPService + } + + fakeKeys struct { + keyRequests *[]tailscale.KeyCapabilities + } + + fakeDevices struct { + deleted *[]string + devices *[]tailscale.Device + } +) + +func (c *fakeTSClient) VIPServices() tsclient.VIPServiceResource { + return &fakeVIPServices{ + vipServices: c.vipServices, + } } -func (f *fakeTSNetServer) CertDomains() []string { - return f.certDomains +func (m *fakeVIPServices) List(_ context.Context) ([]tailscale.VIPService, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + if len(m.vipServices) == 0 { + return nil, tailscale.APIError{Status: http.StatusNotFound} + } + + return slices.Collect(maps.Values(m.vipServices)), nil } -func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error) { - c.Lock() - defer c.Unlock() - c.keyRequests = append(c.keyRequests, caps) - k := &tailscale.Key{ - ID: "key", - Created: time.Now(), - Capabilities: caps, +func (m *fakeVIPServices) Delete(_ context.Context, name string) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.vipServices[name]; !ok { + return tailscale.APIError{Status: http.StatusNotFound} } - return "secret-authkey", k, nil + + delete(m.vipServices, name) + return nil } -func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) { - return &tailscale.Device{ - DeviceID: deviceID, - Hostname: "hostname-" + deviceID, - Addresses: []string{ - "1.2.3.4", - "::1", - }, - }, nil +func (m *fakeVIPServices) Get(_ context.Context, name string) (*tailscale.VIPService, error) { + if svc, ok := m.vipServices[name]; ok { + return &svc, nil + } + + return nil, tailscale.APIError{Status: http.StatusNotFound} } -func (c *fakeTSClient) DeleteDevice(ctx context.Context, deviceID string) error { - c.Lock() - defer c.Unlock() - c.deleted = append(c.deleted, deviceID) +func (m *fakeVIPServices) CreateOrUpdate(_ context.Context, svc tailscale.VIPService) error { + m.mu.Lock() + defer m.mu.Unlock() + + if svc.Addrs == nil { + svc.Addrs = []string{vipTestIP} + } + + m.vipServices[svc.Name] = svc return nil } -func (c *fakeTSClient) KeyRequests() []tailscale.KeyCapabilities { - c.Lock() - defer c.Unlock() - return c.keyRequests +func (c *fakeTSClient) Devices() tsclient.DeviceResource { + return &fakeDevices{ + deleted: &c.deleted, + devices: &c.devices, + } } -func (c *fakeTSClient) Deleted() []string { - c.Lock() - defer c.Unlock() - return c.deleted +func (m *fakeDevices) Delete(_ context.Context, id string) error { + *m.deleted = append(*m.deleted, id) + + return tailscale.APIError{Status: http.StatusNotFound} +} + +func (m *fakeDevices) List(_ context.Context, _ ...tailscale.ListDevicesOptions) ([]tailscale.Device, error) { + return *m.devices, nil +} + +func (m *fakeDevices) Get(_ context.Context, id string) (*tailscale.Device, error) { + if m.devices == nil { + return nil, tailscale.APIError{Status: http.StatusNotFound} + } + + for _, dev := range *m.devices { + if dev.ID == id { + return &dev, nil + } + } + + return nil, tailscale.APIError{Status: http.StatusNotFound} } -// removeHashAnnotation can be used to remove declarative tailscaled config hash -// annotation from proxy StatefulSets to make the tests more maintainable (so -// that we don't have to change the annotation in each test case after any -// change to the configfile contents). -func removeHashAnnotation(sts *appsv1.StatefulSet) { - delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash) - if len(sts.Spec.Template.Annotations) == 0 { - sts.Spec.Template.Annotations = nil +func (c *fakeTSClient) Keys() tsclient.KeyResource { + return &fakeKeys{ + keyRequests: &c.keyRequests, } } +func (m *fakeKeys) CreateAuthKey(_ context.Context, ckr tailscale.CreateKeyRequest) (*tailscale.Key, error) { + *m.keyRequests = append(*m.keyRequests, ckr.Capabilities) + + return &tailscale.Key{Key: "new-authkey"}, nil +} + +func (m *fakeKeys) List(_ context.Context, _ bool) ([]tailscale.Key, error) { + return nil, nil +} + +func (c *fakeTSClient) LoginURL() string { + return c.loginURL +} + +type fakeTSNetServer struct { + certDomains []string +} + +func (f *fakeTSNetServer) CertDomains() []string { + return f.certDomains +} + func removeResourceReqs(sts *appsv1.StatefulSet) { if sts != nil { sts.Spec.Template.Spec.Resources = nil @@ -896,64 +1013,3 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) { } } } - -func (c *fakeTSClient) GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error) { - c.Lock() - defer c.Unlock() - if c.vipServices == nil { - return nil, tailscale.ErrResponse{Status: http.StatusNotFound} - } - svc, ok := c.vipServices[name] - if !ok { - return nil, tailscale.ErrResponse{Status: http.StatusNotFound} - } - return svc, nil -} - -func (c *fakeTSClient) ListVIPServices(ctx context.Context) (map[tailcfg.ServiceName]*tailscale.VIPService, error) { - c.Lock() - defer c.Unlock() - if c.vipServices == nil { - return nil, &tailscale.ErrResponse{Status: http.StatusNotFound} - } - return c.vipServices, nil -} - -func (c *fakeTSClient) CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error { - c.Lock() - defer c.Unlock() - if c.vipServices == nil { - c.vipServices = make(map[tailcfg.ServiceName]*tailscale.VIPService) - } - - if svc.Addrs == nil { - svc.Addrs = []string{vipTestIP} - } - - c.vipServices[svc.Name] = svc - return nil -} - -func (c *fakeTSClient) DeleteVIPService(ctx context.Context, name tailcfg.ServiceName) error { - c.Lock() - defer c.Unlock() - if c.vipServices != nil { - delete(c.vipServices, name) - } - return nil -} - -type fakeLocalClient struct { - status *ipnstate.Status -} - -func (f *fakeLocalClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { - if f.status == nil { - return &ipnstate.Status{ - Self: &ipnstate.PeerStatus{ - DNSName: "test-node.test.ts.net.", - }, - }, nil - } - return f.status, nil -} diff --git a/cmd/k8s-operator/tsclient.go b/cmd/k8s-operator/tsclient.go index f49f84af96ed4..0670d2bcf94a0 100644 --- a/cmd/k8s-operator/tsclient.go +++ b/cmd/k8s-operator/tsclient.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -6,50 +6,65 @@ package main import ( - "context" "fmt" + "net/url" "os" - "golang.org/x/oauth2/clientcredentials" - "tailscale.com/internal/client/tailscale" - "tailscale.com/tailcfg" + "go.uber.org/zap" + "tailscale.com/client/tailscale/v2" + + "tailscale.com/ipn" ) -// defaultTailnet is a value that can be used in Tailscale API calls instead of tailnet name to indicate that the API -// call should be performed on the default tailnet for the provided credentials. const ( - defaultTailnet = "-" - defaultBaseURL = "https://api.tailscale.com" + oidcJWTPath = "/var/run/secrets/tailscale/serviceaccount/token" ) -func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (tsClient, error) { - clientID, err := os.ReadFile(clientIDPath) - if err != nil { - return nil, fmt.Errorf("error reading client ID %q: %w", clientIDPath, err) +func newTSClient(logger *zap.SugaredLogger, clientID, clientIDPath, clientSecretPath, loginServer string) (*tailscale.Client, error) { + baseURL := ipn.DefaultControlURL + if loginServer != "" { + baseURL = loginServer } - clientSecret, err := os.ReadFile(clientSecretPath) + + base, err := url.Parse(baseURL) if err != nil { - return nil, fmt.Errorf("reading client secret %q: %w", clientSecretPath, err) + return nil, err } - credentials := clientcredentials.Config{ - ClientID: string(clientID), - ClientSecret: string(clientSecret), - TokenURL: "https://login.tailscale.com/api/v2/oauth/token", + + client := &tailscale.Client{ + UserAgent: "tailscale-k8s-operator", + BaseURL: base, + } + + if clientID == "" { + // Use static client credentials mounted to disk. + clientIDBytes, err := os.ReadFile(clientIDPath) + if err != nil { + return nil, fmt.Errorf("error reading client ID %q: %w", clientIDPath, err) + } + clientSecretBytes, err := os.ReadFile(clientSecretPath) + if err != nil { + return nil, fmt.Errorf("reading client secret %q: %w", clientSecretPath, err) + } + + client.Auth = &tailscale.OAuth{ + ClientID: string(clientIDBytes), + ClientSecret: string(clientSecretBytes), + } + } else { + // Use workload identity federation. + client.Auth = &tailscale.IdentityFederation{ + ClientID: clientID, + IDTokenFunc: func() (string, error) { + token, err := os.ReadFile(oidcJWTPath) + if err != nil { + return "", err + } + + return string(token), nil + }, + } } - c := tailscale.NewClient(defaultTailnet, nil) - c.UserAgent = "tailscale-k8s-operator" - c.HTTPClient = credentials.Client(ctx) - return c, nil -} -type tsClient interface { - CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error) - Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) - DeleteDevice(ctx context.Context, nodeStableID string) error - // GetVIPService is a method for getting a Tailscale Service. VIPService is the original name for Tailscale Service. - GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error) - // CreateOrUpdateVIPService is a method for creating or updating a Tailscale Service. - CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error - // DeleteVIPService is a method for deleting a Tailscale Service. - DeleteVIPService(ctx context.Context, name tailcfg.ServiceName) error + return client, nil } diff --git a/cmd/k8s-operator/tsrecorder.go b/cmd/k8s-operator/tsrecorder.go index 081543cd384db..86669d212e738 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/cmd/k8s-operator/tsrecorder.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -10,13 +10,15 @@ import ( "encoding/json" "errors" "fmt" - "net/http" "slices" + "strconv" "strings" "sync" + "time" "go.uber.org/zap" xslices "golang.org/x/exp/slices" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -29,9 +31,11 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "tailscale.com/client/tailscale" + "tailscale.com/client/tailscale/v2" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstime" @@ -40,10 +44,11 @@ import ( ) const ( - reasonRecorderCreationFailed = "RecorderCreationFailed" - reasonRecorderCreating = "RecorderCreating" - reasonRecorderCreated = "RecorderCreated" - reasonRecorderInvalid = "RecorderInvalid" + reasonRecorderCreationFailed = "RecorderCreationFailed" + reasonRecorderCreating = "RecorderCreating" + reasonRecorderCreated = "RecorderCreated" + reasonRecorderInvalid = "RecorderInvalid" + reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable" currentProfileKey = "_current-profile" ) @@ -54,33 +59,53 @@ var gaugeRecorderResources = clientmetric.NewGauge(kubetypes.MetricRecorderCount // Recorder CRs. type RecorderReconciler struct { client.Client - l *zap.SugaredLogger - recorder record.EventRecorder - clock tstime.Clock - tsNamespace string - tsClient tsClient - - mu sync.Mutex // protects following - recorders set.Slice[types.UID] // for recorders gauge + log *zap.SugaredLogger + recorder record.EventRecorder + clock tstime.Clock + clients ClientProvider + tsNamespace string + authKeyRateLimits map[string]*rate.Limiter // per-Recorder rate limiters for auth key re-issuance. + authKeyReissuing map[string]bool + mu sync.Mutex // protects following + recorders set.Slice[types.UID] // for recorders gauge } func (r *RecorderReconciler) logger(name string) *zap.SugaredLogger { - return r.l.With("Recorder", name) + return r.log.With("Recorder", name) } -func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { +func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { logger := r.logger(req.Name) logger.Debugf("starting reconcile") defer logger.Debugf("reconcile finished") tsr := new(tsapi.Recorder) - err = r.Get(ctx, req.NamespacedName, tsr) + err := r.Get(ctx, req.NamespacedName, tsr) if apierrors.IsNotFound(err) { logger.Debugf("Recorder not found, assuming it was deleted") return reconcile.Result{}, nil } else if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err) } + + oldTSRStatus := tsr.Status.DeepCopy() + setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { + tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger) + if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) { + // An error encountered here should get returned by the Reconcile function. + if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil { + return reconcile.Result{}, errors.Join(err, updateErr) + } + } + + return reconcile.Result{}, nil + } + + tsClient, err := r.clients.For(tsr.Spec.Tailnet) + if err != nil { + return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderTailnetUnavailable, err.Error()) + } + if markedForDeletion(tsr) { logger.Debugf("Recorder is being deleted, cleaning up resources") ix := xslices.Index(tsr.Finalizers, FinalizerName) @@ -89,7 +114,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return reconcile.Result{}, nil } - if done, err := r.maybeCleanup(ctx, tsr); err != nil { + if done, err := r.maybeCleanup(ctx, tsr, tsClient); err != nil { return reconcile.Result{}, err } else if !done { logger.Debugf("Recorder resource cleanup not yet finished, will retry...") @@ -97,24 +122,12 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques } tsr.Finalizers = slices.Delete(tsr.Finalizers, ix, ix+1) - if err := r.Update(ctx, tsr); err != nil { + if err = r.Update(ctx, tsr); err != nil { return reconcile.Result{}, err } return reconcile.Result{}, nil } - oldTSRStatus := tsr.Status.DeepCopy() - setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger) - if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) { - // An error encountered here should get returned by the Reconcile function. - if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil { - err = errors.Join(err, updateErr) - } - } - return reconcile.Result{}, err - } - if !slices.Contains(tsr.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, @@ -122,18 +135,18 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques // operation is underway. logger.Infof("ensuring Recorder is set up") tsr.Finalizers = append(tsr.Finalizers, FinalizerName) - if err := r.Update(ctx, tsr); err != nil { + if err = r.Update(ctx, tsr); err != nil { return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed) } } - if err := r.validate(ctx, tsr); err != nil { + if err = r.validate(ctx, tsr); err != nil { message := fmt.Sprintf("Recorder is invalid: %s", err) r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderInvalid, message) return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message) } - if err = r.maybeProvision(ctx, tsr); err != nil { + if err = r.maybeProvision(ctx, tsClient, tsr); err != nil { reason := reasonRecorderCreationFailed message := fmt.Sprintf("failed creating Recorder: %s", err) if strings.Contains(err.Error(), optimisticLockErrorMsg) { @@ -151,27 +164,46 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated) } -func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Recorder) error { +func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { logger := r.logger(tsr.Name) + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas + } + r.mu.Lock() r.recorders.Add(tsr.UID) gaugeRecorderResources.Set(int64(r.recorders.Len())) + if _, ok := r.authKeyRateLimits[tsr.Name]; !ok { + r.authKeyRateLimits[tsr.Name] = rate.NewLimiter(rate.Every(30*time.Second), int(replicas)) + } + for replica := range replicas { + name := fmt.Sprintf("%s-%d", tsr.Name, replica) + if _, ok := r.authKeyReissuing[name]; !ok { + r.authKeyReissuing[name] = false + } + } r.mu.Unlock() - if err := r.ensureAuthSecretCreated(ctx, tsr); err != nil { + if err := r.ensureAuthSecretsCreated(ctx, tsClient, tsr); err != nil { return fmt.Errorf("error creating secrets: %w", err) } - // State Secret is precreated so we can use the Recorder CR as its owner ref. - sec := tsrStateSecret(tsr, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) { - s.ObjectMeta.Labels = sec.ObjectMeta.Labels - s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations - }); err != nil { - return fmt.Errorf("error creating state Secret: %w", err) + + // State Secrets are pre-created so we can use the Recorder CR as its owner ref. + for replica := range replicas { + sec := tsrStateSecret(tsr, r.tsNamespace, replica) + _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sec, func(s *corev1.Secret) { + s.ObjectMeta.Labels = sec.ObjectMeta.Labels + s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations + }) + if err != nil { + return fmt.Errorf("error creating state Secret %q: %w", sec.Name, err) + } } + sa := tsrServiceAccount(tsr, r.tsNamespace) - if _, err := createOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error { + _, err := createOrMaybeUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) error { // Perform this check within the update function to make sure we don't // have a race condition between the previous check and the update. if err := saOwnedByRecorder(s, tsr); err != nil { @@ -182,54 +214,68 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations return nil - }); err != nil { + }) + if err != nil { return fmt.Errorf("error creating ServiceAccount: %w", err) } + role := tsrRole(tsr, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels r.ObjectMeta.Annotations = role.ObjectMeta.Annotations r.Rules = role.Rules - }); err != nil { + }) + if err != nil { return fmt.Errorf("error creating Role: %w", err) } + roleBinding := tsrRoleBinding(tsr, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { + _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels r.ObjectMeta.Annotations = roleBinding.ObjectMeta.Annotations r.RoleRef = roleBinding.RoleRef r.Subjects = roleBinding.Subjects - }); err != nil { + }) + if err != nil { return fmt.Errorf("error creating RoleBinding: %w", err) } - ss := tsrStatefulSet(tsr, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { + + ss := tsrStatefulSet(tsr, r.tsNamespace, tsClient.LoginURL()) + _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.Spec = ss.Spec - }); err != nil { + }) + if err != nil { return fmt.Errorf("error creating StatefulSet: %w", err) } // ServiceAccount name may have changed, in which case we need to clean up // the previous ServiceAccount. RoleBinding will already be updated to point // to the new ServiceAccount. - if err := r.maybeCleanupServiceAccounts(ctx, tsr, sa.Name); err != nil { + if err = r.maybeCleanupServiceAccounts(ctx, tsr, sa.Name); err != nil { return fmt.Errorf("error cleaning up ServiceAccounts: %w", err) } + // If we have scaled the recorder down, we will have dangling state secrets + // that we need to clean up. + if err = r.maybeCleanupSecrets(ctx, tsClient, tsr); err != nil { + return fmt.Errorf("error cleaning up Secrets: %w", err) + } + var devices []tsapi.RecorderTailnetDevice + for replica := range replicas { + dev, ok, err := r.getDeviceInfo(ctx, tsClient, tsr.Name, replica) + switch { + case err != nil: + return fmt.Errorf("failed to get device info: %w", err) + case !ok: + logger.Debugf("no Tailscale hostname known yet, waiting for Recorder pod to finish auth") + continue + } - device, ok, err := r.getDeviceInfo(ctx, tsr.Name) - if err != nil { - return fmt.Errorf("failed to get device info: %w", err) + devices = append(devices, dev) } - if !ok { - logger.Debugf("no Tailscale hostname known yet, waiting for Recorder pod to finish auth") - return nil - } - - devices = append(devices, device) tsr.Status.Devices = devices @@ -256,22 +302,87 @@ func saOwnedByRecorder(sa *corev1.ServiceAccount, tsr *tsapi.Recorder) error { func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, tsr *tsapi.Recorder, currentName string) error { logger := r.logger(tsr.Name) - // List all ServiceAccounts owned by this Recorder. + options := []client.ListOption{ + client.InNamespace(r.tsNamespace), + client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)), + } + sas := &corev1.ServiceAccountList{} - if err := r.List(ctx, sas, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels("recorder", tsr.Name, nil))); err != nil { + if err := r.List(ctx, sas, options...); err != nil { return fmt.Errorf("error listing ServiceAccounts for cleanup: %w", err) } - for _, sa := range sas.Items { - if sa.Name == currentName { + + for _, serviceAccount := range sas.Items { + if serviceAccount.Name == currentName { + continue + } + + err := r.Delete(ctx, &serviceAccount) + switch { + case apierrors.IsNotFound(err): + logger.Debugf("ServiceAccount %s not found, likely already deleted", serviceAccount.Name) continue + case err != nil: + return fmt.Errorf("error deleting ServiceAccount %s: %w", serviceAccount.Name, err) + } + } + + return nil +} + +func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { + options := []client.ListOption{ + client.InNamespace(r.tsNamespace), + client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)), + } + + secrets := &corev1.SecretList{} + if err := r.List(ctx, secrets, options...); err != nil { + return fmt.Errorf("error listing Secrets for cleanup: %w", err) + } + + // Get the largest ordinal suffix that we expect. Then we'll go through the list of secrets owned by this + // recorder and remove them. + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas + } + + for _, secret := range secrets.Items { + parts := strings.Split(secret.Name, "-") + if len(parts) == 0 { + continue + } + + ordinal, err := strconv.ParseUint(parts[len(parts)-1], 10, 32) + if err != nil { + return fmt.Errorf("error parsing secret name %q: %w", secret.Name, err) + } + + if int32(ordinal) < replicas { + continue + } + + devicePrefs, ok, err := getDevicePrefs(&secret) + if err != nil { + return err } - if err := r.Delete(ctx, &sa); err != nil { - if apierrors.IsNotFound(err) { - logger.Debugf("ServiceAccount %s not found, likely already deleted", sa.Name) - } else { - return fmt.Errorf("error deleting ServiceAccount %s: %w", sa.Name, err) + + if ok { + r.log.Debugf("deleting device %s", devicePrefs.Config.NodeID) + err = tsClient.Devices().Delete(ctx, string(devicePrefs.Config.NodeID)) + switch { + case tailscale.IsNotFound(err): + // This device has possibly already been deleted in the admin console. So we can ignore this + // and move on to removing the secret. + case err != nil: + return err } } + + if err = r.Delete(ctx, &secret); err != nil { + return err + } } return nil @@ -280,33 +391,40 @@ func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, ts // maybeCleanup just deletes the device from the tailnet. All the kubernetes // resources linked to a Recorder will get cleaned up via owner references // (which we can use because they are all in the same namespace). -func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) { +func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder, tsClient tsclient.Client) (bool, error) { logger := r.logger(tsr.Name) - prefs, ok, err := r.getDevicePrefs(ctx, tsr.Name) - if err != nil { - return false, err - } - if !ok { - logger.Debugf("state Secret %s-0 not found or does not contain node ID, continuing cleanup", tsr.Name) - r.mu.Lock() - r.recorders.Remove(tsr.UID) - gaugeRecorderResources.Set(int64(r.recorders.Len())) - r.mu.Unlock() - return true, nil + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas } - id := string(prefs.Config.NodeID) - logger.Debugf("deleting device %s from control", string(id)) - if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil { - errResp := &tailscale.ErrResponse{} - if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound { - logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id)) - } else { + for replica := range replicas { + devicePrefs, ok, err := r.getDevicePrefs(ctx, tsr.Name, replica) + if err != nil { + return false, err + } + if !ok { + logger.Debugf("state Secret %s-%d not found or does not contain node ID, continuing cleanup", tsr.Name, replica) + r.mu.Lock() + r.recorders.Remove(tsr.UID) + gaugeRecorderResources.Set(int64(r.recorders.Len())) + r.mu.Unlock() + return true, nil + } + + nodeID := string(devicePrefs.Config.NodeID) + logger.Debugf("deleting device %s from control", nodeID) + err = tsClient.Devices().Delete(ctx, nodeID) + switch { + case tailscale.IsNotFound(err): + logger.Debugf("device %s not found, likely because it has already been deleted from control", nodeID) + continue + case err != nil: return false, fmt.Errorf("error deleting device: %w", err) } - } else { - logger.Debugf("device %s deleted from control", string(id)) + + logger.Debugf("device %s deleted from control", nodeID) } // Unlike most log entries in the reconcile loop, this will get printed @@ -317,41 +435,147 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record r.mu.Lock() r.recorders.Remove(tsr.UID) gaugeRecorderResources.Set(int64(r.recorders.Len())) + delete(r.authKeyRateLimits, tsr.Name) + for replica := range replicas { + delete(r.authKeyReissuing, fmt.Sprintf("%s-%d", tsr.Name, replica)) + } r.mu.Unlock() + return true, nil } -func (r *RecorderReconciler) ensureAuthSecretCreated(ctx context.Context, tsr *tsapi.Recorder) error { - logger := r.logger(tsr.Name) - key := types.NamespacedName{ - Namespace: r.tsNamespace, - Name: tsr.Name, - } - if err := r.Get(ctx, key, &corev1.Secret{}); err == nil { - // No updates, already created the auth key. - logger.Debugf("auth Secret %s already exists", key.Name) - return nil - } else if !apierrors.IsNotFound(err) { - return err +func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder) error { + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas } - // Create the auth key Secret which is going to be used by the StatefulSet - // to authenticate with Tailscale. - logger.Debugf("creating authkey for new Recorder") tags := tsr.Spec.Tags if len(tags) == 0 { tags = tsapi.Tags{"tag:k8s"} } - authKey, err := newAuthKey(ctx, r.tsClient, tags.Stringify()) - if err != nil { - return err + + logger := r.logger(tsr.Name) + + for replica := range replicas { + key := types.NamespacedName{ + Namespace: r.tsNamespace, + Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), + } + + existingSecret := &corev1.Secret{} + err := r.Get(ctx, key, existingSecret) + switch { + case err == nil: + reissue, err := r.shouldReissueAuthKey(ctx, tsClient, tsr, replica, existingSecret) + if err != nil { + return fmt.Errorf("error checking auth key reissue for replica %d: %w", replica, err) + } + if !reissue { + logger.Debugf("auth Secret %q already exists, no reissue needed", key.Name) + continue + } + authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) + if err != nil { + return err + } + existingSecret.Data["authkey"] = []byte(authKey) + if err = r.Update(ctx, existingSecret); err != nil { + return err + } + continue + case apierrors.IsNotFound(err): + authKey, err := newAuthKey(ctx, tsClient, tags.Stringify()) + if err != nil { + return err + } + if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey, replica)); err != nil { + return err + } + default: + return fmt.Errorf("failed to get Secret %q: %w", key.Name, err) + } } - logger.Debug("creating a new Secret for the Recorder") - if err := r.Create(ctx, tsrAuthSecret(tsr, r.tsNamespace, authKey)); err != nil { - return err + return nil +} + +// shouldReissueAuthKey returns true if the proxy needs a new auth key. It +// tracks in-flight reissues via authKeyReissuing to avoid duplicate API calls +// across reconciles. +func (r *RecorderReconciler) shouldReissueAuthKey(ctx context.Context, tsClient tsclient.Client, tsr *tsapi.Recorder, replica int32, authSecret *corev1.Secret) (shouldReissue bool, err error) { + stateSecret, err := r.getStateSecret(ctx, tsr.Name, replica) + if err != nil || stateSecret == nil { + return false, err } + stateSecretName := fmt.Sprintf("%s-%d", tsr.Name, replica) + + r.mu.Lock() + reissuing := r.authKeyReissuing[stateSecretName] + r.mu.Unlock() + + if reissuing { + _, requestStillPresent := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !requestStillPresent { + r.mu.Lock() + r.authKeyReissuing[stateSecretName] = false + r.mu.Unlock() + r.log.Debugf("auth key reissue completed for %q", stateSecretName) + return false, nil + } + r.log.Debugf("auth key already in process of re-issuance for %q, waiting", stateSecretName) + return false, nil + } + + defer func() { + r.mu.Lock() + r.authKeyReissuing[stateSecretName] = shouldReissue + r.mu.Unlock() + }() + + brokenAuthkey, ok := stateSecret.Data[kubetypes.KeyReissueAuthkey] + if !ok { + return false, nil + } + + cfgAuthKey := string(authSecret.Data["authkey"]) + empty := cfgAuthKey == "" + broken := cfgAuthKey == string(brokenAuthkey) + + if !empty && !broken { + return false, nil + } + + lim := r.authKeyRateLimits[tsr.Name] + if !lim.Allow() { + r.log.Debugf("auth key re-issuance rate limit exceeded, limit: %.2f, burst: %d, tokens: %.2f", + lim.Limit(), lim.Burst(), lim.Tokens()) + return false, fmt.Errorf("auth key re-issuance rate limit exceeded for Recorder %q, will retry with backoff", tsr.Name) + } + + r.log.Infof("Recorder replica %s failing to auth; attempting cleanup and new key", stateSecretName) + if tsID := stateSecret.Data[kubetypes.KeyDeviceID]; len(tsID) > 0 { + id := tailcfg.StableNodeID(tsID) + if err := r.ensureDeviceDeleted(ctx, tsClient, id, r.log); err != nil { + return false, err + } + } + + return true, nil +} + +func (r *RecorderReconciler) ensureDeviceDeleted(ctx context.Context, tsClient tsclient.Client, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { + logger.Debugf("deleting device %s from control", string(id)) + err := tsClient.Devices().Delete(ctx, string(id)) + switch { + case tailscale.IsNotFound(err): + logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id)) + case err != nil: + return fmt.Errorf("error deleting device: %w", err) + default: + logger.Debugf("device %s deleted from control", string(id)) + } return nil } @@ -360,6 +584,10 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) return errors.New("must either enable UI or use S3 storage to ensure recordings are accessible") } + if tsr.Spec.Replicas != nil && *tsr.Spec.Replicas > 1 && tsr.Spec.Storage.S3 == nil { + return errors.New("must use S3 storage when using multiple replicas to ensure recordings are accessible") + } + // Check any custom ServiceAccount config doesn't conflict with pre-existing // ServiceAccounts. This check is performed once during validation to ensure // errors are raised early, but also again during any Updates to prevent a race. @@ -393,11 +621,11 @@ func (r *RecorderReconciler) validate(ctx context.Context, tsr *tsapi.Recorder) return nil } -func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string) (*corev1.Secret, error) { +func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string, replica int32) (*corev1.Secret, error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: r.tsNamespace, - Name: fmt.Sprintf("%s-0", tsrName), + Name: fmt.Sprintf("%s-%d", tsrName, replica), }, } if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { @@ -411,8 +639,8 @@ func (r *RecorderReconciler) getStateSecret(ctx context.Context, tsrName string) return secret, nil } -func (r *RecorderReconciler) getDevicePrefs(ctx context.Context, tsrName string) (prefs prefs, ok bool, err error) { - secret, err := r.getStateSecret(ctx, tsrName) +func (r *RecorderReconciler) getDevicePrefs(ctx context.Context, tsrName string, replica int32) (prefs prefs, ok bool, err error) { + secret, err := r.getStateSecret(ctx, tsrName, replica) if err != nil || secret == nil { return prefs, false, err } @@ -440,24 +668,21 @@ func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) { return prefs, ok, nil } -func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) (d tsapi.RecorderTailnetDevice, ok bool, err error) { - secret, err := r.getStateSecret(ctx, tsrName) +func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsClient tsclient.Client, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) { + secret, err := r.getStateSecret(ctx, tsrName, replica) if err != nil || secret == nil { return tsapi.RecorderTailnetDevice{}, false, err } - return getDeviceInfo(ctx, r.tsClient, secret) -} - -func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) { prefs, ok, err := getDevicePrefs(secret) if !ok || err != nil { return tsapi.RecorderTailnetDevice{}, false, err } // TODO(tomhjp): The profile info doesn't include addresses, which is why we - // need the API. Should we instead update the profile to include addresses? - device, err := tsClient.Device(ctx, string(prefs.Config.NodeID), nil) + // need the API. Should maybe update tsrecorder to write IPs to the state + // Secret like containerboot does. + device, err := tsClient.Devices().Get(ctx, string(prefs.Config.NodeID)) if err != nil { return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err) } diff --git a/cmd/k8s-operator/tsrecorder_specs.go b/cmd/k8s-operator/tsrecorder_specs.go index 7c6e80aed56fd..5a93bc22b546c 100644 --- a/cmd/k8s-operator/tsrecorder_specs.go +++ b/cmd/k8s-operator/tsrecorder_specs.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -7,35 +7,41 @@ package main import ( "fmt" + "maps" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/types/ptr" "tailscale.com/version" ) -func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet { - return &appsv1.StatefulSet{ +func tsrStatefulSet(tsr *tsapi.Recorder, namespace string, loginServer string) *appsv1.StatefulSet { + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas + } + + ss := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: tsr.Name, Namespace: namespace, - Labels: labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Labels), + Labels: tsrLabels("recorder", tsr.Name, tsr.Spec.StatefulSet.Labels), OwnerReferences: tsrOwnerReference(tsr), Annotations: tsr.Spec.StatefulSet.Annotations, }, Spec: appsv1.StatefulSetSpec{ - Replicas: ptr.To[int32](1), + Replicas: new(replicas), Selector: &metav1.LabelSelector{ - MatchLabels: labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels), + MatchLabels: tsrLabels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels), }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: tsr.Name, Namespace: namespace, - Labels: labels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels), + Labels: tsrLabels("recorder", tsr.Name, tsr.Spec.StatefulSet.Pod.Labels), Annotations: tsr.Spec.StatefulSet.Pod.Annotations, }, Spec: corev1.PodSpec{ @@ -59,7 +65,7 @@ func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet { ImagePullPolicy: tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy, Resources: tsr.Spec.StatefulSet.Pod.Container.Resources, SecurityContext: tsr.Spec.StatefulSet.Pod.Container.SecurityContext, - Env: env(tsr), + Env: tsrEnv(tsr, loginServer), EnvFrom: func() []corev1.EnvFromSource { if tsr.Spec.Storage.S3 == nil || tsr.Spec.Storage.S3.Credentials.Secret.Name == "" { return nil @@ -95,6 +101,28 @@ func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet { }, }, } + + for replica := range replicas { + volumeName := fmt.Sprintf("authkey-%d", replica) + + ss.Spec.Template.Spec.Containers[0].VolumeMounts = append(ss.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: volumeName, + ReadOnly: true, + MountPath: fmt.Sprintf("/etc/tailscaled/%s-%d", ss.Name, replica), + }) + + ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), + Items: []corev1.KeyToPath{{Key: "authkey", Path: "authkey"}}, + }, + }, + }) + } + + return ss } func tsrServiceAccount(tsr *tsapi.Recorder, namespace string) *corev1.ServiceAccount { @@ -102,7 +130,7 @@ func tsrServiceAccount(tsr *tsapi.Recorder, namespace string) *corev1.ServiceAcc ObjectMeta: metav1.ObjectMeta{ Name: tsrServiceAccountName(tsr), Namespace: namespace, - Labels: labels("recorder", tsr.Name, nil), + Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), Annotations: tsr.Spec.StatefulSet.Pod.ServiceAccount.Annotations, }, @@ -120,11 +148,24 @@ func tsrServiceAccountName(tsr *tsapi.Recorder) string { } func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role { + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas + } + + resourceNames := make([]string, 0) + for replica := range replicas { + resourceNames = append(resourceNames, + fmt.Sprintf("%s-%d", tsr.Name, replica), // State secret. + fmt.Sprintf("%s-auth-%d", tsr.Name, replica), // Auth key secret. + ) + } + return &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: tsr.Name, Namespace: namespace, - Labels: labels("recorder", tsr.Name, nil), + Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), }, Rules: []rbacv1.PolicyRule{ @@ -136,10 +177,7 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role { "patch", "update", }, - ResourceNames: []string{ - tsr.Name, // Contains the auth key. - fmt.Sprintf("%s-0", tsr.Name), // Contains the node state. - }, + ResourceNames: resourceNames, }, { APIGroups: []string{""}, @@ -159,7 +197,7 @@ func tsrRoleBinding(tsr *tsapi.Recorder, namespace string) *rbacv1.RoleBinding { ObjectMeta: metav1.ObjectMeta{ Name: tsr.Name, Namespace: namespace, - Labels: labels("recorder", tsr.Name, nil), + Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), }, Subjects: []rbacv1.Subject{ @@ -176,12 +214,12 @@ func tsrRoleBinding(tsr *tsapi.Recorder, namespace string) *rbacv1.RoleBinding { } } -func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string) *corev1.Secret { +func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string, replica int32) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, - Name: tsr.Name, - Labels: labels("recorder", tsr.Name, nil), + Name: fmt.Sprintf("%s-auth-%d", tsr.Name, replica), + Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), }, StringData: map[string]string{ @@ -190,30 +228,19 @@ func tsrAuthSecret(tsr *tsapi.Recorder, namespace string, authKey string) *corev } } -func tsrStateSecret(tsr *tsapi.Recorder, namespace string) *corev1.Secret { +func tsrStateSecret(tsr *tsapi.Recorder, namespace string, replica int32) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-0", tsr.Name), + Name: fmt.Sprintf("%s-%d", tsr.Name, replica), Namespace: namespace, - Labels: labels("recorder", tsr.Name, nil), + Labels: tsrLabels("recorder", tsr.Name, nil), OwnerReferences: tsrOwnerReference(tsr), }, } } -func env(tsr *tsapi.Recorder) []corev1.EnvVar { +func tsrEnv(tsr *tsapi.Recorder, loginServer string) []corev1.EnvVar { envs := []corev1.EnvVar{ - { - Name: "TS_AUTHKEY", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: tsr.Name, - }, - Key: "authkey", - }, - }, - }, { Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ @@ -231,6 +258,10 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar { }, }, }, + { + Name: "TS_AUTHKEY_FILE", + Value: "/etc/tailscaled/$(POD_NAME)/authkey", + }, { Name: "TS_STATE", Value: "kube:$(POD_NAME)", @@ -239,6 +270,10 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar { Name: "TSRECORDER_HOSTNAME", Value: "$(POD_NAME)", }, + { + Name: "TSRECORDER_LOGIN_SERVER", + Value: loginServer, + }, } for _, env := range tsr.Spec.StatefulSet.Pod.Container.Env { @@ -276,18 +311,16 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar { return envs } -func labels(app, instance string, customLabels map[string]string) map[string]string { - l := make(map[string]string, len(customLabels)+3) - for k, v := range customLabels { - l[k] = v - } +func tsrLabels(app, instance string, customLabels map[string]string) map[string]string { + labels := make(map[string]string, len(customLabels)+3) + maps.Copy(labels, customLabels) // ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ - l["app.kubernetes.io/name"] = app - l["app.kubernetes.io/instance"] = instance - l["app.kubernetes.io/managed-by"] = "tailscale-operator" + labels["app.kubernetes.io/name"] = app + labels["app.kubernetes.io/instance"] = instance + labels["app.kubernetes.io/managed-by"] = "tailscale-operator" - return l + return labels } func tsrOwnerReference(owner metav1.Object) []metav1.OwnerReference { diff --git a/cmd/k8s-operator/tsrecorder_specs_test.go b/cmd/k8s-operator/tsrecorder_specs_test.go index 94a8a816c69f5..13da8a3c8781f 100644 --- a/cmd/k8s-operator/tsrecorder_specs_test.go +++ b/cmd/k8s-operator/tsrecorder_specs_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -12,17 +12,18 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/types/ptr" ) func TestRecorderSpecs(t *testing.T) { - t.Run("ensure spec fields are passed through correctly", func(t *testing.T) { + t.Run("spec-fields-passthrough", func(t *testing.T) { tsr := &tsapi.Recorder{ ObjectMeta: metav1.ObjectMeta{ Name: "test", }, Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), StatefulSet: tsapi.RecorderStatefulSet{ Labels: map[string]string{ "ss-label-key": "ss-label-value", @@ -49,7 +50,7 @@ func TestRecorderSpecs(t *testing.T) { }, }, SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: ptr.To[int64](1000), + RunAsUser: new(int64(1000)), }, ImagePullSecrets: []corev1.LocalObjectReference{{ Name: "img-pull", @@ -60,7 +61,7 @@ func TestRecorderSpecs(t *testing.T) { Tolerations: []corev1.Toleration{{ Key: "key", Value: "value", - TolerationSeconds: ptr.To[int64](60), + TolerationSeconds: new(int64(60)), }}, Container: tsapi.RecorderContainer{ Env: []tsapi.Env{{ @@ -90,7 +91,7 @@ func TestRecorderSpecs(t *testing.T) { }, } - ss := tsrStatefulSet(tsr, tsNamespace) + ss := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) // StatefulSet-level. if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" { @@ -101,10 +102,10 @@ func TestRecorderSpecs(t *testing.T) { } // Pod-level. - if diff := cmp.Diff(ss.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" { + if diff := cmp.Diff(ss.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Labels)); diff != "" { t.Errorf("(-got +want):\n%s", diff) } - if diff := cmp.Diff(ss.Spec.Template.Labels, labels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" { + if diff := cmp.Diff(ss.Spec.Template.Labels, tsrLabels("recorder", "test", tsr.Spec.StatefulSet.Pod.Labels)); diff != "" { t.Errorf("(-got +want):\n%s", diff) } if diff := cmp.Diff(ss.Spec.Template.Spec.Affinity, tsr.Spec.StatefulSet.Pod.Affinity); diff != "" { @@ -124,7 +125,7 @@ func TestRecorderSpecs(t *testing.T) { } // Container-level. - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr)); diff != "" { + if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, tsrEnv(tsr, tsLoginServer)); diff != "" { t.Errorf("(-got +want):\n%s", diff) } if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" { @@ -139,5 +140,17 @@ func TestRecorderSpecs(t *testing.T) { if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Resources, tsr.Spec.StatefulSet.Pod.Container.Resources); diff != "" { t.Errorf("(-got +want):\n%s", diff) } + + if *ss.Spec.Replicas != *tsr.Spec.Replicas { + t.Errorf("expected %d replicas, got %d", *tsr.Spec.Replicas, *ss.Spec.Replicas) + } + + if len(ss.Spec.Template.Spec.Volumes) != int(*tsr.Spec.Replicas)+1 { + t.Errorf("expected %d volumes, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Volumes)) + } + + if len(ss.Spec.Template.Spec.Containers[0].VolumeMounts) != int(*tsr.Spec.Replicas)+1 { + t.Errorf("expected %d volume mounts, got %d", *tsr.Spec.Replicas+1, len(ss.Spec.Template.Spec.Containers[0].VolumeMounts)) + } }) } diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go index e6d56ef2f04c6..8f189728c0207 100644 --- a/cmd/k8s-operator/tsrecorder_test.go +++ b/cmd/k8s-operator/tsrecorder_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -8,11 +8,13 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "testing" "github.com/google/go-cmp/cmp" "go.uber.org/zap" + "golang.org/x/time/rate" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -20,12 +22,18 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/client/tailscale/v2" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/tsclient" "tailscale.com/tstest" ) -const tsNamespace = "tailscale" +const ( + tsNamespace = "tailscale" + tsLoginServer = "example.tailscale.com" +) func TestRecorder(t *testing.T) { tsr := &tsapi.Recorder{ @@ -33,6 +41,9 @@ func TestRecorder(t *testing.T) { Name: "test", Finalizers: []string{"tailscale.com/finalizer"}, }, + Spec: tsapi.RecorderSpec{ + Replicas: new(int32(3)), + }, } fc := fake.NewClientBuilder(). @@ -40,17 +51,19 @@ func TestRecorder(t *testing.T) { WithObjects(tsr). WithStatusSubresource(tsr). Build() - tsClient := &fakeTSClient{} + tsClient := &fakeTSClient{loginURL: tsLoginServer} zl, _ := zap.NewDevelopment() fr := record.NewFakeRecorder(2) cl := tstest.NewClock(tstest.ClockOpts{}) reconciler := &RecorderReconciler{ - tsNamespace: tsNamespace, - Client: fc, - tsClient: tsClient, - recorder: fr, - l: zl.Sugar(), - clock: cl, + tsNamespace: tsNamespace, + Client: fc, + clients: tsclient.NewProvider(tsClient), + recorder: fr, + log: zl.Sugar(), + clock: cl, + authKeyRateLimits: make(map[string]*rate.Limiter), + authKeyReissuing: make(map[string]bool), } t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { @@ -76,6 +89,15 @@ func TestRecorder(t *testing.T) { }) expectReconciled(t, reconciler, "", tsr.Name) + expectedEvent = "Warning RecorderInvalid Recorder is invalid: must use S3 storage when using multiple replicas to ensure recordings are accessible" + expectEvents(t, fr, []string{expectedEvent}) + + tsr.Spec.Storage.S3 = &tsapi.S3{} + mustUpdate(t, fc, "", "test", func(t *tsapi.Recorder) { + t.Spec = tsr.Spec + }) + expectReconciled(t, reconciler, "", tsr.Name) + // Only check part of this error message, because it's defined in an // external package and may change. if err := fc.Get(context.Background(), client.ObjectKey{ @@ -176,33 +198,65 @@ func TestRecorder(t *testing.T) { }) t.Run("populate_node_info_in_state_secret_and_see_it_appear_in_status", func(t *testing.T) { - bytes, err := json.Marshal(map[string]any{ - "Config": map[string]any{ - "NodeID": "nodeid-123", - "UserProfile": map[string]any{ - "LoginName": "test-0.example.ts.net", + const key = "profile-abc" + + for replica := range *tsr.Spec.Replicas { + bytes, err := json.Marshal(map[string]any{ + "Config": map[string]any{ + "NodeID": fmt.Sprintf("node-%d", replica), + "UserProfile": map[string]any{ + "LoginName": fmt.Sprintf("test-%d.example.ts.net", replica), + }, }, - }, - }) - if err != nil { - t.Fatal(err) + }) + if err != nil { + t.Fatal(err) + } + + name := fmt.Sprintf("%s-%d", "test", replica) + mustUpdate(t, fc, tsNamespace, name, func(s *corev1.Secret) { + s.Data = map[string][]byte{ + currentProfileKey: []byte(key), + key: bytes, + } + }) } - const key = "profile-abc" - mustUpdate(t, fc, tsNamespace, "test-0", func(s *corev1.Secret) { - s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, - } - }) + tsClient.devices = []tailscale.Device{ + { + ID: "node-0", + Hostname: "hostname-node-0", + Addresses: []string{"1.2.3.4", "::1"}, + }, + { + ID: "node-1", + Hostname: "hostname-node-1", + Addresses: []string{"1.2.3.4", "::1"}, + }, + { + ID: "node-2", + Hostname: "hostname-node-2", + Addresses: []string{"1.2.3.4", "::1"}, + }, + } expectReconciled(t, reconciler, "", tsr.Name) tsr.Status.Devices = []tsapi.RecorderTailnetDevice{ { - Hostname: "hostname-nodeid-123", + Hostname: "hostname-node-0", TailnetIPs: []string{"1.2.3.4", "::1"}, URL: "https://test-0.example.ts.net", }, + { + Hostname: "hostname-node-1", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-1.example.ts.net", + }, + { + Hostname: "hostname-node-2", + TailnetIPs: []string{"1.2.3.4", "::1"}, + URL: "https://test-2.example.ts.net", + }, } expectEqual(t, fc, tsr) }) @@ -218,7 +272,7 @@ func TestRecorder(t *testing.T) { if expected := 0; reconciler.recorders.Len() != expected { t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len()) } - if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-123"}); diff != "" { + if diff := cmp.Diff(tsClient.deleted, []string{"node-0", "node-1", "node-2"}); diff != "" { t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) } // The fake client does not clean up objects whose owner has been @@ -229,26 +283,38 @@ func TestRecorder(t *testing.T) { func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recorder, shouldExist bool) { t.Helper() - auth := tsrAuthSecret(tsr, tsNamespace, "secret-authkey") - state := tsrStateSecret(tsr, tsNamespace) + var replicas int32 = 1 + if tsr.Spec.Replicas != nil { + replicas = *tsr.Spec.Replicas + } + role := tsrRole(tsr, tsNamespace) roleBinding := tsrRoleBinding(tsr, tsNamespace) serviceAccount := tsrServiceAccount(tsr, tsNamespace) - statefulSet := tsrStatefulSet(tsr, tsNamespace) + statefulSet := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) if shouldExist { - expectEqual(t, fc, auth) - expectEqual(t, fc, state) expectEqual(t, fc, role) expectEqual(t, fc, roleBinding) expectEqual(t, fc, serviceAccount) expectEqual(t, fc, statefulSet, removeResourceReqs) } else { - expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name) - expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name) expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name) expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name) expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name) expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name) } + + for replica := range replicas { + auth := tsrAuthSecret(tsr, tsNamespace, "new-authkey", replica) + state := tsrStateSecret(tsr, tsNamespace, replica) + + if shouldExist { + expectEqual(t, fc, auth) + expectEqual(t, fc, state) + } else { + expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name) + expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name) + } + } } diff --git a/cmd/k8s-proxy/internal/config/config.go b/cmd/k8s-proxy/internal/config/config.go new file mode 100644 index 0000000000000..c12383d45c470 --- /dev/null +++ b/cmd/k8s-proxy/internal/config/config.go @@ -0,0 +1,263 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package config provides watchers for the various supported ways to load a +// config file for k8s-proxy; currently file or Kubernetes Secret. +package config + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/util/testenv" +) + +type configLoader struct { + logger *zap.SugaredLogger + client clientcorev1.CoreV1Interface + + cfgChan chan<- *conf.Config + previous []byte + + once sync.Once // For use in tests. To close cfgIgnored. + cfgIgnored chan struct{} // For use in tests. +} + +func NewConfigLoader(logger *zap.SugaredLogger, client clientcorev1.CoreV1Interface, cfgChan chan<- *conf.Config) *configLoader { + return &configLoader{ + logger: logger, + client: client, + cfgChan: cfgChan, + } +} + +func (ld *configLoader) WatchConfig(ctx context.Context, path string) error { + secretNamespacedName, isKubeSecret := strings.CutPrefix(path, "kube:") + if isKubeSecret { + secretNamespace, secretName, ok := strings.Cut(secretNamespacedName, string(types.Separator)) + if !ok { + return fmt.Errorf("invalid Kubernetes Secret reference %q, expected format /", path) + } + if err := ld.watchConfigSecretChanges(ctx, secretNamespace, secretName); err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error watching config Secret %q: %w", secretNamespacedName, err) + } + + return nil + } + + if err := ld.watchConfigFileChanges(ctx, path); err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error watching config file %q: %w", path, err) + } + + return nil +} + +func (ld *configLoader) reloadConfig(ctx context.Context, raw []byte) error { + if bytes.Equal(raw, ld.previous) { + if ld.cfgIgnored != nil && testenv.InTest() { + ld.once.Do(func() { + close(ld.cfgIgnored) + }) + } + return nil + } + + cfg, err := conf.Load(raw) + if err != nil { + return fmt.Errorf("error loading config: %w", err) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case ld.cfgChan <- &cfg: + } + + ld.previous = raw + return nil +} + +func (ld *configLoader) watchConfigFileChanges(ctx context.Context, path string) error { + var ( + tickChan <-chan time.Time + eventChan <-chan fsnotify.Event + errChan <-chan error + ) + + if w, err := fsnotify.NewWatcher(); err != nil { + // Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor. + // See https://github.com/tailscale/tailscale/issues/15081 + ld.logger.Infof("Failed to create fsnotify watcher on config file %q; watching for changes on 5s timer: %v", path, err) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + tickChan = ticker.C + } else { + dir := filepath.Dir(path) + file := filepath.Base(path) + ld.logger.Infof("Watching directory %q for changes to config file %q", dir, file) + defer w.Close() + if err := w.Add(dir); err != nil { + return fmt.Errorf("failed to add fsnotify watch: %w", err) + } + eventChan = w.Events + errChan = w.Errors + } + + // Read the initial config file, but after the watcher is already set up to + // avoid an unlucky race condition if the config file is edited in between. + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading config file %q: %w", path, err) + } + if err := ld.reloadConfig(ctx, b); err != nil { + return fmt.Errorf("error loading initial config file %q: %w", path, err) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err, ok := <-errChan: + if !ok { + // Watcher was closed. + return nil + } + return fmt.Errorf("watcher error: %w", err) + case <-tickChan: + case ev, ok := <-eventChan: + if !ok { + // Watcher was closed. + return nil + } + if ev.Name != path || ev.Op&fsnotify.Write == 0 { + // Ignore irrelevant events. + continue + } + } + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading config file: %w", err) + } + // Writers such as os.WriteFile may truncate the file before writing + // new contents, so it's possible to read an empty file if we read before + // the write has completed. + if len(b) == 0 { + continue + } + if err := ld.reloadConfig(ctx, b); err != nil { + return fmt.Errorf("error reloading config file %q: %v", path, err) + } + } +} + +func (ld *configLoader) watchConfigSecretChanges(ctx context.Context, secretNamespace, secretName string) error { + secrets := ld.client.Secrets(secretNamespace) + w, err := secrets.Watch(ctx, metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + // Re-watch regularly to avoid relying on long-lived connections. + // See https://github.com/kubernetes-client/javascript/issues/596#issuecomment-786419380 + TimeoutSeconds: new(int64(600)), + FieldSelector: fmt.Sprintf("metadata.name=%s", secretName), + Watch: true, + }) + if err != nil { + return fmt.Errorf("failed to watch config Secret %q: %w", secretName, err) + } + defer func() { + // May not be the original watcher by the time we exit. + if w != nil { + w.Stop() + } + }() + + // Get the initial config Secret now we've got the watcher set up. + secret, err := secrets.Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get config Secret %q: %w", secretName, err) + } + + if err := ld.configFromSecret(ctx, secret); err != nil { + return fmt.Errorf("error loading initial config: %w", err) + } + + ld.logger.Infof("Watching config Secret %q for changes", secretName) + for { + var secret *corev1.Secret + select { + case <-ctx.Done(): + return ctx.Err() + case ev, ok := <-w.ResultChan(): + if !ok { + w.Stop() + w, err = secrets.Watch(ctx, metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + TimeoutSeconds: new(int64(600)), + FieldSelector: fmt.Sprintf("metadata.name=%s", secretName), + Watch: true, + }) + if err != nil { + return fmt.Errorf("failed to re-watch config Secret %q: %w", secretName, err) + } + continue + } + + switch ev.Type { + case watch.Added, watch.Modified: + // New config available to load. + var ok bool + secret, ok = ev.Object.(*corev1.Secret) + if !ok { + return fmt.Errorf("unexpected object type %T in watch event for config Secret %q", ev.Object, secretName) + } + if secret == nil || secret.Data == nil { + continue + } + if err := ld.configFromSecret(ctx, secret); err != nil { + return fmt.Errorf("error reloading config Secret %q: %v", secret.Name, err) + } + case watch.Error: + return fmt.Errorf("error watching config Secret %q: %v", secretName, ev.Object) + default: + // Ignore, no action required. + continue + } + } + } +} + +func (ld *configLoader) configFromSecret(ctx context.Context, s *corev1.Secret) error { + b := s.Data[kubetypes.KubeAPIServerConfigFile] + if len(b) == 0 { + return fmt.Errorf("config Secret %q does not contain expected config in key %q", s.Name, kubetypes.KubeAPIServerConfigFile) + } + + if err := ld.reloadConfig(ctx, b); err != nil { + return err + } + + return nil +} diff --git a/cmd/k8s-proxy/internal/config/config_test.go b/cmd/k8s-proxy/internal/config/config_test.go new file mode 100644 index 0000000000000..aedd29d4e1877 --- /dev/null +++ b/cmd/k8s-proxy/internal/config/config_test.go @@ -0,0 +1,244 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package config + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" +) + +func TestWatchConfig(t *testing.T) { + type phase struct { + config string + cancel bool + expectedConf *conf.ConfigV1Alpha1 + expectedErr string + } + + // Same set of behaviour tests for each config source. + for _, env := range []string{"file", "kube"} { + t.Run(env, func(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + initialConfig string + phases []phase + }{ + { + name: "no_config", + phases: []phase{{ + expectedErr: "error loading initial config", + }}, + }, + { + name: "valid_config", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{{ + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: new("abc123"), + }, + }}, + }, + { + name: "can_cancel", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: new("abc123"), + }, + }, + { + cancel: true, + }, + }, + }, + { + name: "can_reload", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: new("abc123"), + }, + }, + { + config: `{"version": "v1alpha1", "authKey": "def456"}`, + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: new("def456"), + }, + }, + }, + }, + { + name: "ignores_events_with_no_changes", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: new("abc123"), + }, + }, + { + config: `{"version": "v1alpha1", "authKey": "abc123"}`, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + cl := fake.NewClientset() + + var cfgPath string + var writeFile func(*testing.T, string) + if env == "file" { + cfgPath = filepath.Join(root, kubetypes.KubeAPIServerConfigFile) + writeFile = func(t *testing.T, content string) { + if err := os.WriteFile(cfgPath, []byte(content), 0o644); err != nil { + t.Fatalf("error writing config file %q: %v", cfgPath, err) + } + } + } else { + cfgPath = "kube:default/config-secret" + writeFile = func(t *testing.T, content string) { + s := secretFrom(content) + mustCreateOrUpdate(t, cl, s) + } + } + configChan := make(chan *conf.Config) + loader := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan) + loader.cfgIgnored = make(chan struct{}) + errs := make(chan error) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + writeFile(t, tc.initialConfig) + go func() { + errs <- loader.WatchConfig(ctx, cfgPath) + }() + + for i, p := range tc.phases { + if p.config != "" { + writeFile(t, p.config) + } + if p.cancel { + cancel() + } + + select { + case cfg := <-configChan: + if diff := cmp.Diff(*p.expectedConf, cfg.Parsed); diff != "" { + t.Errorf("unexpected config (-want +got):\n%s", diff) + } + case err := <-errs: + if p.cancel { + if err != nil { + t.Fatalf("unexpected error after cancel: %v", err) + } + } else if p.expectedErr == "" { + t.Fatalf("unexpected error: %v", err) + } else if !strings.Contains(err.Error(), p.expectedErr) { + t.Fatalf("expected error to contain %q, got %q", p.expectedErr, err.Error()) + } + case <-loader.cfgIgnored: + if p.expectedConf != nil { + t.Fatalf("expected config to be reloaded, but got ignored signal") + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected event in phase: %d", i) + } + } + }) + } + }) + } +} + +func TestWatchConfigSecret_Rewatches(t *testing.T) { + cl := fake.NewClientset() + var watchCount int + var watcher *watch.RaceFreeFakeWatcher + expected := []string{ + `{"version": "v1alpha1", "authKey": "abc123"}`, + `{"version": "v1alpha1", "authKey": "def456"}`, + `{"version": "v1alpha1", "authKey": "ghi789"}`, + } + cl.PrependWatchReactor("secrets", func(action ktesting.Action) (handled bool, ret watch.Interface, err error) { + watcher = watch.NewRaceFreeFake() + watcher.Add(secretFrom(expected[watchCount])) + if action.GetVerb() == "watch" && action.GetResource().Resource == "secrets" { + watchCount++ + } + return true, watcher, nil + }) + + configChan := make(chan *conf.Config) + loader := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan) + + mustCreateOrUpdate(t, cl, secretFrom(expected[0])) + + errs := make(chan error) + go func() { + errs <- loader.watchConfigSecretChanges(t.Context(), "default", "config-secret") + }() + + for i := range 2 { + select { + case cfg := <-configChan: + if exp := expected[i]; cfg.Parsed.AuthKey == nil || !strings.Contains(exp, *cfg.Parsed.AuthKey) { + t.Fatalf("expected config to have authKey %q, got: %v", exp, cfg.Parsed.AuthKey) + } + if i == 0 { + watcher.Stop() + } + case err := <-errs: + t.Fatalf("unexpected error: %v", err) + case <-loader.cfgIgnored: + t.Fatalf("expected config to be reloaded, but got ignored signal") + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected event") + } + } + + if watchCount != 2 { + t.Fatalf("expected 2 watch API calls, got %d", watchCount) + } +} + +func secretFrom(content string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "config-secret", + }, + Data: map[string][]byte{ + kubetypes.KubeAPIServerConfigFile: []byte(content), + }, + } +} + +func mustCreateOrUpdate(t *testing.T, cl *fake.Clientset, s *corev1.Secret) { + t.Helper() + if _, err := cl.CoreV1().Secrets("default").Create(t.Context(), s, metav1.CreateOptions{}); err != nil { + if _, updateErr := cl.CoreV1().Secrets("default").Update(t.Context(), s, metav1.UpdateOptions{}); updateErr != nil { + t.Fatalf("error writing config Secret %q: %v", s.Name, updateErr) + } + } +} diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go new file mode 100644 index 0000000000000..673493f58cecd --- /dev/null +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -0,0 +1,546 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// k8s-proxy proxies between tailnet and Kubernetes cluster traffic. +// Currently, it only supports proxying tailnet clients to the Kubernetes API +// server. +package main + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "reflect" + "strconv" + "strings" + "syscall" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/utils/strings/slices" + "tailscale.com/client/local" + "tailscale.com/cmd/k8s-proxy/internal/config" + "tailscale.com/health" + "tailscale.com/hostinfo" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + + // we need to import this package so that the `kube:` ipn store gets registered + _ "tailscale.com/ipn/store/kubestore" + apiproxy "tailscale.com/k8s-operator/api-proxy" + "tailscale.com/kube/certs" + healthz "tailscale.com/kube/health" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + klc "tailscale.com/kube/localclient" + "tailscale.com/kube/metrics" + "tailscale.com/kube/services" + "tailscale.com/kube/state" + "tailscale.com/tailcfg" + "tailscale.com/tsnet" +) + +const ( + // proxyProtocolV2 enables PROXY protocol v2 to preserve original client + // connection info after TLS termination. + proxyProtocolV2 = 2 +) + +func main() { + encoderCfg := zap.NewProductionEncoderConfig() + encoderCfg.EncodeTime = zapcore.RFC3339TimeEncoder + logger := zap.Must(zap.Config{ + Level: zap.NewAtomicLevelAt(zap.DebugLevel), + Encoding: "json", + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: encoderCfg, + }.Build()).Sugar() + defer logger.Sync() + + if err := run(logger); err != nil { + logger.Fatal(err.Error()) + } +} + +func run(logger *zap.SugaredLogger) error { + var ( + configPath = os.Getenv("TS_K8S_PROXY_CONFIG") + podUID = os.Getenv("POD_UID") + podIP = os.Getenv("POD_IP") + ) + if configPath == "" { + return errors.New("TS_K8S_PROXY_CONFIG unset") + } + + // serveCtx to live for the lifetime of the process, only gets cancelled + // once the Tailscale Service has been drained + serveCtx, serveCancel := context.WithCancel(context.Background()) + defer serveCancel() + + // ctx to cancel to start the shutdown process. + ctx, cancel := context.WithCancel(serveCtx) + defer cancel() + + sigsChan := make(chan os.Signal, 1) + signal.Notify(sigsChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + select { + case <-ctx.Done(): + case s := <-sigsChan: + logger.Infof("Received shutdown signal %s, exiting", s) + cancel() + } + }() + + var group *errgroup.Group + group, ctx = errgroup.WithContext(ctx) + + restConfig, err := getRestConfig(logger) + if err != nil { + return fmt.Errorf("error getting rest config: %w", err) + } + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("error creating Kubernetes clientset: %w", err) + } + + // Load and watch config. + cfgChan := make(chan *conf.Config) + cfgLoader := config.NewConfigLoader(logger, clientset.CoreV1(), cfgChan) + group.Go(func() error { + return cfgLoader.WatchConfig(ctx, configPath) + }) + + // Get initial config. + var cfg *conf.Config + select { + case <-ctx.Done(): + return group.Wait() + case cfg = <-cfgChan: + } + + if cfg.Parsed.LogLevel != nil { + level, err := zapcore.ParseLevel(*cfg.Parsed.LogLevel) + if err != nil { + return fmt.Errorf("error parsing log level %q: %w", *cfg.Parsed.LogLevel, err) + } + logger = logger.WithOptions(zap.IncreaseLevel(level)) + } + + // TODO:(ChaosInTheCRD) This is a temporary workaround until we can set static endpoints using prefs + if se := cfg.Parsed.StaticEndpoints; len(se) > 0 { + logger.Debugf("setting static endpoints '%v' via TS_DEBUG_PRETENDPOINT environment variable", cfg.Parsed.StaticEndpoints) + ses := make([]string, len(se)) + for i, e := range se { + ses[i] = e.String() + } + + err := os.Setenv("TS_DEBUG_PRETENDPOINT", strings.Join(ses, ",")) + if err != nil { + return err + } + } + + if cfg.Parsed.App != nil { + hostinfo.SetApp(*cfg.Parsed.App) + } + + // TODO(tomhjp): Pass this setting directly into the store instead of using + // environment variables. + if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true) { + os.Setenv("TS_CERT_SHARE_MODE", "rw") + } else { + os.Setenv("TS_CERT_SHARE_MODE", "ro") + } + + st, err := getStateStore(cfg.Parsed.State, logger) + if err != nil { + return err + } + + // If Pod UID unset, assume we're running outside of a cluster/not managed + // by the operator, so no need to set additional state keys. + var kc kubeclient.Client + var stateSecretName string + if podUID != "" { + if err := state.SetInitialKeys(st, podUID); err != nil { + return fmt.Errorf("error setting initial state: %w", err) + } + + if cfg.Parsed.State != nil { + if name, ok := strings.CutPrefix(*cfg.Parsed.State, "kube:"); ok { + stateSecretName = name + + kc, err = kubeclient.New(k8sProxyFieldManager) + if err != nil { + return err + } + + var configAuthKey string + if cfg.Parsed.AuthKey != nil { + configAuthKey = *cfg.Parsed.AuthKey + } + if err := resetState(ctx, kc, stateSecretName, podUID, configAuthKey); err != nil { + return fmt.Errorf("error resetting state: %w", err) + } + } + } + } + + var authKey string + if cfg.Parsed.AuthKey != nil { + authKey = *cfg.Parsed.AuthKey + } + + ts := &tsnet.Server{ + Logf: logger.Named("tsnet").Debugf, + UserLogf: logger.Named("tsnet").Infof, + Store: st, + AuthKey: authKey, + } + + if cfg.Parsed.ServerURL != nil { + ts.ControlURL = *cfg.Parsed.ServerURL + } + + if cfg.Parsed.Hostname != nil { + ts.Hostname = *cfg.Parsed.Hostname + } + + lc, err := ts.LocalClient() + if err != nil { + return fmt.Errorf("error getting local client: %w", err) + } + + // Make sure we crash loop if Up doesn't complete in reasonable time. + upCtx, upCancel := context.WithTimeout(ctx, 30*time.Second) + defer upCancel() + + // ts.Up() deliberately ignores NeedsLogin because it fires transiently + // during normal auth-key login. We can watch for the login-state health + // warning here though, which only fires on terminal auth failure, and + // cancel early. + go func() { + w, err := lc.WatchIPNBus(upCtx, ipn.NotifyInitialHealthState) + if err != nil { + return + } + defer w.Close() + for { + n, err := w.Next() + if err != nil { + logger.Debugf("failed to process message from ipn bus: %s", err.Error()) + return + } + if n.Health != nil { + if _, ok := n.Health.Warnings[health.LoginStateWarnable.Code]; ok { + upCancel() + return + } + } + } + }() + + if _, err := ts.Up(upCtx); err != nil { + if kc != nil && stateSecretName != "" { + return handleAuthKeyReissue(ctx, lc, kc, stateSecretName, authKey, cfgChan, logger) + } + return err + } + + defer ts.Close() + + reissueCh := make(chan struct{}, 1) + if podUID != "" { + group.Go(func() error { + return state.KeepKeysUpdated(ctx, st, klc.New(lc)) + }) + + if kc != nil && stateSecretName != "" { + needsReissue, err := checkInitialAuthState(ctx, lc) + if err != nil { + return fmt.Errorf("error checking initial auth state: %w", err) + } + if needsReissue { + logger.Info("Auth key missing or invalid after startup, requesting new key from operator") + return handleAuthKeyReissue(ctx, lc, kc, stateSecretName, authKey, cfgChan, logger) + } + + group.Go(func() error { + return monitorAuthHealth(ctx, lc, reissueCh, logger) + }) + } + } + + if cfg.Parsed.HealthCheckEnabled.EqualBool(true) || cfg.Parsed.MetricsEnabled.EqualBool(true) { + addr := podIP + if addr == "" { + addr = cfg.GetLocalAddr() + } + + addrPort := getLocalAddrPort(addr, cfg.GetLocalPort()) + mux := http.NewServeMux() + localSrv := &http.Server{Addr: addrPort, Handler: mux} + + if cfg.Parsed.MetricsEnabled.EqualBool(true) { + logger.Infof("Running metrics endpoint at %s/metrics", addrPort) + metrics.RegisterMetricsHandlers(mux, lc, "") + } + + if cfg.Parsed.HealthCheckEnabled.EqualBool(true) { + ipV4, _ := ts.TailscaleIPs() + hz := healthz.RegisterHealthHandlers(mux, ipV4.String(), logger.Infof) + group.Go(func() error { + err := hz.MonitorHealth(ctx, lc) + if err == nil || errors.Is(err, context.Canceled) { + return nil + } + return err + }) + } + + group.Go(func() error { + errChan := make(chan error) + go func() { + if err := localSrv.ListenAndServe(); err != nil { + errChan <- err + } + close(errChan) + }() + + select { + case <-ctx.Done(): + sCtx, scancel := context.WithTimeout(serveCtx, 10*time.Second) + defer scancel() + return localSrv.Shutdown(sCtx) + case err := <-errChan: + return err + } + }) + } + + if v, ok := cfg.Parsed.AcceptRoutes.Get(); ok { + _, err = lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + RouteAllSet: true, + Prefs: ipn.Prefs{RouteAll: v}, + }) + if err != nil { + return fmt.Errorf("error editing prefs: %w", err) + } + } + + // TODO(tomhjp): There seems to be a bug that on restart the device does + // not get reassigned it's already working Service IPs unless we clear and + // reset the serve config. + if err := lc.SetServeConfig(ctx, &ipn.ServeConfig{}); err != nil { + return fmt.Errorf("error clearing existing ServeConfig: %w", err) + } + + var cm *certs.CertManager + if shouldIssueCerts(cfg) { + logger.Infof("Will issue TLS certs for Tailscale Service") + cm = certs.NewCertManager(klc.New(lc), logger.Infof) + } + if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil { + return err + } + + if cfg.Parsed.AdvertiseServices != nil { + if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: cfg.Parsed.AdvertiseServices, + }, + }); err != nil { + return fmt.Errorf("error setting prefs AdvertiseServices: %w", err) + } + } + + // Setup for the API server proxy. + mode := kubetypes.APIServerProxyModeAuth + if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.Mode != nil { + mode = *cfg.Parsed.APIServerProxy.Mode + } + ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, mode, false) + if err != nil { + return fmt.Errorf("error creating api server proxy: %w", err) + } + + group.Go(func() error { + if err := ap.Run(serveCtx); err != nil { + return fmt.Errorf("error running API server proxy: %w", err) + } + + return nil + }) + + for { + select { + case <-ctx.Done(): + // Context cancelled, exit. + logger.Info("Context cancelled, exiting") + shutdownCtx, shutdownCancel := context.WithTimeout(serveCtx, 20*time.Second) + unadvertiseErr := services.EnsureServicesNotAdvertised(shutdownCtx, lc, logger.Infof) + shutdownCancel() + serveCancel() + return errors.Join(unadvertiseErr, group.Wait()) + case cfg = <-cfgChan: + // Handle config reload. + // TODO(tomhjp): Make auth mode reloadable. + var prefs ipn.MaskedPrefs + cfgLogger := logger + currentPrefs, err := lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting current prefs: %w", err) + } + if !slices.Equal(currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices) { + cfgLogger = cfgLogger.With("AdvertiseServices", fmt.Sprintf("%v -> %v", currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices)) + prefs.AdvertiseServicesSet = true + prefs.Prefs.AdvertiseServices = cfg.Parsed.AdvertiseServices + } + if cfg.Parsed.Hostname != nil && *cfg.Parsed.Hostname != currentPrefs.Hostname { + cfgLogger = cfgLogger.With("Hostname", fmt.Sprintf("%s -> %s", currentPrefs.Hostname, *cfg.Parsed.Hostname)) + prefs.HostnameSet = true + prefs.Hostname = *cfg.Parsed.Hostname + } + if v, ok := cfg.Parsed.AcceptRoutes.Get(); ok && v != currentPrefs.RouteAll { + cfgLogger = cfgLogger.With("AcceptRoutes", fmt.Sprintf("%v -> %v", currentPrefs.RouteAll, v)) + prefs.RouteAllSet = true + prefs.Prefs.RouteAll = v + } + if !prefs.IsEmpty() { + if _, err := lc.EditPrefs(ctx, &prefs); err != nil { + return fmt.Errorf("error editing prefs: %w", err) + } + } + if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil { + return fmt.Errorf("error setting serve config: %w", err) + } + + cfgLogger.Infof("Config reloaded") + case <-reissueCh: + return handleAuthKeyReissue(ctx, lc, kc, stateSecretName, authKey, cfgChan, logger) + } + } +} + +func getLocalAddrPort(addr string, port uint16) string { + return net.JoinHostPort(addr, strconv.FormatUint(uint64(port), 10)) +} + +func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) { + p := "mem:" + if path != nil { + p = *path + } else { + logger.Warn("No state Secret provided; using in-memory store, which will lose state on restart") + } + st, err := store.New(logger.Errorf, p) + if err != nil { + return nil, fmt.Errorf("error creating state store: %w", err) + } + + return st, nil +} + +func getRestConfig(logger *zap.SugaredLogger) (*rest.Config, error) { + restConfig, err := rest.InClusterConfig() + switch err { + case nil: + return restConfig, nil + case rest.ErrNotInCluster: + logger.Info("Not running in-cluster, falling back to kubeconfig") + default: + return nil, fmt.Errorf("error getting in-cluster config: %w", err) + } + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, nil) + restConfig, err = clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("error loading kubeconfig: %w", err) + } + + return restConfig, nil +} + +func apiServerProxyService(cfg *conf.Config) tailcfg.ServiceName { + if cfg.Parsed.APIServerProxy != nil && + cfg.Parsed.APIServerProxy.Enabled.EqualBool(true) && + cfg.Parsed.APIServerProxy.ServiceName != nil && + *cfg.Parsed.APIServerProxy.ServiceName != "" { + return tailcfg.ServiceName(*cfg.Parsed.APIServerProxy.ServiceName) + } + + return "" +} + +func shouldIssueCerts(cfg *conf.Config) bool { + return cfg.Parsed.APIServerProxy != nil && + cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true) +} + +// setServeConfig sets up serve config such that it's serving for the passed in +// Tailscale Service, and does nothing if it's already up to date. +func setServeConfig(ctx context.Context, lc *local.Client, cm *certs.CertManager, name tailcfg.ServiceName) error { + existingServeConfig, err := lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("error getting existing serve config: %w", err) + } + + // Ensure serve config is cleared if no Tailscale Service. + if name == "" { + if reflect.DeepEqual(*existingServeConfig, ipn.ServeConfig{}) { + // Already up to date. + return nil + } + + if cm != nil { + cm.EnsureCertLoops(ctx, &ipn.ServeConfig{}) + } + return lc.SetServeConfig(ctx, &ipn.ServeConfig{}) + } + + status, err := lc.StatusWithoutPeers(ctx) + if err != nil { + return fmt.Errorf("error getting local client status: %w", err) + } + serviceSNI := fmt.Sprintf("%s.%s", name.WithoutPrefix(), status.CurrentTailnet.MagicDNSSuffix) + + serveConfig := ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + name: { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + TCPForward: "localhost:80", + TerminateTLS: serviceSNI, + ProxyProtocol: proxyProtocolV2, + }, + }, + }, + }, + } + + if reflect.DeepEqual(*existingServeConfig, serveConfig) { + // Already up to date. + return nil + } + + if cm != nil { + cm.EnsureCertLoops(ctx, &serveConfig) + } + return lc.SetServeConfig(ctx, &serveConfig) +} diff --git a/cmd/k8s-proxy/kube.go b/cmd/k8s-proxy/kube.go new file mode 100644 index 0000000000000..1d9348f1a3bea --- /dev/null +++ b/cmd/k8s-proxy/kube.go @@ -0,0 +1,161 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "go.uber.org/zap" + "tailscale.com/client/local" + "tailscale.com/health" + "tailscale.com/ipn" + "tailscale.com/kube/authkey" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubeapi" + "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" +) + +const k8sProxyFieldManager = "tailscale-k8s-proxy" + +// resetState clears k8s-proxy state from previous runs and sets +// initial values. This ensures the operator doesn't use stale state when a Pod +// is first recreated. +// +// It also clears the reissue_authkey marker if the operator has actioned it +// (i.e., the config now has a different auth key than what was marked for +// reissue). +func resetState(ctx context.Context, kc kubeclient.Client, stateSecretName string, podUID string, configAuthKey string) error { + existingSecret, err := kc.GetSecret(ctx, stateSecretName) + switch { + case kubeclient.IsNotFoundErr(err): + return nil + case err != nil: + return fmt.Errorf("failed to read state Secret %q to reset state: %w", stateSecretName, err) + } + + s := &kubeapi.Secret{ + Data: map[string][]byte{ + kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), + }, + } + if podUID != "" { + s.Data[kubetypes.KeyPodUID] = []byte(podUID) + } + + // Only clear reissue_authkey if the operator has actioned it. + brokenAuthkey, ok := existingSecret.Data[kubetypes.KeyReissueAuthkey] + if ok && configAuthKey != "" && string(brokenAuthkey) != configAuthKey { + s.Data[kubetypes.KeyReissueAuthkey] = nil + } + + return kc.StrategicMergePatchSecret(ctx, stateSecretName, s, k8sProxyFieldManager) +} + +// needsAuthKeyReissue reports whether the given backend state and health +// warnings indicate a terminal auth failure requiring a new key from the +// operator. +func needsAuthKeyReissue(backendState string, healthWarnings []string) bool { + if backendState == ipn.NeedsLogin.String() { + return true + } + loginWarnableCode := string(health.LoginStateWarnable.Code) + for _, h := range healthWarnings { + if strings.Contains(h, loginWarnableCode) { + return true + } + } + return false +} + +// checkInitialAuthState checks if the tsnet server is in an auth failure state +// immediately after coming up. Returns true if auth key reissue is needed. +func checkInitialAuthState(ctx context.Context, lc *local.Client) (bool, error) { + status, err := lc.Status(ctx) + if err != nil { + return false, fmt.Errorf("error getting status: %w", err) + } + return needsAuthKeyReissue(status.BackendState, status.Health), nil +} + +// monitorAuthHealth watches the IPN bus for auth failures and triggers reissue +// when needed. Runs until context is cancelled or auth failure is detected. +func monitorAuthHealth(ctx context.Context, lc *local.Client, reissueCh chan<- struct{}, logger *zap.SugaredLogger) error { + w, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialHealthState) + if err != nil { + return fmt.Errorf("failed to watch IPN bus for auth health: %w", err) + } + defer w.Close() + + for { + if ctx.Err() != nil { + return ctx.Err() + } + n, err := w.Next() + if err != nil { + return err + } + if n.Health != nil { + if _, ok := n.Health.Warnings[health.LoginStateWarnable.Code]; ok { + logger.Info("Auth key failed to authenticate (may be expired or single-use), requesting new key from operator") + select { + case reissueCh <- struct{}{}: + case <-ctx.Done(): + } + return nil + } + } + } +} + +// handleAuthKeyReissue orchestrates the auth key reissue flow: +// 1. Disconnect from control +// 2. Set reissue marker in state Secret +// 3. Wait for operator to provide new key +// 4. Exit cleanly (Kubernetes will restart the pod with the new key) +func handleAuthKeyReissue(ctx context.Context, lc *local.Client, kc kubeclient.Client, stateSecretName string, currentAuthKey string, cfgChan <-chan *conf.Config, logger *zap.SugaredLogger) error { + if err := lc.DisconnectControl(ctx); err != nil { + return fmt.Errorf("error disconnecting from control: %w", err) + } + if err := authkey.SetReissueAuthKey(ctx, kc, stateSecretName, currentAuthKey, k8sProxyFieldManager); err != nil { + return fmt.Errorf("failed to set reissue_authkey in Kubernetes Secret: %w", err) + } + + var mu sync.Mutex + var latestAuthKey string + notify := make(chan struct{}, 1) + + // we use this go func to abstract away conf.Config from the shared function + go func() { + for cfg := range cfgChan { + if cfg.Parsed.AuthKey != nil { + mu.Lock() + latestAuthKey = *cfg.Parsed.AuthKey + mu.Unlock() + select { + case notify <- struct{}{}: + default: + } + } + } + }() + + getAuthKey := func() string { + mu.Lock() + defer mu.Unlock() + return latestAuthKey + } + clearFn := func(ctx context.Context) error { + return authkey.ClearReissueAuthKey(ctx, kc, stateSecretName, k8sProxyFieldManager) + } + + return authkey.WaitForAuthKeyReissue(ctx, currentAuthKey, 10*time.Minute, getAuthKey, clearFn, notify) +} diff --git a/cmd/k8s-proxy/kube_test.go b/cmd/k8s-proxy/kube_test.go new file mode 100644 index 0000000000000..c7e0f33d02b9e --- /dev/null +++ b/cmd/k8s-proxy/kube_test.go @@ -0,0 +1,141 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/health" + "tailscale.com/kube/kubeapi" + "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" +) + +func TestResetState(t *testing.T) { + tests := []struct { + name string + existingData map[string][]byte + podUID string + configAuthKey string + wantPatched map[string][]byte + }{ + { + name: "sets_capver_and_pod_uid", + existingData: map[string][]byte{ + kubetypes.KeyDeviceID: []byte("device-123"), + kubetypes.KeyDeviceFQDN: []byte("node.tailnet"), + kubetypes.KeyDeviceIPs: []byte(`["100.64.0.1"]`), + }, + podUID: "pod-123", + configAuthKey: "new-key", + wantPatched: map[string][]byte{ + kubetypes.KeyPodUID: []byte("pod-123"), + }, + }, + { + name: "clears_reissue_marker_when_actioned", + existingData: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-key"), + }, + podUID: "pod-123", + configAuthKey: "new-key", + wantPatched: map[string][]byte{ + kubetypes.KeyPodUID: []byte("pod-123"), + kubetypes.KeyReissueAuthkey: nil, + }, + }, + { + name: "keeps_reissue_marker_when_not_actioned", + existingData: map[string][]byte{ + kubetypes.KeyReissueAuthkey: []byte("old-key"), + }, + podUID: "pod-123", + configAuthKey: "old-key", + wantPatched: map[string][]byte{ + kubetypes.KeyPodUID: []byte("pod-123"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.wantPatched[kubetypes.KeyCapVer] = fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + + var patched map[string][]byte + kc := &kubeclient.FakeClient{ + GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { + return &kubeapi.Secret{Data: tt.existingData}, nil + }, + StrategicMergePatchSecretImpl: func(ctx context.Context, name string, s *kubeapi.Secret, fm string) error { + patched = s.Data + return nil + }, + } + + err := resetState(context.Background(), kc, "test-secret", tt.podUID, tt.configAuthKey) + if err != nil { + t.Fatalf("resetState() error = %v", err) + } + + if diff := cmp.Diff(tt.wantPatched, patched); diff != "" { + t.Errorf("resetState() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestNeedsAuthKeyReissue(t *testing.T) { + loginWarnableCode := string(health.LoginStateWarnable.Code) + + tests := []struct { + name string + backendState string + health []string + want bool + }{ + { + name: "running_healthy", + backendState: "Running", + want: false, + }, + { + name: "needs_login", + backendState: "NeedsLogin", + want: true, + }, + { + name: "running_with_login_warning", + backendState: "Running", + health: []string{"warning: " + loginWarnableCode + ": you are logged out"}, + want: true, + }, + { + name: "running_with_unrelated_warning", + backendState: "Running", + health: []string{"dns-not-working"}, + want: false, + }, + { + name: "running_no_warnings", + backendState: "Running", + health: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := needsAuthKeyReissue(tt.backendState, tt.health) + if got != tt.want { + t.Errorf("needsAuthKeyReissue() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/mkmanifest/main.go b/cmd/mkmanifest/main.go index fb3c729f12d21..d08700341e7dc 100644 --- a/cmd/mkmanifest/main.go +++ b/cmd/mkmanifest/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The mkmanifest command is a simple helper utility to create a '.syso' file diff --git a/cmd/mkpkg/main.go b/cmd/mkpkg/main.go index 5e26b07f8f9f8..ecf108c2ec236 100644 --- a/cmd/mkpkg/main.go +++ b/cmd/mkpkg/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // mkpkg builds the Tailscale rpm and deb packages. @@ -24,7 +24,7 @@ func parseFiles(s string, typ string) (files.Contents, error) { return nil, nil } var contents files.Contents - for _, f := range strings.Split(s, ",") { + for f := range strings.SplitSeq(s, ",") { fs := strings.Split(f, ":") if len(fs) != 2 { return nil, fmt.Errorf("unparseable file field %q", f) @@ -41,7 +41,7 @@ func parseEmptyDirs(s string) files.Contents { return nil } var contents files.Contents - for _, d := range strings.Split(s, ",") { + for d := range strings.SplitSeq(s, ",") { contents = append(contents, &files.Content{Type: files.TypeDir, Destination: d}) } return contents diff --git a/cmd/mkversion/mkversion.go b/cmd/mkversion/mkversion.go index c8c8bf17930f6..ec9b0bb85ace4 100644 --- a/cmd/mkversion/mkversion.go +++ b/cmd/mkversion/mkversion.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // mkversion gets version info from git and outputs a bunch of shell variables diff --git a/cmd/nardump/nardump.go b/cmd/nardump/nardump.go index f8947b02b852c..38a2a67319595 100644 --- a/cmd/nardump/nardump.go +++ b/cmd/nardump/nardump.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // nardump is like nix-store --dump, but in Go, writing a NAR @@ -9,22 +9,13 @@ // git-pull-oss.sh having Nix available. package main -// For the format, see: -// See https://gist.github.com/jbeda/5c79d2b1434f0018d693 - import ( - "bufio" - "crypto/sha256" - "encoding/base64" - "encoding/binary" "flag" "fmt" - "io" - "io/fs" "log" "os" - "path" - "sort" + + "tailscale.com/cmd/nardump/nardump" ) var sri = flag.Bool("sri", false, "print SRI") @@ -34,167 +25,16 @@ func main() { if flag.NArg() != 1 { log.Fatal("usage: nardump ") } - arg := flag.Arg(0) - if err := os.Chdir(arg); err != nil { - log.Fatal(err) - } + fsys := os.DirFS(flag.Arg(0)) if *sri { - hash := sha256.New() - if err := writeNAR(hash, os.DirFS(".")); err != nil { + s, err := nardump.SRI(fsys) + if err != nil { log.Fatal(err) } - fmt.Printf("sha256-%s\n", base64.StdEncoding.EncodeToString(hash.Sum(nil))) + fmt.Println(s) return } - bw := bufio.NewWriter(os.Stdout) - if err := writeNAR(bw, os.DirFS(".")); err != nil { + if err := nardump.WriteNAR(os.Stdout, fsys); err != nil { log.Fatal(err) } - bw.Flush() -} - -// writeNARError is a sentinel panic type that's recovered by writeNAR -// and converted into the wrapped error. -type writeNARError struct{ err error } - -// narWriter writes NAR files. -type narWriter struct { - w io.Writer - fs fs.FS -} - -// writeNAR writes a NAR file to w from the root of fs. -func writeNAR(w io.Writer, fs fs.FS) (err error) { - defer func() { - if e := recover(); e != nil { - if we, ok := e.(writeNARError); ok { - err = we.err - return - } - panic(e) - } - }() - nw := &narWriter{w: w, fs: fs} - nw.str("nix-archive-1") - return nw.writeDir(".") -} - -func (nw *narWriter) writeDir(dirPath string) error { - ents, err := fs.ReadDir(nw.fs, dirPath) - if err != nil { - return err - } - sort.Slice(ents, func(i, j int) bool { - return ents[i].Name() < ents[j].Name() - }) - nw.str("(") - nw.str("type") - nw.str("directory") - for _, ent := range ents { - nw.str("entry") - nw.str("(") - nw.str("name") - nw.str(ent.Name()) - nw.str("node") - mode := ent.Type() - sub := path.Join(dirPath, ent.Name()) - var err error - switch { - case mode.IsDir(): - err = nw.writeDir(sub) - case mode.IsRegular(): - err = nw.writeRegular(sub) - case mode&os.ModeSymlink != 0: - err = nw.writeSymlink(sub) - default: - return fmt.Errorf("unsupported file type %v at %q", sub, mode) - } - if err != nil { - return err - } - nw.str(")") - } - nw.str(")") - return nil -} - -func (nw *narWriter) writeRegular(path string) error { - nw.str("(") - nw.str("type") - nw.str("regular") - fi, err := fs.Stat(nw.fs, path) - if err != nil { - return err - } - if fi.Mode()&0111 != 0 { - nw.str("executable") - nw.str("") - } - contents, err := fs.ReadFile(nw.fs, path) - if err != nil { - return err - } - nw.str("contents") - if err := writeBytes(nw.w, contents); err != nil { - return err - } - nw.str(")") - return nil -} - -func (nw *narWriter) writeSymlink(path string) error { - nw.str("(") - nw.str("type") - nw.str("symlink") - nw.str("target") - // broken symlinks are valid in a nar - // given we do os.chdir(dir) and os.dirfs(".") above - // readlink now resolves relative links even if they are broken - link, err := os.Readlink(path) - if err != nil { - return err - } - nw.str(link) - nw.str(")") - return nil -} - -func (nw *narWriter) str(s string) { - if err := writeString(nw.w, s); err != nil { - panic(writeNARError{err}) - } -} - -func writeString(w io.Writer, s string) error { - var buf [8]byte - binary.LittleEndian.PutUint64(buf[:], uint64(len(s))) - if _, err := w.Write(buf[:]); err != nil { - return err - } - if _, err := io.WriteString(w, s); err != nil { - return err - } - return writePad(w, len(s)) -} - -func writeBytes(w io.Writer, b []byte) error { - var buf [8]byte - binary.LittleEndian.PutUint64(buf[:], uint64(len(b))) - if _, err := w.Write(buf[:]); err != nil { - return err - } - if _, err := w.Write(b); err != nil { - return err - } - return writePad(w, len(b)) -} - -func writePad(w io.Writer, n int) error { - pad := n % 8 - if pad == 0 { - return nil - } - var zeroes [8]byte - _, err := w.Write(zeroes[:8-pad]) - return err } diff --git a/cmd/nardump/nardump/nardump.go b/cmd/nardump/nardump/nardump.go new file mode 100644 index 0000000000000..ab9ff1f3cdcd8 --- /dev/null +++ b/cmd/nardump/nardump/nardump.go @@ -0,0 +1,193 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package nardump writes a NAR (Nix Archive) representation of an +// fs.FS to an io.Writer, or summarizes it as a Subresource Integrity +// hash, as used by Nix flake.nix vendor and toolchain hashes. +// +// For the format, see: +// https://gist.github.com/jbeda/5c79d2b1434f0018d693 +package nardump + +import ( + "bufio" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "io/fs" + "path" + "sort" +) + +// WriteNAR writes a NAR-encoded representation of fsys, rooted at +// the FS root, to w. +// +// The encoder issues many small writes; if w is not already a +// *bufio.Writer, WriteNAR wraps it in one and flushes on return so +// the caller doesn't have to. +// +// fsys must implement fs.ReadLinkFS to encode any symlinks it +// contains; os.DirFS satisfies this on Go 1.25+. +func WriteNAR(w io.Writer, fsys fs.FS) (err error) { + defer func() { + if e := recover(); e != nil { + if we, ok := e.(writeNARError); ok { + err = we.err + return + } + panic(e) + } + }() + bw, ok := w.(*bufio.Writer) + if !ok { + bw = bufio.NewWriter(w) + defer func() { + if flushErr := bw.Flush(); err == nil { + err = flushErr + } + }() + } + nw := &narWriter{w: bw, fs: fsys} + nw.str("nix-archive-1") + return nw.writeDir(".") +} + +// SRI returns the Subresource Integrity hash of the NAR encoding of +// fsys, in the form "sha256-". This is the format Nix +// expects for vendorHash and similar fields. +func SRI(fsys fs.FS) (string, error) { + h := sha256.New() + if err := WriteNAR(h, fsys); err != nil { + return "", err + } + return "sha256-" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil +} + +// writeNARError is a sentinel panic type that's recovered by +// WriteNAR and converted into the wrapped error. +type writeNARError struct{ err error } + +// narWriter writes NAR files. +type narWriter struct { + w io.Writer + fs fs.FS +} + +func (nw *narWriter) writeDir(dirPath string) error { + ents, err := fs.ReadDir(nw.fs, dirPath) + if err != nil { + return err + } + sort.Slice(ents, func(i, j int) bool { + return ents[i].Name() < ents[j].Name() + }) + nw.str("(") + nw.str("type") + nw.str("directory") + for _, ent := range ents { + nw.str("entry") + nw.str("(") + nw.str("name") + nw.str(ent.Name()) + nw.str("node") + mode := ent.Type() + sub := path.Join(dirPath, ent.Name()) + var err error + switch { + case mode.IsDir(): + err = nw.writeDir(sub) + case mode.IsRegular(): + err = nw.writeRegular(sub) + case mode&fs.ModeSymlink != 0: + err = nw.writeSymlink(sub) + default: + return fmt.Errorf("unsupported file type %v at %q", sub, mode) + } + if err != nil { + return err + } + nw.str(")") + } + nw.str(")") + return nil +} + +func (nw *narWriter) writeRegular(p string) error { + nw.str("(") + nw.str("type") + nw.str("regular") + fi, err := fs.Stat(nw.fs, p) + if err != nil { + return err + } + if fi.Mode()&0111 != 0 { + nw.str("executable") + nw.str("") + } + contents, err := fs.ReadFile(nw.fs, p) + if err != nil { + return err + } + nw.str("contents") + if err := writeBytes(nw.w, contents); err != nil { + return err + } + nw.str(")") + return nil +} + +func (nw *narWriter) writeSymlink(p string) error { + nw.str("(") + nw.str("type") + nw.str("symlink") + nw.str("target") + link, err := fs.ReadLink(nw.fs, p) + if err != nil { + return err + } + nw.str(link) + nw.str(")") + return nil +} + +func (nw *narWriter) str(s string) { + if err := writeString(nw.w, s); err != nil { + panic(writeNARError{err}) + } +} + +func writeString(w io.Writer, s string) error { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(len(s))) + if _, err := w.Write(buf[:]); err != nil { + return err + } + if _, err := io.WriteString(w, s); err != nil { + return err + } + return writePad(w, len(s)) +} + +func writeBytes(w io.Writer, b []byte) error { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(len(b))) + if _, err := w.Write(buf[:]); err != nil { + return err + } + if _, err := w.Write(b); err != nil { + return err + } + return writePad(w, len(b)) +} + +func writePad(w io.Writer, n int) error { + pad := n % 8 + if pad == 0 { + return nil + } + var zeroes [8]byte + _, err := w.Write(zeroes[:8-pad]) + return err +} diff --git a/cmd/nardump/nardump/nardump_test.go b/cmd/nardump/nardump/nardump_test.go new file mode 100644 index 0000000000000..16b690ee257f0 --- /dev/null +++ b/cmd/nardump/nardump/nardump_test.go @@ -0,0 +1,55 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package nardump + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" +) + +// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar. +func setupTmpdir(t *testing.T) string { + t.Helper() + tmpdir := t.TempDir() + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(os.MkdirAll(filepath.Join(tmpdir, "sub/dir"), 0755)) + must(os.Symlink("brokenfile", filepath.Join(tmpdir, "brokenlink"))) + must(os.Symlink("sub/dir", filepath.Join(tmpdir, "dirl"))) + must(os.Symlink("/abs/nonexistentdir", filepath.Join(tmpdir, "dirb"))) + f, err := os.Create(filepath.Join(tmpdir, "sub/dir/file1")) + must(err) + f.Close() + f, err = os.Create(filepath.Join(tmpdir, "file2m")) + must(err) + must(f.Truncate(2 * 1024 * 1024)) + f.Close() + must(os.Symlink("../file2m", filepath.Join(tmpdir, "sub/goodlink"))) + return tmpdir +} + +func TestWriteNAR(t *testing.T) { + if runtime.GOOS == "windows" { + // Skip test on Windows as the Nix package manager is not supported on this platform + t.Skip("nix package manager is not available on Windows") + } + dir := setupTmpdir(t) + // obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir + const expected = "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442" + h := sha256.New() + if err := WriteNAR(h, os.DirFS(dir)); err != nil { + t.Fatal(err) + } + if got := fmt.Sprintf("%x", h.Sum(nil)); got != expected { + t.Fatalf("sha256sum of nar: got %s, want %s", got, expected) + } +} diff --git a/cmd/nardump/nardump_test.go b/cmd/nardump/nardump_test.go deleted file mode 100644 index 3b87e7962d638..0000000000000 --- a/cmd/nardump/nardump_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package main - -import ( - "crypto/sha256" - "fmt" - "os" - "runtime" - "testing" -) - -// setupTmpdir sets up a known golden layout, covering all allowed file/folder types in a nar -func setupTmpdir(t *testing.T) string { - tmpdir := t.TempDir() - pwd, _ := os.Getwd() - os.Chdir(tmpdir) - defer os.Chdir(pwd) - os.MkdirAll("sub/dir", 0755) - os.Symlink("brokenfile", "brokenlink") - os.Symlink("sub/dir", "dirl") - os.Symlink("/abs/nonexistentdir", "dirb") - os.Create("sub/dir/file1") - f, _ := os.Create("file2m") - _ = f.Truncate(2 * 1024 * 1024) - f.Close() - os.Symlink("../file2m", "sub/goodlink") - return tmpdir -} - -func TestWriteNar(t *testing.T) { - if runtime.GOOS == "windows" { - // Skip test on Windows as the Nix package manager is not supported on this platform - t.Skip("nix package manager is not available on Windows") - } - dir := setupTmpdir(t) - t.Run("nar", func(t *testing.T) { - // obtained via `nix-store --dump /tmp/... | sha256sum` of the above test dir - expected := "727613a36f41030e93a4abf2649c3ec64a2757ccff364e3f6f7d544eb976e442" - h := sha256.New() - os.Chdir(dir) - err := writeNAR(h, os.DirFS(".")) - if err != nil { - t.Fatal(err) - } - hash := fmt.Sprintf("%x", h.Sum(nil)) - if expected != hash { - t.Fatal("sha256sum of nar not matched", hash, expected) - } - }) -} diff --git a/cmd/natc/ippool/consensusippool.go b/cmd/natc/ippool/consensusippool.go index 3bc21bd0357dd..d595d3e7ddc7a 100644 --- a/cmd/natc/ippool/consensusippool.go +++ b/cmd/natc/ippool/consensusippool.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package ippool @@ -30,6 +30,7 @@ type ConsensusIPPool struct { IPSet *netipx.IPSet perPeerMap *syncs.Map[tailcfg.NodeID, *consensusPerPeerState] consensus commandExecutor + clusterController clusterController unusedAddressLifetime time.Duration } @@ -149,16 +150,26 @@ func (ipp *ConsensusIPPool) domainLookup(from tailcfg.NodeID, addr netip.Addr) ( return ww, true } +type ClusterOpts struct { + Tag string + StateDir string + FollowOnly bool +} + // StartConsensus is part of the IPPool interface. It starts the raft background routines that handle consensus. -func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string, clusterStateDir string) error { +func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, opts ClusterOpts) error { cfg := tsconsensus.DefaultConfig() cfg.ServeDebugMonitor = true - cfg.StateDirPath = clusterStateDir - cns, err := tsconsensus.Start(ctx, ts, ipp, clusterTag, cfg) + cfg.StateDirPath = opts.StateDir + cns, err := tsconsensus.Start(ctx, ts, ipp, tsconsensus.BootstrapOpts{ + Tag: opts.Tag, + FollowOnly: opts.FollowOnly, + }, cfg) if err != nil { return err } ipp.consensus = cns + ipp.clusterController = cns return nil } @@ -411,9 +422,9 @@ func (ipp *ConsensusIPPool) applyCheckoutAddr(nid tailcfg.NodeID, domain string, } // Apply is part of the raft.FSM interface. It takes an incoming log entry and applies it to the state. -func (ipp *ConsensusIPPool) Apply(l *raft.Log) any { +func (ipp *ConsensusIPPool) Apply(lg *raft.Log) any { var c tsconsensus.Command - if err := json.Unmarshal(l.Data, &c); err != nil { + if err := json.Unmarshal(lg.Data, &c); err != nil { panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error())) } switch c.Name { @@ -433,3 +444,18 @@ func (ipp *ConsensusIPPool) Apply(l *raft.Log) any { type commandExecutor interface { ExecuteCommand(tsconsensus.Command) (tsconsensus.CommandResult, error) } + +type clusterController interface { + GetClusterConfiguration() (raft.Configuration, error) + DeleteClusterServer(id raft.ServerID) (uint64, error) +} + +// GetClusterConfiguration gets the consensus implementation's cluster configuration +func (ipp *ConsensusIPPool) GetClusterConfiguration() (raft.Configuration, error) { + return ipp.clusterController.GetClusterConfiguration() +} + +// DeleteClusterServer removes a server from the consensus implementation's cluster configuration +func (ipp *ConsensusIPPool) DeleteClusterServer(id raft.ServerID) (uint64, error) { + return ipp.clusterController.DeleteClusterServer(id) +} diff --git a/cmd/natc/ippool/consensusippool_test.go b/cmd/natc/ippool/consensusippool_test.go index 242cdffaf26d3..fe42b2b223a8b 100644 --- a/cmd/natc/ippool/consensusippool_test.go +++ b/cmd/natc/ippool/consensusippool_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package ippool diff --git a/cmd/natc/ippool/consensusippoolserialize.go b/cmd/natc/ippool/consensusippoolserialize.go index 97dc02f2c7d7c..be3312d300bad 100644 --- a/cmd/natc/ippool/consensusippoolserialize.go +++ b/cmd/natc/ippool/consensusippoolserialize.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package ippool diff --git a/cmd/natc/ippool/ippool.go b/cmd/natc/ippool/ippool.go index 5a2dcbec911e0..641702f5d31e8 100644 --- a/cmd/natc/ippool/ippool.go +++ b/cmd/natc/ippool/ippool.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // ippool implements IP address storage, creation, and retrieval for cmd/natc diff --git a/cmd/natc/ippool/ippool_test.go b/cmd/natc/ippool/ippool_test.go index 8d474f86a97ed..af0053c2f54d8 100644 --- a/cmd/natc/ippool/ippool_test.go +++ b/cmd/natc/ippool/ippool_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package ippool @@ -30,7 +30,7 @@ func TestIPPoolExhaustion(t *testing.T) { from := tailcfg.NodeID(12345) - for i := 0; i < 5; i++ { + for range 5 { for _, domain := range domains { addr, err := pool.IPForDomain(from, domain) if err != nil { diff --git a/cmd/natc/ippool/ipx.go b/cmd/natc/ippool/ipx.go index 8259a56dbf30e..4f52d6ede049a 100644 --- a/cmd/natc/ippool/ipx.go +++ b/cmd/natc/ippool/ipx.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package ippool diff --git a/cmd/natc/ippool/ipx_test.go b/cmd/natc/ippool/ipx_test.go index 2e2b9d3d45baf..cb6889b683978 100644 --- a/cmd/natc/ippool/ipx_test.go +++ b/cmd/natc/ippool/ipx_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package ippool diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index fdbce3da189b2..877f16cc02689 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The natc command is a work-in-progress implementation of a NAT based @@ -8,6 +8,7 @@ package main import ( "context" + "encoding/json" "errors" "expvar" "flag" @@ -23,6 +24,7 @@ import ( "time" "github.com/gaissmai/bart" + "github.com/hashicorp/raft" "github.com/inetaf/tcpproxy" "github.com/peterbourgon/ff/v3" "go4.org/netipx" @@ -50,18 +52,20 @@ func main() { // Parse flags fs := flag.NewFlagSet("natc", flag.ExitOnError) var ( - debugPort = fs.Int("debug-port", 8893, "Listening port for debug/metrics endpoint") - hostname = fs.String("hostname", "", "Hostname to register the service under") - siteID = fs.Uint("site-id", 1, "an integer site ID to use for the ULA prefix which allows for multiple proxies to act in a HA configuration") - v4PfxStr = fs.String("v4-pfx", "100.64.1.0/24", "comma-separated list of IPv4 prefixes to advertise") - dnsServers = fs.String("dns-servers", "", "comma separated list of upstream DNS to use, including host and port (use system if empty)") - verboseTSNet = fs.Bool("verbose-tsnet", false, "enable verbose logging in tsnet") - printULA = fs.Bool("print-ula", false, "print the ULA prefix and exit") - ignoreDstPfxStr = fs.String("ignore-destinations", "", "comma-separated list of prefixes to ignore") - wgPort = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic") - clusterTag = fs.String("cluster-tag", "", "optionally run in a consensus cluster with other nodes with this tag") - server = fs.String("login-server", ipn.DefaultControlURL, "the base URL of control server") - stateDir = fs.String("state-dir", "", "path to directory in which to store app state") + debugPort = fs.Int("debug-port", 8893, "Listening port for debug/metrics endpoint") + hostname = fs.String("hostname", "", "Hostname to register the service under") + siteID = fs.Uint("site-id", 1, "an integer site ID to use for the ULA prefix which allows for multiple proxies to act in a HA configuration") + v4PfxStr = fs.String("v4-pfx", "100.64.1.0/24", "comma-separated list of IPv4 prefixes to advertise") + dnsServers = fs.String("dns-servers", "", "comma separated list of upstream DNS to use, including host and port (use system if empty)") + verboseTSNet = fs.Bool("verbose-tsnet", false, "enable verbose logging in tsnet") + printULA = fs.Bool("print-ula", false, "print the ULA prefix and exit") + ignoreDstPfxStr = fs.String("ignore-destinations", "", "comma-separated list of prefixes to ignore") + wgPort = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic") + clusterTag = fs.String("cluster-tag", "", "optionally run in a consensus cluster with other nodes with this tag") + server = fs.String("login-server", ipn.DefaultControlURL, "the base URL of control server") + stateDir = fs.String("state-dir", "", "path to directory in which to store app state") + clusterFollowOnly = fs.Bool("follow-only", false, "Try to find a leader with the cluster tag or exit.") + clusterAdminPort = fs.Int("cluster-admin-port", 8081, "Port on localhost for the cluster admin HTTP API") ) ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC")) @@ -78,14 +82,14 @@ func main() { log.Fatalf("site-id must be in the range [0, 65535]") } - var ignoreDstTable *bart.Table[bool] + var ignoreDstTable *bart.Lite for s := range strings.SplitSeq(*ignoreDstPfxStr, ",") { s := strings.TrimSpace(s) if s == "" { continue } if ignoreDstTable == nil { - ignoreDstTable = &bart.Table[bool]{} + ignoreDstTable = &bart.Lite{} } pfx, err := netip.ParsePrefix(s) if err != nil { @@ -94,7 +98,7 @@ func main() { if pfx.Masked() != pfx { log.Fatalf("prefix %v is not normalized (bits are set outside the mask)", pfx) } - ignoreDstTable.Insert(pfx, true) + ignoreDstTable.Insert(pfx) } ts := &tsnet.Server{ Hostname: *hostname, @@ -145,7 +149,7 @@ func main() { } var prefixes []netip.Prefix - for _, s := range strings.Split(*v4PfxStr, ",") { + for s := range strings.SplitSeq(*v4PfxStr, ",") { p := netip.MustParsePrefix(strings.TrimSpace(s)) if p.Masked() != p { log.Fatalf("v4 prefix %v is not a masked prefix", p) @@ -163,7 +167,11 @@ func main() { if err != nil { log.Fatalf("Creating cluster state dir failed: %v", err) } - err = cipp.StartConsensus(ctx, ts, *clusterTag, clusterStateDir) + err = cipp.StartConsensus(ctx, ts, ippool.ClusterOpts{ + Tag: *clusterTag, + StateDir: clusterStateDir, + FollowOnly: *clusterFollowOnly, + }) if err != nil { log.Fatalf("StartConsensus: %v", err) } @@ -174,6 +182,12 @@ func main() { } }() ipp = cipp + + go func() { + // This listens on localhost only, so that only those with access to the host machine + // can remove servers from the cluster config. + log.Print(http.ListenAndServe(fmt.Sprintf("127.0.0.1:%d", *clusterAdminPort), httpClusterAdmin(cipp))) + }() } else { ipp = &ippool.SingleMachineIPPool{IPSet: addrPool} } @@ -262,7 +276,7 @@ type connector struct { // and if any of the ip addresses in response to the lookup match any 'ignore destinations' prefix we will // return a dns response that contains the ip addresses we discovered with the lookup (ie not the // natc behavior, which would return a dummy ip address pointing at natc). - ignoreDsts *bart.Table[bool] + ignoreDsts *bart.Lite // ipPool contains the per-peer IPv4 address assignments. ipPool ippool.IPPool @@ -358,8 +372,7 @@ func (c *connector) handleDNS(pc net.PacketConn, buf []byte, remoteAddr *net.UDP addrQCount++ if _, ok := resolves[q.Name.String()]; !ok { addrs, err := c.resolver.LookupNetIP(ctx, "ip", q.Name.String()) - var dnsErr *net.DNSError - if errors.As(err, &dnsErr) && dnsErr.IsNotFound { + if dnsErr, ok := errors.AsType[*net.DNSError](err); ok && dnsErr.IsNotFound { continue } if err != nil { @@ -525,7 +538,7 @@ func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool { return false } for _, a := range dstAddrs { - if _, ok := c.ignoreDsts.Lookup(a); ok { + if c.ignoreDsts.Contains(a) { return true } } @@ -628,3 +641,32 @@ func getClusterStatePath(stateDirFlag string) (string, error) { return dirPath, nil } + +func httpClusterAdmin(ipp *ippool.ConsensusIPPool) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { + c, err := ipp.GetClusterConfiguration() + if err != nil { + log.Printf("cluster admin http: error getClusterConfig: %v", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + if err := json.NewEncoder(w).Encode(c); err != nil { + log.Printf("cluster admin http: error encoding raft configuration: %v", err) + } + }) + mux.HandleFunc("DELETE /{id}", func(w http.ResponseWriter, r *http.Request) { + idString := r.PathValue("id") + id := raft.ServerID(idString) + idx, err := ipp.DeleteClusterServer(id) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := json.NewEncoder(w).Encode(idx); err != nil { + log.Printf("cluster admin http: error encoding delete index: %v", err) + return + } + }) + return mux +} diff --git a/cmd/natc/natc_test.go b/cmd/natc/natc_test.go index c0a66deb8a4da..00c94868ec8a2 100644 --- a/cmd/natc/natc_test.go +++ b/cmd/natc/natc_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -268,13 +268,13 @@ func TestDNSResponse(t *testing.T) { }, }, }, - ignoreDsts: &bart.Table[bool]{}, + ignoreDsts: &bart.Lite{}, routes: routes, v6ULA: v6ULA, ipPool: &ippool.SingleMachineIPPool{IPSet: addrPool}, dnsAddr: dnsAddr, } - c.ignoreDsts.Insert(netip.MustParsePrefix("8.8.4.4/32"), true) + c.ignoreDsts.Insert(netip.MustParsePrefix("8.8.4.4/32")) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -411,9 +411,9 @@ func TestDNSResponse(t *testing.T) { } func TestIgnoreDestination(t *testing.T) { - ignoreDstTable := &bart.Table[bool]{} - ignoreDstTable.Insert(netip.MustParsePrefix("192.168.1.0/24"), true) - ignoreDstTable.Insert(netip.MustParsePrefix("10.0.0.0/8"), true) + ignoreDstTable := &bart.Lite{} + ignoreDstTable.Insert(netip.MustParsePrefix("192.168.1.0/24")) + ignoreDstTable.Insert(netip.MustParsePrefix("10.0.0.0/8")) c := &connector{ ignoreDsts: ignoreDstTable, diff --git a/cmd/netlogfmt/main.go b/cmd/netlogfmt/main.go index 65e87098fec5e..af7baae4676bd 100644 --- a/cmd/netlogfmt/main.go +++ b/cmd/netlogfmt/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // netlogfmt parses a stream of JSON log messages from stdin and @@ -44,25 +44,53 @@ import ( "github.com/dsnet/try" jsonv2 "github.com/go-json-experiment/json" "github.com/go-json-experiment/json/jsontext" + "tailscale.com/tailcfg" + "tailscale.com/tstime" + "tailscale.com/types/bools" "tailscale.com/types/logid" "tailscale.com/types/netlogtype" "tailscale.com/util/must" ) var ( - resolveNames = flag.Bool("resolve-names", false, "convert tailscale IP addresses to hostnames; must also specify --api-key and --tailnet-id") - apiKey = flag.String("api-key", "", "API key to query the Tailscale API with; see https://login.tailscale.com/admin/settings/keys") - tailnetName = flag.String("tailnet-name", "", "tailnet domain name to lookup devices in; see https://login.tailscale.com/admin/settings/general") + resolveNames = flag.Bool("resolve-names", false, "This is equivalent to specifying \"--resolve-addrs=name\".") + resolveAddrs = flag.String("resolve-addrs", "", "Resolve each tailscale IP address as a node ID, name, or user.\n"+ + "If network flow logs do not support embedded node information,\n"+ + "then --api-key and --tailnet-name must also be provided.\n"+ + "Valid values include \"nodeId\", \"name\", or \"user\".") + apiKey = flag.String("api-key", "", "The API key to query the Tailscale API with.\nSee https://login.tailscale.com/admin/settings/keys") + tailnetName = flag.String("tailnet-name", "", "The Tailnet name to lookup nodes within.\nSee https://login.tailscale.com/admin/settings/general") ) -var namesByAddr map[netip.Addr]string +var ( + tailnetNodesByAddr map[netip.Addr]netlogtype.Node + tailnetNodesByID map[tailcfg.StableNodeID]netlogtype.Node +) func main() { flag.Parse() if *resolveNames { - namesByAddr = mustMakeNamesByAddr() + *resolveAddrs = "name" + } + *resolveAddrs = strings.ToLower(*resolveAddrs) // make case-insensitive + *resolveAddrs = strings.TrimSuffix(*resolveAddrs, "s") // allow plural form + *resolveAddrs = strings.ReplaceAll(*resolveAddrs, " ", "") // ignore spaces + *resolveAddrs = strings.ReplaceAll(*resolveAddrs, "-", "") // ignore dashes + *resolveAddrs = strings.ReplaceAll(*resolveAddrs, "_", "") // ignore underscores + switch *resolveAddrs { + case "": + case "id", "nodeid": + *resolveAddrs = "nodeid" + case "name", "hostname": + *resolveAddrs = "name" + case "user", "tag", "usertag", "taguser": + *resolveAddrs = "user" // tag resolution is implied + default: + log.Fatalf("--resolve-addrs must be \"nodeId\", \"name\", or \"user\"") } + mustLoadTailnetNodes() + // The logic handles a stream of arbitrary JSON. // So long as a JSON object seems like a network log message, // then this will unmarshal and print it. @@ -103,7 +131,7 @@ func processArray(dec *jsontext.Decoder) { func processObject(dec *jsontext.Decoder) { var hasTraffic bool - var rawMsg []byte + var rawMsg jsontext.Value try.E1(dec.ReadToken()) // parse '{' for dec.PeekKind() != '}' { // Capture any members that could belong to a network log message. @@ -111,13 +139,13 @@ func processObject(dec *jsontext.Decoder) { case "virtualTraffic", "subnetTraffic", "exitTraffic", "physicalTraffic": hasTraffic = true fallthrough - case "logtail", "nodeId", "logged", "start", "end": + case "logtail", "nodeId", "logged", "srcNode", "dstNodes", "start", "end": if len(rawMsg) == 0 { rawMsg = append(rawMsg, '{') } else { rawMsg = append(rawMsg[:len(rawMsg)-1], ',') } - rawMsg = append(append(append(rawMsg, '"'), name.String()...), '"') + rawMsg, _ = jsontext.AppendQuote(rawMsg, name.String()) rawMsg = append(rawMsg, ':') rawMsg = append(rawMsg, try.E1(dec.ReadValue())...) rawMsg = append(rawMsg, '}') @@ -145,6 +173,32 @@ type message struct { } func printMessage(msg message) { + var nodesByAddr map[netip.Addr]netlogtype.Node + var tailnetDNS string // e.g., ".acme-corp.ts.net" + if *resolveAddrs != "" { + nodesByAddr = make(map[netip.Addr]netlogtype.Node) + insertNode := func(node netlogtype.Node) { + for _, addr := range node.Addresses { + nodesByAddr[addr] = node + } + } + for _, node := range msg.DstNodes { + insertNode(node) + } + insertNode(msg.SrcNode) + + // Derive the Tailnet DNS of the self node. + detectTailnetDNS := func(nodeName string) { + if prefix, ok := strings.CutSuffix(nodeName, ".ts.net"); ok { + if i := strings.LastIndexByte(prefix, '.'); i > 0 { + tailnetDNS = nodeName[i:] + } + } + } + detectTailnetDNS(msg.SrcNode.Name) + detectTailnetDNS(tailnetNodesByID[msg.NodeID].Name) + } + // Construct a table of network traffic per connection. rows := [][7]string{{3: "Tx[P/s]", 4: "Tx[B/s]", 5: "Rx[P/s]", 6: "Rx[B/s]"}} duration := msg.End.Sub(msg.Start) @@ -175,16 +229,25 @@ func printMessage(msg message) { if !a.IsValid() { return "" } - if name, ok := namesByAddr[a.Addr()]; ok { - if a.Port() == 0 { - return name + name := a.Addr().String() + node, ok := tailnetNodesByAddr[a.Addr()] + if !ok { + node, ok = nodesByAddr[a.Addr()] + } + if ok { + switch *resolveAddrs { + case "nodeid": + name = cmp.Or(string(node.NodeID), name) + case "name": + name = cmp.Or(strings.TrimSuffix(string(node.Name), tailnetDNS), name) + case "user": + name = cmp.Or(bools.IfElse(len(node.Tags) > 0, fmt.Sprint(node.Tags), node.User), name) } - return name + ":" + strconv.Itoa(int(a.Port())) } - if a.Port() == 0 { - return a.Addr().String() + if a.Port() != 0 { + return name + ":" + strconv.Itoa(int(a.Port())) } - return a.String() + return name } for _, cc := range traffic { row := [7]string{ @@ -232,7 +295,7 @@ func printMessage(msg message) { fmt.Printf("NodeID: %s\n", msg.NodeID) } formatTime := func(t time.Time) string { - return t.In(time.Local).Format("2006-01-02 15:04:05.000") + return t.Local().Format(tstime.DateSpTimeMilliZ) } switch { case !msg.Logged.IsZero(): @@ -279,8 +342,10 @@ func printMessage(msg message) { } } -func mustMakeNamesByAddr() map[netip.Addr]string { +func mustLoadTailnetNodes() { switch { + case *apiKey == "" && *tailnetName == "": + return // rely on embedded node information in the logs themselves case *apiKey == "": log.Fatalf("--api-key must be specified with --resolve-names") case *tailnetName == "": @@ -300,57 +365,19 @@ func mustMakeNamesByAddr() map[netip.Addr]string { // Unmarshal the API response. var m struct { - Devices []struct { - Name string `json:"name"` - Addrs []netip.Addr `json:"addresses"` - } `json:"devices"` + Devices []netlogtype.Node `json:"devices"` } must.Do(json.Unmarshal(b, &m)) - // Construct a unique mapping of Tailscale IP addresses to hostnames. - // For brevity, we start with the first segment of the name and - // use more segments until we find the shortest prefix that is unique - // for all names in the tailnet. - seen := make(map[string]bool) - namesByAddr := make(map[netip.Addr]string) -retry: - for i := range 10 { - clear(seen) - clear(namesByAddr) - for _, d := range m.Devices { - name := fieldPrefix(d.Name, i) - if seen[name] { - continue retry - } - seen[name] = true - for _, a := range d.Addrs { - namesByAddr[a] = name - } - } - return namesByAddr - } - panic("unable to produce unique mapping of address to names") -} - -// fieldPrefix returns the first n number of dot-separated segments. -// -// Example: -// -// fieldPrefix("foo.bar.baz", 0) returns "" -// fieldPrefix("foo.bar.baz", 1) returns "foo" -// fieldPrefix("foo.bar.baz", 2) returns "foo.bar" -// fieldPrefix("foo.bar.baz", 3) returns "foo.bar.baz" -// fieldPrefix("foo.bar.baz", 4) returns "foo.bar.baz" -func fieldPrefix(s string, n int) string { - s0 := s - for i := 0; i < n && len(s) > 0; i++ { - if j := strings.IndexByte(s, '.'); j >= 0 { - s = s[j+1:] - } else { - s = "" + // Construct a mapping of Tailscale IP addresses to node information. + tailnetNodesByAddr = make(map[netip.Addr]netlogtype.Node) + tailnetNodesByID = make(map[tailcfg.StableNodeID]netlogtype.Node) + for _, node := range m.Devices { + for _, addr := range node.Addresses { + tailnetNodesByAddr[addr] = node } + tailnetNodesByID[node.NodeID] = node } - return strings.TrimSuffix(s0[:len(s0)-len(s)], ".") } func appendRepeatByte(b []byte, c byte, n int) []byte { diff --git a/cmd/nginx-auth/nginx-auth.go b/cmd/nginx-auth/nginx-auth.go index 09da74da1d3c8..6b791eb6c35fa 100644 --- a/cmd/nginx-auth/nginx-auth.go +++ b/cmd/nginx-auth/nginx-auth.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build linux diff --git a/cmd/omitsize/omitsize.go b/cmd/omitsize/omitsize.go new file mode 100644 index 0000000000000..84863865991bc --- /dev/null +++ b/cmd/omitsize/omitsize.go @@ -0,0 +1,229 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// The omitsize tool prints out how large the Tailscale binaries are with +// different build tags. +package main + +import ( + "crypto/sha256" + "flag" + "fmt" + "log" + "maps" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + + "tailscale.com/feature/featuretags" + "tailscale.com/util/set" +) + +var ( + cacheDir = flag.String("cachedir", "", "if non-empty, use this directory to store cached size results to speed up subsequent runs. The tool does not consider the git status when deciding whether to use the cache. It's on you to nuke it between runs if the tree changed.") + features = flag.String("features", "", "comma-separated list of features to list in the table, without the ts_omit_ prefix. It may also contain a '+' sign(s) for ANDing features together. If empty, all omittable features are considered one at a time.") + + showRemovals = flag.Bool("show-removals", false, "if true, show a table of sizes removing one feature at a time from the full set.") +) + +// allOmittable returns the list of all build tags that remove features. +var allOmittable = sync.OnceValue(func() []string { + var ret []string // all build tags that can be omitted + for k := range featuretags.Features { + if k.IsOmittable() { + ret = append(ret, k.OmitTag()) + } + } + slices.Sort(ret) + return ret +}) + +func main() { + flag.Parse() + + // rows is a set (usually of size 1) of feature(s) to add/remove, without deps + // included at this point (as dep direction depends on whether we're adding or removing, + // so it's expanded later) + var rows []set.Set[featuretags.FeatureTag] + + if *features == "" { + for _, k := range slices.Sorted(maps.Keys(featuretags.Features)) { + if k.IsOmittable() { + rows = append(rows, set.Of(k)) + } + } + } else { + for v := range strings.SplitSeq(*features, ",") { + s := set.Set[featuretags.FeatureTag]{} + for fts := range strings.SplitSeq(v, "+") { + ft := featuretags.FeatureTag(fts) + if _, ok := featuretags.Features[ft]; !ok { + log.Fatalf("unknown feature %q", v) + } + s.Add(ft) + } + rows = append(rows, s) + } + } + + minD := measure("tailscaled", allOmittable()...) + minC := measure("tailscale", allOmittable()...) + minBoth := measure("tailscaled", append(slices.Clone(allOmittable()), "ts_include_cli")...) + + if *showRemovals { + baseD := measure("tailscaled") + baseC := measure("tailscale") + baseBoth := measure("tailscaled", "ts_include_cli") + + fmt.Printf("Starting with everything and removing a feature...\n\n") + + fmt.Printf("%9s %9s %9s\n", "tailscaled", "tailscale", "combined (linux/amd64)") + fmt.Printf("%9d %9d %9d\n", baseD, baseC, baseBoth) + + fmt.Printf("-%8d -%8d -%8d .. remove *\n", baseD-minD, baseC-minC, baseBoth-minBoth) + + for _, s := range rows { + title, tags := computeRemove(s) + sizeD := measure("tailscaled", tags...) + sizeC := measure("tailscale", tags...) + sizeBoth := measure("tailscaled", append(slices.Clone(tags), "ts_include_cli")...) + saveD := max(baseD-sizeD, 0) + saveC := max(baseC-sizeC, 0) + saveBoth := max(baseBoth-sizeBoth, 0) + fmt.Printf("-%8d -%8d -%8d .. remove %s\n", saveD, saveC, saveBoth, title) + + } + } + + fmt.Printf("\nStarting at a minimal binary and adding one feature back...\n\n") + fmt.Printf("%9s %9s %9s\n", "tailscaled", "tailscale", "combined (linux/amd64)") + fmt.Printf("%9d %9d %9d omitting everything\n", minD, minC, minBoth) + for _, s := range rows { + title, tags := computeAdd(s) + sizeD := measure("tailscaled", tags...) + sizeC := measure("tailscale", tags...) + sizeBoth := measure("tailscaled", append(tags, "ts_include_cli")...) + + fmt.Printf("+%8d +%8d +%8d .. add %s\n", max(sizeD-minD, 0), max(sizeC-minC, 0), max(sizeBoth-minBoth, 0), title) + } + +} + +// computeAdd returns a human-readable title of a set of features and the build +// tags to use to add that set of features to a minimal binary, including their +// feature dependencies. +func computeAdd(s set.Set[featuretags.FeatureTag]) (title string, tags []string) { + allSet := set.Set[featuretags.FeatureTag]{} // s + all their outbound dependencies + var explicitSorted []string // string versions of s, sorted + for ft := range s { + allSet.AddSet(featuretags.Requires(ft)) + if ft.IsOmittable() { + explicitSorted = append(explicitSorted, string(ft)) + } + } + slices.Sort(explicitSorted) + + var removeTags []string + for ft := range allSet { + if ft.IsOmittable() { + removeTags = append(removeTags, ft.OmitTag()) + } + } + + var titleBuf strings.Builder + titleBuf.WriteString(strings.Join(explicitSorted, "+")) + var and []string + for ft := range allSet { + if !s.Contains(ft) { + and = append(and, string(ft)) + } + } + if len(and) > 0 { + slices.Sort(and) + fmt.Fprintf(&titleBuf, " (and %s)", strings.Join(and, "+")) + } + tags = allExcept(allOmittable(), removeTags) + return titleBuf.String(), tags +} + +// computeRemove returns a human-readable title of a set of features and the build +// tags to use to remove that set of features from a full binary, including removing +// any features that depend on features in the provided set. +func computeRemove(s set.Set[featuretags.FeatureTag]) (title string, tags []string) { + allSet := set.Set[featuretags.FeatureTag]{} // s + all their inbound dependencies + var explicitSorted []string // string versions of s, sorted + for ft := range s { + allSet.AddSet(featuretags.RequiredBy(ft)) + if ft.IsOmittable() { + explicitSorted = append(explicitSorted, string(ft)) + } + } + slices.Sort(explicitSorted) + + var removeTags []string + for ft := range allSet { + if ft.IsOmittable() { + removeTags = append(removeTags, ft.OmitTag()) + } + } + + var titleBuf strings.Builder + titleBuf.WriteString(strings.Join(explicitSorted, "+")) + + var and []string + for ft := range allSet { + if !s.Contains(ft) { + and = append(and, string(ft)) + } + } + if len(and) > 0 { + slices.Sort(and) + fmt.Fprintf(&titleBuf, " (and %s)", strings.Join(and, "+")) + } + + return titleBuf.String(), removeTags +} + +func allExcept(all, omit []string) []string { + return slices.DeleteFunc(slices.Clone(all), func(s string) bool { return slices.Contains(omit, s) }) +} + +func measure(bin string, tags ...string) int64 { + tags = slices.Clone(tags) + slices.Sort(tags) + tags = slices.Compact(tags) + comma := strings.Join(tags, ",") + + var cacheFile string + if *cacheDir != "" { + cacheFile = filepath.Join(*cacheDir, fmt.Sprintf("%02x", sha256.Sum256(fmt.Appendf(nil, "%s-%s.size", bin, comma)))) + if v, err := os.ReadFile(cacheFile); err == nil { + if size, err := strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64); err == nil { + return size + } + } + } + + cmd := exec.Command("go", "build", "-trimpath", "-ldflags=-w -s", "-tags", strings.Join(tags, ","), "-o", "tmpbin", "./cmd/"+bin) + log.Printf("# Measuring %v", cmd.Args) + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH=amd64") + out, err := cmd.CombinedOutput() + if err != nil { + log.Fatalf("error measuring %q: %v, %s\n", bin, err, out) + } + fi, err := os.Stat("tmpbin") + if err != nil { + log.Fatal(err) + } + n := fi.Size() + if cacheFile != "" { + if err := os.WriteFile(cacheFile, fmt.Appendf(nil, "%d", n), 0644); err != nil { + log.Fatalf("error writing size to cache: %v\n", err) + } + } + return n +} diff --git a/cmd/pgproxy/pgproxy.go b/cmd/pgproxy/pgproxy.go index e102c8ae47411..a138eacdce4fc 100644 --- a/cmd/pgproxy/pgproxy.go +++ b/cmd/pgproxy/pgproxy.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The pgproxy server is a proxy for the Postgres wire protocol. @@ -291,7 +291,7 @@ func (p *proxy) serve(sessionID int64, c net.Conn) error { Certificates: p.downstreamCert, MinVersion: tls.VersionTLS12, }) - if err = uptc.HandshakeContext(ctx); err != nil { + if err = s.HandshakeContext(ctx); err != nil { p.errors.Add("client-tls", 1) return fmt.Errorf("client TLS handshake: %v", err) } diff --git a/cmd/printdep/printdep.go b/cmd/printdep/printdep.go index 044283209c08c..f5aeab7a561b6 100644 --- a/cmd/printdep/printdep.go +++ b/cmd/printdep/printdep.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The printdep command is a build system tool for printing out information @@ -19,6 +19,7 @@ var ( goToolchain = flag.Bool("go", false, "print the supported Go toolchain git hash (a github.com/tailscale/go commit)") goToolchainURL = flag.Bool("go-url", false, "print the URL to the tarball of the Tailscale Go toolchain") alpine = flag.Bool("alpine", false, "print the tag of alpine docker image") + next = flag.Bool("next", false, "if set, modifies --go or --go-url to use the upcoming/unreleased/rc Go release version instead") ) func main() { @@ -27,8 +28,12 @@ func main() { fmt.Println(strings.TrimSpace(ts.AlpineDockerTag)) return } + goRev := strings.TrimSpace(ts.GoToolchainRev) + if *next { + goRev = strings.TrimSpace(ts.GoToolchainNextRev) + } if *goToolchain { - fmt.Println(strings.TrimSpace(ts.GoToolchainRev)) + fmt.Println(goRev) } if *goToolchainURL { switch runtime.GOOS { @@ -36,6 +41,6 @@ func main() { default: log.Fatalf("unsupported GOOS %q", runtime.GOOS) } - fmt.Printf("https://github.com/tailscale/go/releases/download/build-%s/%s-%s.tar.gz\n", strings.TrimSpace(ts.GoToolchainRev), runtime.GOOS, runtime.GOARCH) + fmt.Printf("https://github.com/tailscale/go/releases/download/build-%s/%s-%s.tar.gz\n", goRev, runtime.GOOS, runtime.GOARCH) } } diff --git a/cmd/proxy-test-server/proxy-test-server.go b/cmd/proxy-test-server/proxy-test-server.go index 9f8c94a384ea5..2c705670446ba 100644 --- a/cmd/proxy-test-server/proxy-test-server.go +++ b/cmd/proxy-test-server/proxy-test-server.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The proxy-test-server command is a simple HTTP proxy server for testing diff --git a/cmd/proxy-to-grafana/proxy-to-grafana.go b/cmd/proxy-to-grafana/proxy-to-grafana.go index 27f5e338c8d65..23f2640597d59 100644 --- a/cmd/proxy-to-grafana/proxy-to-grafana.go +++ b/cmd/proxy-to-grafana/proxy-to-grafana.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // proxy-to-grafana is a reverse proxy which identifies users based on their diff --git a/cmd/proxy-to-grafana/proxy-to-grafana_test.go b/cmd/proxy-to-grafana/proxy-to-grafana_test.go index 4831d54364943..be217043f12d3 100644 --- a/cmd/proxy-to-grafana/proxy-to-grafana_test.go +++ b/cmd/proxy-to-grafana/proxy-to-grafana_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/sniproxy/handlers.go b/cmd/sniproxy/handlers.go index 1973eecc017a3..157b9b75f885a 100644 --- a/cmd/sniproxy/handlers.go +++ b/cmd/sniproxy/handlers.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/sniproxy/handlers_test.go b/cmd/sniproxy/handlers_test.go index 4f9fc6a34b184..ad0637421cecc 100644 --- a/cmd/sniproxy/handlers_test.go +++ b/cmd/sniproxy/handlers_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/sniproxy/server.go b/cmd/sniproxy/server.go index b322b6f4b1137..0ff301fe92136 100644 --- a/cmd/sniproxy/server.go +++ b/cmd/sniproxy/server.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/sniproxy/server_test.go b/cmd/sniproxy/server_test.go index d56f2aa754f85..8e06e8abedf8c 100644 --- a/cmd/sniproxy/server_test.go +++ b/cmd/sniproxy/server_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/sniproxy/sniproxy.go b/cmd/sniproxy/sniproxy.go index c020b4a1f1605..f7ebc6abaa4e5 100644 --- a/cmd/sniproxy/sniproxy.go +++ b/cmd/sniproxy/sniproxy.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The sniproxy is an outbound SNI proxy. It receives TLS connections over @@ -138,10 +138,10 @@ func run(ctx context.Context, ts *tsnet.Server, wgPort int, hostname string, pro } // Finally, start mainloop to configure app connector based on information - // in the netmap. - // We set the NotifyInitialNetMap flag so we will always get woken with the - // current netmap, before only being woken on changes. - bus, err := lc.WatchIPNBus(ctx, ipn.NotifyWatchEngineUpdates|ipn.NotifyInitialNetMap|ipn.NotifyNoPrivateKeys) + // in the self node's CapMap. We set NotifyInitialNetMap so the first + // Notify carries the current self node (now via Notify.SelfChange); + // subsequent self changes wake us up too. + bus, err := lc.WatchIPNBus(ctx, ipn.NotifyWatchEngineUpdates|ipn.NotifyInitialNetMap) if err != nil { log.Fatalf("watching IPN bus: %v", err) } @@ -155,28 +155,30 @@ func run(ctx context.Context, ts *tsnet.Server, wgPort int, hostname string, pro log.Fatalf("reading IPN bus: %v", err) } - // NetMap contains app-connector configuration - if nm := msg.NetMap; nm != nil && nm.SelfNode.Valid() { - var c appctype.AppConnectorConfig - nmConf, err := tailcfg.UnmarshalNodeCapViewJSON[appctype.AppConnectorConfig](nm.SelfNode.CapMap(), configCapKey) - if err != nil { - log.Printf("failed to read app connector configuration from coordination server: %v", err) - } else if len(nmConf) > 0 { - c = nmConf[0] - } + self := msg.SelfChange + if self == nil { + continue + } + var c appctype.AppConnectorConfig + // View() lets us reuse the existing CapView decoder. + nmConf, err := tailcfg.UnmarshalNodeCapViewJSON[appctype.AppConnectorConfig](self.View().CapMap(), configCapKey) + if err != nil { + log.Printf("failed to read app connector configuration from coordination server: %v", err) + } else if len(nmConf) > 0 { + c = nmConf[0] + } - if c.AdvertiseRoutes { - if err := s.advertiseRoutesFromConfig(ctx, &c); err != nil { - log.Printf("failed to advertise routes: %v", err) - } + if c.AdvertiseRoutes { + if err := s.advertiseRoutesFromConfig(ctx, &c); err != nil { + log.Printf("failed to advertise routes: %v", err) } - - // Backwards compatibility: combine any configuration from control with flags specified - // on the command line. This is intentionally done after we advertise any routes - // because its never correct to advertise the nodes native IP addresses. - s.mergeConfigFromFlags(&c, ports, forwards) - s.srv.Configure(&c) } + + // Backwards compatibility: combine any configuration from control with flags specified + // on the command line. This is intentionally done after we advertise any routes + // because its never correct to advertise the nodes native IP addresses. + s.mergeConfigFromFlags(&c, ports, forwards) + s.srv.Configure(&c) } } @@ -225,7 +227,7 @@ func (s *sniproxy) mergeConfigFromFlags(out *appctype.AppConnectorConfig, ports, Addrs: []netip.Addr{ip4, ip6}, } if ports != "" { - for _, portStr := range strings.Split(ports, ",") { + for portStr := range strings.SplitSeq(ports, ",") { port, err := strconv.ParseUint(portStr, 10, 16) if err != nil { log.Fatalf("invalid port: %s", portStr) @@ -238,7 +240,7 @@ func (s *sniproxy) mergeConfigFromFlags(out *appctype.AppConnectorConfig, ports, } var forwardConfigFromFlags []appctype.DNATConfig - for _, forwStr := range strings.Split(forwards, ",") { + for forwStr := range strings.SplitSeq(forwards, ",") { if forwStr == "" { continue } diff --git a/cmd/sniproxy/sniproxy_test.go b/cmd/sniproxy/sniproxy_test.go index cd2e070bd336f..a404799d29d7d 100644 --- a/cmd/sniproxy/sniproxy_test.go +++ b/cmd/sniproxy/sniproxy_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -152,17 +152,17 @@ func TestSNIProxyWithNetmapConfig(t *testing.T) { configCapKey: []tailcfg.RawMessage{tailcfg.RawMessage(b)}, }) - // Lets spin up a second node (to represent the client). + // Let's spin up a second node (to represent the client). client, _, _ := startNode(t, ctx, controlURL, "client") // Make sure that the sni node has received its config. - l, err := sni.LocalClient() + lc, err := sni.LocalClient() if err != nil { t.Fatal(err) } gotConfigured := false for range 100 { - s, err := l.StatusWithoutPeers(ctx) + s, err := lc.StatusWithoutPeers(ctx) if err != nil { t.Fatal(err) } @@ -176,7 +176,7 @@ func TestSNIProxyWithNetmapConfig(t *testing.T) { t.Error("sni node never received its configuration from the coordination server!") } - // Lets make the client open a connection to the sniproxy node, and + // Let's make the client open a connection to the sniproxy node, and // make sure it results in a connection to our test listener. w, err := client.Dial(ctx, "tcp", fmt.Sprintf("%s:%d", ip, ln.Addr().(*net.TCPAddr).Port)) if err != nil { @@ -208,10 +208,10 @@ func TestSNIProxyWithFlagConfig(t *testing.T) { sni, _, ip := startNode(t, ctx, controlURL, "snitest") go run(ctx, sni, 0, sni.Hostname, false, 0, "", fmt.Sprintf("tcp/%d/localhost", ln.Addr().(*net.TCPAddr).Port)) - // Lets spin up a second node (to represent the client). + // Let's spin up a second node (to represent the client). client, _, _ := startNode(t, ctx, controlURL, "client") - // Lets make the client open a connection to the sniproxy node, and + // Let's make the client open a connection to the sniproxy node, and // make sure it results in a connection to our test listener. w, err := client.Dial(ctx, "tcp", fmt.Sprintf("%s:%d", ip, ln.Addr().(*net.TCPAddr).Port)) if err != nil { diff --git a/cmd/speedtest/speedtest.go b/cmd/speedtest/speedtest.go index 9a457ed6c7486..e11c4ad1d90bb 100644 --- a/cmd/speedtest/speedtest.go +++ b/cmd/speedtest/speedtest.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Program speedtest provides the speedtest command. The reason to keep it separate from @@ -72,8 +72,7 @@ var speedtestArgs struct { func runSpeedtest(ctx context.Context, args []string) error { if _, _, err := net.SplitHostPort(speedtestArgs.host); err != nil { - var addrErr *net.AddrError - if errors.As(err, &addrErr) && addrErr.Err == "missing port in address" { + if addrErr, ok := errors.AsType[*net.AddrError](err); ok && addrErr.Err == "missing port in address" { // if no port is provided, append the default port speedtestArgs.host = net.JoinHostPort(speedtestArgs.host, strconv.Itoa(speedtest.DefaultPort)) } diff --git a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go index 39af584ecd481..4f4262d9febc0 100644 --- a/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go +++ b/cmd/ssh-auth-none-demo/ssh-auth-none-demo.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // ssh-auth-none-demo is a demo SSH server that's meant to run on the @@ -28,8 +28,8 @@ import ( "path/filepath" "time" - gossh "golang.org/x/crypto/ssh" - "tailscale.com/tempfork/gliderlabs/ssh" + gliderssh "github.com/tailscale/gliderssh" + "golang.org/x/crypto/ssh" ) // keyTypes are the SSH key types that we either try to read from the @@ -60,23 +60,23 @@ func main() { log.Fatal("no host keys") } - srv := &ssh.Server{ + srv := &gliderssh.Server{ Addr: *addr, Version: "Tailscale", Handler: handleSessionPostSSHAuth, - ServerConfigCallback: func(ctx ssh.Context) *gossh.ServerConfig { + ServerConfigCallback: func(ctx gliderssh.Context) *ssh.ServerConfig { start := time.Now() - var spac gossh.ServerPreAuthConn - return &gossh.ServerConfig{ - PreAuthConnCallback: func(conn gossh.ServerPreAuthConn) { + var spac ssh.ServerPreAuthConn + return &ssh.ServerConfig{ + PreAuthConnCallback: func(conn ssh.ServerPreAuthConn) { spac = conn }, NoClientAuth: true, // required for the NoClientAuthCallback to run - NoClientAuthCallback: func(cm gossh.ConnMetadata) (*gossh.Permissions, error) { + NoClientAuthCallback: func(cm ssh.ConnMetadata) (*ssh.Permissions, error) { spac.SendAuthBanner(fmt.Sprintf("# Banner: doing none auth at %v\r\n", time.Since(start))) if cm.User() == "denyme" { - return nil, &gossh.BannerError{ + return nil, &ssh.BannerError{ Err: errors.New("denying access"), Message: "denyme is not allowed to access this machine\n", } @@ -86,19 +86,20 @@ func main() { if cm.User() == "banners" { totalBanners = 5 } + for banner := 2; banner <= totalBanners; banner++ { time.Sleep(time.Second) if banner == totalBanners { - spac.SendAuthBanner(fmt.Sprintf("# Banner%d: access granted at %v\r\n", banner, time.Since(start))) + spac.SendAuthBanner(fmt.Sprintf("# Final banner saying access granted (+%v)\r\n", time.Since(start).Round(time.Millisecond))) } else { - spac.SendAuthBanner(fmt.Sprintf("# Banner%d at %v\r\n", banner, time.Since(start))) + spac.SendAuthBanner(fmt.Sprintf("# Another banner saying we're still waiting for auth server-side (+%v)\r\n", time.Since(start).Round(time.Millisecond))) } } return nil, nil }, - BannerCallback: func(cm gossh.ConnMetadata) string { + BannerCallback: func(cm ssh.ConnMetadata) string { log.Printf("Got connection from user %q, %q from %v", cm.User(), cm.ClientVersion(), cm.RemoteAddr()) - return fmt.Sprintf("# Banner for user %q, %q\n", cm.User(), cm.ClientVersion()) + return fmt.Sprintf("# Example URL in auth bannner for %q, %q: https://github.com/tailscale/tailscale\r\n", cm.User(), cm.ClientVersion()) }, } }, @@ -115,7 +116,7 @@ func main() { log.Printf("done") } -func handleSessionPostSSHAuth(s ssh.Session) { +func handleSessionPostSSHAuth(s gliderssh.Session) { log.Printf("Started session from user %q", s.User()) fmt.Fprintf(s, "Hello user %q, it worked.\n", s.User()) @@ -136,6 +137,8 @@ func handleSessionPostSSHAuth(s ssh.Session) { } }() + fmt.Fprintf(s, "We're past auth phase now. Goodbye in ...\n") + for i := 10; i > 0; i-- { fmt.Fprintf(s, "%v ...\n", i) time.Sleep(time.Second) @@ -143,13 +146,13 @@ func handleSessionPostSSHAuth(s ssh.Session) { s.Exit(0) } -func getHostKeys(dir string) (ret []ssh.Signer, err error) { +func getHostKeys(dir string) (ret []gliderssh.Signer, err error) { for _, typ := range keyTypes { hostKey, err := hostKeyFileOrCreate(dir, typ) if err != nil { return nil, err } - signer, err := gossh.ParsePrivateKey(hostKey) + signer, err := ssh.ParsePrivateKey(hostKey) if err != nil { return nil, err } diff --git a/cmd/stunc/stunc.go b/cmd/stunc/stunc.go index c4b2eedd39f90..e51cd15ba2248 100644 --- a/cmd/stunc/stunc.go +++ b/cmd/stunc/stunc.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Command stunc makes a STUN request to a STUN server and prints the result. diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index da768039431fe..239b728e1915e 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -1,8 +1,9 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depaware) + đŸ’Ŗ crypto/internal/entropy/v1.0.0 from crypto/internal/fips140/drbg github.com/beorn7/perks/quantile from github.com/prometheus/client_golang/prometheus đŸ’Ŗ github.com/cespare/xxhash/v2 from github.com/prometheus/client_golang/prometheus - github.com/go-json-experiment/json from tailscale.com/types/opt + github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ @@ -14,9 +15,9 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+ github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+ - LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus - LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs - LD github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs + L github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus + L github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs + L github.com/prometheus/procfs/internal/util from github.com/prometheus/procfs đŸ’Ŗ go4.org/mem from tailscale.com/metrics+ go4.org/netipx from tailscale.com/net/tsaddr google.golang.org/protobuf/encoding/protodelim from github.com/prometheus/common/expfmt @@ -38,6 +39,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar đŸ’Ŗ google.golang.org/protobuf/internal/impl from google.golang.org/protobuf/internal/filetype+ google.golang.org/protobuf/internal/order from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/pragma from google.golang.org/protobuf/encoding/prototext+ + đŸ’Ŗ google.golang.org/protobuf/internal/protolazy from google.golang.org/protobuf/internal/impl+ google.golang.org/protobuf/internal/set from google.golang.org/protobuf/encoding/prototext đŸ’Ŗ google.golang.org/protobuf/internal/strs from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/internal/version from google.golang.org/protobuf/runtime/protoimpl @@ -46,78 +48,84 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar google.golang.org/protobuf/reflect/protoregistry from google.golang.org/protobuf/encoding/prototext+ google.golang.org/protobuf/runtime/protoiface from google.golang.org/protobuf/internal/impl+ google.golang.org/protobuf/runtime/protoimpl from github.com/prometheus/client_model/go+ - google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ + đŸ’Ŗ google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+ tailscale.com from tailscale.com/version tailscale.com/envknob from tailscale.com/tsweb+ tailscale.com/feature from tailscale.com/tsweb + tailscale.com/feature/buildfeatures from tailscale.com/feature+ tailscale.com/kube/kubetypes from tailscale.com/envknob tailscale.com/metrics from tailscale.com/net/stunserver+ tailscale.com/net/netaddr from tailscale.com/net/tsaddr tailscale.com/net/stun from tailscale.com/net/stunserver tailscale.com/net/stunserver from tailscale.com/cmd/stund tailscale.com/net/tsaddr from tailscale.com/tsweb - tailscale.com/syncs from tailscale.com/metrics - tailscale.com/tailcfg from tailscale.com/version + tailscale.com/syncs from tailscale.com/metrics+ + tailscale.com/tailcfg from tailscale.com/version+ + tailscale.com/tstime from tailscale.com/tsweb tailscale.com/tsweb from tailscale.com/cmd/stund+ tailscale.com/tsweb/promvarz from tailscale.com/cmd/stund tailscale.com/tsweb/varz from tailscale.com/tsweb+ tailscale.com/types/dnstype from tailscale.com/tailcfg tailscale.com/types/ipproto from tailscale.com/tailcfg - tailscale.com/types/key from tailscale.com/tailcfg + tailscale.com/types/key from tailscale.com/tailcfg+ tailscale.com/types/lazy from tailscale.com/version+ tailscale.com/types/logger from tailscale.com/tsweb+ tailscale.com/types/opt from tailscale.com/envknob+ - tailscale.com/types/ptr from tailscale.com/tailcfg+ + tailscale.com/types/persist from tailscale.com/feature tailscale.com/types/result from tailscale.com/util/lineiter tailscale.com/types/structs from tailscale.com/tailcfg+ tailscale.com/types/tkatype from tailscale.com/tailcfg+ tailscale.com/types/views from tailscale.com/net/tsaddr+ + tailscale.com/util/bufiox from tailscale.com/types/key tailscale.com/util/ctxkey from tailscale.com/tsweb+ L đŸ’Ŗ tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/tailcfg tailscale.com/util/lineiter from tailscale.com/version/distro - tailscale.com/util/mak from tailscale.com/syncs + tailscale.com/util/mak from tailscale.com/syncs+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto tailscale.com/util/rands from tailscale.com/tsweb + tailscale.com/util/set from tailscale.com/types/key tailscale.com/util/slicesx from tailscale.com/tailcfg - tailscale.com/util/testenv from tailscale.com/types/logger + tailscale.com/util/testenv from tailscale.com/types/logger+ tailscale.com/util/vizerror from tailscale.com/tailcfg+ tailscale.com/version from tailscale.com/envknob+ tailscale.com/version/distro from tailscale.com/envknob golang.org/x/crypto/blake2b from golang.org/x/crypto/nacl/box - golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305 - golang.org/x/crypto/chacha20poly1305 from crypto/tls+ - golang.org/x/crypto/cryptobyte from crypto/ecdsa+ - golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ golang.org/x/crypto/curve25519 from golang.org/x/crypto/nacl/box+ - golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ - golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/internal/alias from golang.org/x/crypto/nacl/secretbox + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/nacl/secretbox golang.org/x/crypto/nacl/box from tailscale.com/types/key golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ - golang.org/x/exp/constraints from tailscale.com/tsweb/varz - golang.org/x/net/dns/dnsmessage from net+ - golang.org/x/net/http/httpguts from net/http - golang.org/x/net/http/httpproxy from net/http - golang.org/x/net/http2/hpack from net/http - golang.org/x/net/idna from golang.org/x/net/http/httpguts+ - D golang.org/x/net/route from net + golang.org/x/exp/constraints from tailscale.com/tsweb/varz+ golang.org/x/sys/cpu from golang.org/x/crypto/blake2b+ LD golang.org/x/sys/unix from github.com/prometheus/procfs+ W golang.org/x/sys/windows from github.com/prometheus/client_golang/prometheus - golang.org/x/text/secure/bidirule from golang.org/x/net/idna - golang.org/x/text/transform from golang.org/x/text/secure/bidirule+ - golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ - golang.org/x/text/unicode/norm from golang.org/x/net/idna + vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ + vendor/golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + vendor/golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + vendor/golang.org/x/crypto/internal/alias from vendor/golang.org/x/crypto/chacha20+ + vendor/golang.org/x/crypto/internal/poly1305 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/net/dns/dnsmessage from net + vendor/golang.org/x/net/http/httpguts from net/http+ + vendor/golang.org/x/net/http/httpproxy from net/http + vendor/golang.org/x/net/http2/hpack from net/http+ + vendor/golang.org/x/net/idna from net/http+ + vendor/golang.org/x/sys/cpu from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/text/secure/bidirule from vendor/golang.org/x/net/idna + vendor/golang.org/x/text/transform from vendor/golang.org/x/text/secure/bidirule+ + vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ + vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna bufio from compress/flate+ bytes from bufio+ cmp from slices+ - compress/flate from compress/gzip + compress/flate from compress/gzip+ compress/gzip from google.golang.org/protobuf/internal/impl+ container/list from crypto/tls+ context from crypto/tls+ crypto from crypto/ecdh+ - crypto/aes from crypto/internal/hpke+ + crypto/aes from crypto/tls+ crypto/cipher from crypto/aes+ crypto/des from crypto/tls+ crypto/dsa from crypto/x509 @@ -125,11 +133,14 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar crypto/ecdsa from crypto/tls+ crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ + crypto/fips140 from crypto/tls/internal/fips140tls+ + crypto/hkdf from crypto/hpke+ crypto/hmac from crypto/tls + crypto/hpke from crypto/tls crypto/internal/boring from crypto/aes+ crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring - crypto/internal/entropy from crypto/internal/fips140/drbg + crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ crypto/internal/fips140 from crypto/internal/fips140/aes+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ @@ -144,7 +155,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/mlkem crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -154,22 +165,25 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ crypto/internal/fips140/tls12 from crypto/tls crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140cache from crypto/ecdsa+ crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140deps/time from crypto/internal/entropy/v1.0.0 crypto/internal/fips140hash from crypto/ecdsa+ crypto/internal/fips140only from crypto/cipher+ - crypto/internal/hpke from crypto/tls crypto/internal/impl from crypto/internal/fips140/aes+ - crypto/internal/randutil from crypto/dsa+ - crypto/internal/sysrand from crypto/internal/entropy+ + crypto/internal/rand from crypto/dsa+ + crypto/internal/randutil from crypto/internal/rand + crypto/internal/sysrand from crypto/internal/fips140/drbg crypto/md5 from crypto/tls+ + crypto/mlkem from crypto/hpke+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls crypto/rsa from crypto/tls+ crypto/sha1 from crypto/tls+ crypto/sha256 from crypto/tls+ - crypto/sha3 from crypto/internal/fips140hash + crypto/sha3 from crypto/internal/fips140hash+ crypto/sha512 from crypto/ecdsa+ crypto/subtle from crypto/cipher+ crypto/tls from net/http+ @@ -209,9 +223,8 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar internal/goarch from crypto/internal/fips140deps/cpu+ internal/godebug from crypto/internal/fips140deps/godebug+ internal/godebugs from internal/godebug+ - internal/goexperiment from hash/maphash+ + internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ - internal/itoa from internal/poll+ internal/msan from internal/runtime/maps+ internal/nettrace from net+ internal/oserror from io/fs+ @@ -220,21 +233,31 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar internal/profilerecord from runtime+ internal/race from internal/poll+ internal/reflectlite from context+ + D internal/routebsd from net internal/runtime/atomic from internal/runtime/exithook+ + L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime + internal/runtime/gc from runtime+ + internal/runtime/gc/scan from runtime internal/runtime/maps from reflect+ internal/runtime/math from internal/runtime/maps+ + internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ - L internal/runtime/syscall from runtime+ + L internal/runtime/syscall/linux from internal/runtime/cgroup+ + W internal/runtime/syscall/windows from internal/syscall/windows+ + internal/saferio from encoding/asn1 internal/singleflight from net + internal/strconv from internal/poll+ internal/stringslite from embed+ internal/sync from sync+ + internal/synctest from sync internal/syscall/execenv from os LD internal/syscall/unix from crypto/internal/sysrand+ W internal/syscall/windows from crypto/internal/sysrand+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os + internal/trace/tracev2 from runtime+ internal/unsafeheader from internal/reflectlite+ io from bufio+ io/fs from crypto/x509+ @@ -252,18 +275,19 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ - net/http/httptrace from net/http + net/http/httptrace from net/http+ net/http/internal from net/http net/http/internal/ascii from net/http + net/http/internal/httpcommon from net/http net/http/pprof from tailscale.com/tsweb net/netip from go4.org/netipx+ - net/textproto from golang.org/x/net/http/httpguts+ + net/textproto from mime/multipart+ net/url from crypto/x509+ os from crypto/internal/sysrand+ os/signal from tailscale.com/cmd/stund path from github.com/prometheus/client_golang/prometheus/internal+ path/filepath from crypto/x509+ - reflect from crypto/x509+ + reflect from encoding/asn1+ regexp from github.com/prometheus/client_golang/prometheus/internal+ regexp/syntax from regexp runtime from crypto/internal/fips140+ @@ -275,6 +299,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar sort from compress/flate+ strconv from compress/flate+ strings from bufio+ + W structs from internal/syscall/windows sync from compress/flate+ sync/atomic from context+ syscall from crypto/internal/sysrand+ @@ -285,4 +310,4 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar unicode/utf8 from bufio+ unique from net/netip unsafe from bytes+ - weak from unique + weak from unique+ diff --git a/cmd/stund/stund.go b/cmd/stund/stund.go index 1055d966f42c5..a27e520444464 100644 --- a/cmd/stund/stund.go +++ b/cmd/stund/stund.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The stund binary is a standalone STUN server. diff --git a/cmd/stunstamp/stunstamp.go b/cmd/stunstamp/stunstamp.go index c3842e2e8b3be..743d6aec3c9d8 100644 --- a/cmd/stunstamp/stunstamp.go +++ b/cmd/stunstamp/stunstamp.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The stunstamp binary measures round-trip latency with DERPs. @@ -34,10 +34,10 @@ import ( "github.com/golang/snappy" "github.com/prometheus/prometheus/prompb" "github.com/tcnksm/go-httpstat" - "tailscale.com/logtail/backoff" "tailscale.com/net/stun" "tailscale.com/net/tcpinfo" "tailscale.com/tailcfg" + "tailscale.com/util/backoff" ) var ( @@ -135,18 +135,18 @@ type lportsPool struct { ports []int } -func (l *lportsPool) get() int { - l.Lock() - defer l.Unlock() - ret := l.ports[0] - l.ports = append(l.ports[:0], l.ports[1:]...) +func (pl *lportsPool) get() int { + pl.Lock() + defer pl.Unlock() + ret := pl.ports[0] + pl.ports = append(pl.ports[:0], pl.ports[1:]...) return ret } -func (l *lportsPool) put(i int) { - l.Lock() - defer l.Unlock() - l.ports = append(l.ports, int(i)) +func (pl *lportsPool) put(i int) { + pl.Lock() + defer pl.Unlock() + pl.ports = append(pl.ports, int(i)) } var ( @@ -173,19 +173,19 @@ func init() { // measure dial time. type lportForTCPConn int -func (l *lportForTCPConn) Close() error { - if *l == 0 { +func (lp *lportForTCPConn) Close() error { + if *lp == 0 { return nil } - lports.put(int(*l)) + lports.put(int(*lp)) return nil } -func (l *lportForTCPConn) Write([]byte) (int, error) { +func (lp *lportForTCPConn) Write([]byte) (int, error) { return 0, errors.New("unimplemented") } -func (l *lportForTCPConn) Read([]byte) (int, error) { +func (lp *lportForTCPConn) Read([]byte) (int, error) { return 0, errors.New("unimplemented") } @@ -889,8 +889,7 @@ func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSer reqCtx, cancel := context.WithTimeout(context.Background(), time.Second*30) writeErr = client.write(reqCtx, ts) cancel() - var re recoverableErr - recoverable := errors.As(writeErr, &re) + _, recoverable := errors.AsType[recoverableErr](writeErr) if writeErr != nil { log.Printf("remote write error(recoverable=%v): %v", recoverable, writeErr) } diff --git a/cmd/stunstamp/stunstamp_default.go b/cmd/stunstamp/stunstamp_default.go index a244d9aea6410..3f6613cd060ee 100644 --- a/cmd/stunstamp/stunstamp_default.go +++ b/cmd/stunstamp/stunstamp_default.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !linux diff --git a/cmd/stunstamp/stunstamp_linux.go b/cmd/stunstamp/stunstamp_linux.go index 387805feff2f1..201e2f83b384c 100644 --- a/cmd/stunstamp/stunstamp_linux.go +++ b/cmd/stunstamp/stunstamp_linux.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/sync-containers/main.go b/cmd/sync-containers/main.go index 6317b4943ae82..ab2a38bd66dab 100644 --- a/cmd/sync-containers/main.go +++ b/cmd/sync-containers/main.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 @@ -65,9 +65,9 @@ func main() { } add, remove := diffTags(stags, dtags) - if l := len(add); l > 0 { + if ln := len(add); ln > 0 { log.Printf("%d tags to push: %s", len(add), strings.Join(add, ", ")) - if *max > 0 && l > *max { + if *max > 0 && ln > *max { log.Printf("Limiting sync to %d tags", *max) add = add[:*max] } diff --git a/cmd/systray/systray.go b/cmd/systray/systray.go index 0185a1bc2dc5e..68a3397820274 100644 --- a/cmd/systray/systray.go +++ b/cmd/systray/systray.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build cgo || !darwin @@ -7,9 +7,19 @@ package main import ( + "flag" + + "tailscale.com/client/local" "tailscale.com/client/systray" + "tailscale.com/paths" ) +var socket = flag.String("socket", paths.DefaultTailscaledSocket(), "path to tailscaled socket") +var theme = flag.String("theme", "dark", "color theme for Tailscale icon: dark, dark:nobg, light, light:nobg") + func main() { - new(systray.Menu).Run() + flag.Parse() + lc := &local.Client{Socket: *socket} + systray.SetTheme(*theme) + new(systray.Menu).Run(lc) } diff --git a/cmd/tailscale/cli/advertise.go b/cmd/tailscale/cli/advertise.go deleted file mode 100644 index 83d1a35aa8a14..0000000000000 --- a/cmd/tailscale/cli/advertise.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package cli - -import ( - "context" - "flag" - "fmt" - "strings" - - "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/envknob" - "tailscale.com/ipn" - "tailscale.com/tailcfg" -) - -var advertiseArgs struct { - services string // comma-separated list of services to advertise -} - -// TODO(naman): This flag may move to set.go or serve_v2.go after the WIPCode -// envknob is not needed. -func advertiseCmd() *ffcli.Command { - if !envknob.UseWIPCode() { - return nil - } - return &ffcli.Command{ - Name: "advertise", - ShortUsage: "tailscale advertise --services=", - ShortHelp: "Advertise this node as a destination for a service", - Exec: runAdvertise, - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("advertise") - fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")") - return fs - })(), - } -} - -func runAdvertise(ctx context.Context, args []string) error { - if len(args) > 0 { - return flag.ErrHelp - } - - services, err := parseServiceNames(advertiseArgs.services) - if err != nil { - return err - } - - _, err = localClient.EditPrefs(ctx, &ipn.MaskedPrefs{ - AdvertiseServicesSet: true, - Prefs: ipn.Prefs{ - AdvertiseServices: services, - }, - }) - return err -} - -// parseServiceNames takes a comma-separated list of service names -// (eg. "svc:hello,svc:webserver,svc:catphotos"), splits them into -// a list and validates each service name. If valid, it returns -// the service names in a slice of strings. -func parseServiceNames(servicesArg string) ([]string, error) { - var services []string - if servicesArg != "" { - services = strings.Split(servicesArg, ",") - for _, svc := range services { - err := tailcfg.ServiceName(svc).Validate() - if err != nil { - return nil, fmt.Errorf("service %q: %s", svc, err) - } - } - } - return services, nil -} diff --git a/cmd/tailscale/cli/appcroutes.go b/cmd/tailscale/cli/appcroutes.go new file mode 100644 index 0000000000000..04cbcdd832258 --- /dev/null +++ b/cmd/tailscale/cli/appcroutes.go @@ -0,0 +1,153 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "slices" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/types/appctype" +) + +var appcRoutesArgs struct { + all bool + domainMap bool + n bool +} + +var appcRoutesCmd = &ffcli.Command{ + Name: "appc-routes", + ShortUsage: "tailscale appc-routes", + Exec: runAppcRoutesInfo, + ShortHelp: "Print the current app connector routes", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("appc-routes") + fs.BoolVar(&appcRoutesArgs.all, "all", false, "Print learned domains and routes and extra policy configured routes.") + fs.BoolVar(&appcRoutesArgs.domainMap, "map", false, "Print the map of learned domains: [routes].") + fs.BoolVar(&appcRoutesArgs.n, "n", false, "Print the total number of routes this node advertises.") + return fs + })(), + LongHelp: strings.TrimSpace(` +The 'tailscale appc-routes' command prints the current App Connector route status. + +By default this command prints the domains configured in the app connector configuration and how many routes have been +learned for each domain. + +--all prints the routes learned from the domains configured in the app connector configuration; and any extra routes provided +in the the policy app connector 'routes' field. + +--map prints the routes learned from the domains configured in the app connector configuration. + +-n prints the total number of routes advertised by this device, whether learned, set in the policy, or set locally. + +For more information about App Connectors, refer to +https://tailscale.com/kb/1281/app-connectors +`), +} + +func getAllOutput(ri *appctype.RouteInfo) (string, error) { + domains, err := json.MarshalIndent(ri.Domains, " ", " ") + if err != nil { + return "", err + } + control, err := json.MarshalIndent(ri.Control, " ", " ") + if err != nil { + return "", err + } + s := fmt.Sprintf(`Learned Routes +============== +%s + +Routes from Policy +================== +%s +`, domains, control) + return s, nil +} + +type domainCount struct { + domain string + count int +} + +func getSummarizeLearnedOutput(ri *appctype.RouteInfo) string { + x := make([]domainCount, len(ri.Domains)) + i := 0 + maxDomainWidth := 0 + for k, v := range ri.Domains { + if len(k) > maxDomainWidth { + maxDomainWidth = len(k) + } + x[i] = domainCount{domain: k, count: len(v)} + i++ + } + slices.SortFunc(x, func(i, j domainCount) int { + if i.count > j.count { + return -1 + } + if i.count < j.count { + return 1 + } + if i.domain > j.domain { + return 1 + } + if i.domain < j.domain { + return -1 + } + return 0 + }) + var s strings.Builder + fmtString := fmt.Sprintf("%%-%ds %%d\n", maxDomainWidth) // eg "%-10s %d\n" + for _, dc := range x { + s.WriteString(fmt.Sprintf(fmtString, dc.domain, dc.count)) + } + return s.String() +} + +func runAppcRoutesInfo(ctx context.Context, args []string) error { + prefs, err := localClient.GetPrefs(ctx) + if err != nil { + return err + } + if !prefs.AppConnector.Advertise { + fmt.Println("not a connector") + return nil + } + + if appcRoutesArgs.n { + fmt.Println(len(prefs.AdvertiseRoutes)) + return nil + } + + routeInfo, err := localClient.GetAppConnectorRouteInfo(ctx) + if err != nil { + return err + } + + if appcRoutesArgs.domainMap { + domains, err := json.Marshal(routeInfo.Domains) + if err != nil { + return err + } + fmt.Println(string(domains)) + return nil + } + + if appcRoutesArgs.all { + s, err := getAllOutput(&routeInfo) + if err != nil { + return err + } + fmt.Println(s) + return nil + } + + fmt.Print(getSummarizeLearnedOutput(&routeInfo)) + return nil +} diff --git a/cmd/tailscale/cli/bugreport.go b/cmd/tailscale/cli/bugreport.go index d671f3df60d76..3ffaffa8b1fa5 100644 --- a/cmd/tailscale/cli/bugreport.go +++ b/cmd/tailscale/cli/bugreport.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -10,7 +10,7 @@ import ( "fmt" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/client/tailscale" + "tailscale.com/client/local" ) var bugReportCmd = &ffcli.Command{ @@ -40,7 +40,7 @@ func runBugReport(ctx context.Context, args []string) error { default: return errors.New("unknown arguments") } - opts := tailscale.BugReportOpts{ + opts := local.BugReportOpts{ Note: note, Diagnose: bugReportArgs.diagnose, } diff --git a/cmd/tailscale/cli/cert.go b/cmd/tailscale/cli/cert.go index 9c8eca5b7d7d0..bab83901f0727 100644 --- a/cmd/tailscale/cli/cert.go +++ b/cmd/tailscale/cli/cert.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !js && !ts_omit_acme + package cli import ( @@ -21,23 +23,30 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "software.sslmate.com/src/go-pkcs12" "tailscale.com/atomicfile" + "tailscale.com/feature/buildfeatures" + "tailscale.com/health" "tailscale.com/ipn" + "tailscale.com/tsconst" "tailscale.com/version" ) -var certCmd = &ffcli.Command{ - Name: "cert", - Exec: runCert, - ShortHelp: "Get TLS certs", - ShortUsage: "tailscale cert [flags] ", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("cert") - fs.StringVar(&certArgs.certFile, "cert-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.crt if --cert-file and --key-file are both unset") - fs.StringVar(&certArgs.keyFile, "key-file", "", "output key file or \"-\" for stdout; defaults to DOMAIN.key if --cert-file and --key-file are both unset") - fs.BoolVar(&certArgs.serve, "serve-demo", false, "if true, serve on port :443 using the cert as a demo, instead of writing out the files to disk") - fs.DurationVar(&certArgs.minValidity, "min-validity", 0, "ensure the certificate is valid for at least this duration; the output certificate is never expired if this flag is unset or 0, but the lifetime may vary; the maximum allowed min-validity depends on the CA") - return fs - })(), +func init() { + maybeCertCmd = func() *ffcli.Command { + return &ffcli.Command{ + Name: "cert", + Exec: runCert, + ShortHelp: "Get TLS certs", + ShortUsage: "tailscale cert [flags] ", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("cert") + fs.StringVar(&certArgs.certFile, "cert-file", "", "output cert file or \"-\" for stdout; defaults to DOMAIN.crt if --cert-file and --key-file are both unset") + fs.StringVar(&certArgs.keyFile, "key-file", "", "output key file or \"-\" for stdout; defaults to DOMAIN.key if --cert-file and --key-file are both unset") + fs.BoolVar(&certArgs.serve, "serve-demo", false, "if true, serve on port :443 using the cert as a demo, instead of writing out the files to disk") + fs.DurationVar(&certArgs.minValidity, "min-validity", 0, "ensure the certificate is valid for at least this duration; the output certificate is never expired if this flag is unset or 0, but the lifetime may vary; the maximum allowed min-validity depends on the CA") + return fs + })(), + } + } } var certArgs struct { @@ -102,8 +111,14 @@ func runCert(ctx context.Context, args []string) error { log.SetFlags(0) } if certArgs.certFile == "" && certArgs.keyFile == "" { - certArgs.certFile = domain + ".crt" - certArgs.keyFile = domain + ".key" + fileBase := strings.Replace(domain, "*.", "wildcard_.", 1) + certArgs.certFile = fileBase + ".crt" + certArgs.keyFile = fileBase + ".key" + } + if buildfeatures.HasHealth { + watchCtx, cancel := context.WithCancel(ctx) + defer cancel() + go watchCertPendingHealth(watchCtx, domain) } certPEM, keyPEM, err := localClient.CertPairWithValidity(ctx, domain, certArgs.minValidity) if err != nil { @@ -160,6 +175,44 @@ func runCert(ctx context.Context, args []string) error { return nil } +// watchCertPendingHealth subscribes to the IPN bus and prints the +// [tsconst.HealthWarnableTLSCertPending] warning to stderr if it appears +// for domain while a cert fetch is in flight. It returns once it has +// printed the warning or ctx is done. +// +// Subscription is delayed 1 second so we don't print anything when the +// daemon returns a cached cert quickly. +func watchCertPendingHealth(ctx context.Context, domain string) { + select { + case <-time.After(1 * time.Second): + case <-ctx.Done(): + return + } + watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialHealthState|ipn.NotifyNoNetMap) + if err != nil { + return + } + defer watcher.Close() + for { + n, err := watcher.Next() + if err != nil { + return + } + if n.Health == nil { + continue + } + ws, ok := n.Health.Warnings[tsconst.HealthWarnableTLSCertPending] + if !ok { + continue + } + if !strings.Contains(ws.Args[health.ArgDomains], domain) { + continue + } + fmt.Fprintf(os.Stderr, "%s: %s\n", ws.Title, ws.Text) + return + } +} + func writeIfChanged(filename string, contents []byte, mode os.FileMode) (changed bool, err error) { if filename == "-" { Stdout.Write(contents) diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index d7e8e5ca22dce..27e7bd2a2c692 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package cli contains the cmd/tailscale CLI code in a package that can be included @@ -6,7 +6,9 @@ package cli import ( + "bytes" "context" + "encoding/json" "errors" "flag" "fmt" @@ -17,16 +19,16 @@ import ( "strings" "sync" "text/tabwriter" + "time" - "github.com/mattn/go-colorable" - "github.com/mattn/go-isatty" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/client/local" - "tailscale.com/client/tailscale" "tailscale.com/cmd/tailscale/cli/ffcomplete" "tailscale.com/envknob" + "tailscale.com/feature" "tailscale.com/paths" "tailscale.com/util/slicesx" + "tailscale.com/util/testenv" "tailscale.com/version/distro" ) @@ -91,8 +93,8 @@ var localClient = local.Client{ Socket: paths.DefaultTailscaledSocket(), } -// Run runs the CLI. The args do not include the binary name. -func Run(args []string) (err error) { +// RunWithContext runs the CLI. The args do not include the binary name. +func RunWithContext(ctx context.Context, args []string) (err error) { if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 && len(args) == 0 { // We're running on gokrazy and the user did not specify 'up'. // Don't run the tailscale CLI and spam logs with usage; just exit. @@ -112,7 +114,7 @@ func Run(args []string) (err error) { } var warnOnce sync.Once - tailscale.SetVersionMismatchHandler(func(clientVer, serverVer string) { + local.SetVersionMismatchHandler(func(clientVer, serverVer string) { warnOnce.Do(func() { fmt.Fprintf(Stderr, "Warning: client version %q != tailscaled server version %q\n", clientVer, serverVer) }) @@ -123,7 +125,7 @@ func Run(args []string) (err error) { if errors.Is(err, flag.ErrHelp) { return nil } - if noexec := (ffcli.NoExecError{}); errors.As(err, &noexec) { + if noexec, ok := errors.AsType[ffcli.NoExecError](err); ok { // When the user enters an unknown subcommand, ffcli tries to run // the closest valid parent subcommand with everything else as args, // returning NoExecError if it doesn't have an Exec function. @@ -162,8 +164,8 @@ func Run(args []string) (err error) { return } - err = rootCmd.Run(context.Background()) - if tailscale.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" { + err = rootCmd.Run(ctx) + if local.IsAccessDeniedError(err) && os.Getuid() != 0 && runtime.GOOS != "windows" { return fmt.Errorf("%v\n\nUse 'sudo tailscale %s'.\nTo not require root, use 'sudo tailscale set --operator=$USER' once.", err, strings.Join(args, " ")) } if errors.Is(err, flag.ErrHelp) { @@ -172,6 +174,11 @@ func Run(args []string) (err error) { return err } +// Run is equivalent to calling [RunWithContext] with the background context. +func Run(args []string) (err error) { + return RunWithContext(context.Background(), args) +} + type onceFlagValue struct { flag.Value set bool @@ -193,23 +200,57 @@ func (v *onceFlagValue) IsBoolFlag() bool { return ok && bf.IsBoolFlag() } -// noDupFlagify modifies c recursively to make all the -// flag values be wrappers that permit setting the value -// at most once. -func noDupFlagify(c *ffcli.Command) { - if c.FlagSet != nil { - c.FlagSet.VisitAll(func(f *flag.Flag) { - f.Value = &onceFlagValue{Value: f.Value} - }) +// noDupFlagify modifies c recursively to make all the flag values be +// wrappers that permit setting the value at most once. If tb is +// non-nil, the original values are restored when the test completes. +func noDupFlagify(c *ffcli.Command, tb testenv.TB) { + if tb == nil && testenv.InTest() { + return } - for _, sub := range c.Subcommands { - noDupFlagify(sub) + type restore struct { + f *flag.Flag + v flag.Value + } + var restores []restore + var walk func(*ffcli.Command) + walk = func(c *ffcli.Command) { + if c.FlagSet != nil { + c.FlagSet.VisitAll(func(f *flag.Flag) { + if tb != nil { + restores = append(restores, restore{f, f.Value}) + } + f.Value = &onceFlagValue{Value: f.Value} + }) + } + for _, sub := range c.Subcommands { + walk(sub) + } + } + walk(c) + if tb != nil { + tb.Cleanup(func() { + for _, r := range restores { + r.f.Value = r.v + } + }) } } -var fileCmd func() *ffcli.Command +var ( + fileCmd, + sysPolicyCmd, + maybeRoutecheckCmd, + maybeWebCmd, + maybeDriveCmd, + maybeTailnetLockCmd, + maybeFunnelCmd, + maybeServeCmd, + maybeCertCmd, + maybeUpdateCmd, + _ func() *ffcli.Command +) -func newRootCmd() *ffcli.Command { +func newRootCmd(tb ...testenv.TB) *ffcli.Command { rootfs := newFlagSet("tailscale") rootfs.Func("socket", "path to tailscaled socket", func(s string) error { localClient.Socket = s @@ -217,8 +258,10 @@ func newRootCmd() *ffcli.Command { return nil }) rootfs.Lookup("socket").DefValue = localClient.Socket + jsonDocs := rootfs.Bool("json-docs", false, hidden+"print JSON-encoded docs for all subcommands and flags") - rootCmd := &ffcli.Command{ + var rootCmd *ffcli.Command + rootCmd = &ffcli.Command{ Name: "tailscale", ShortUsage: "tailscale [flags] [command flags]", ShortHelp: "The easiest, most secure way to use WireGuard.", @@ -232,12 +275,14 @@ change in the future. upCmd, downCmd, setCmd, + getCmd, loginCmd, logoutCmd, switchCmd, configureCmd(), - syspolicyCmd, + nilOrCall(sysPolicyCmd), netcheckCmd, + nilOrCall(maybeRoutecheckCmd), ipCmd, dnsCmd, statusCmd, @@ -245,26 +290,32 @@ change in the future. pingCmd, ncCmd, sshCmd, - funnelCmd(), - serveCmd(), + nilOrCall(maybeFunnelCmd), + nilOrCall(maybeServeCmd), versionCmd, - webCmd, + nilOrCall(maybeWebCmd), nilOrCall(fileCmd), bugReportCmd, - certCmd, - netlockCmd, + nilOrCall(maybeCertCmd), + nilOrCall(maybeTailnetLockCmd), licensesCmd, exitNodeCmd(), - updateCmd, + nilOrCall(maybeUpdateCmd), whoisCmd, + whoamiCmd, debugCmd(), - driveCmd, + nilOrCall(maybeDriveCmd), idTokenCmd, - advertiseCmd(), configureHostCmd(), + systrayCmd, + appcRoutesCmd, + waitCmd, ), FlagSet: rootfs, Exec: func(ctx context.Context, args []string) error { + if *jsonDocs { + return printJSONDocs(rootCmd) + } if len(args) > 0 { return fmt.Errorf("tailscale: unknown subcommand: %s", args[0]) } @@ -276,11 +327,19 @@ change in the future. if w.UsageFunc == nil { w.UsageFunc = usageFunc } + if w.FlagSet != nil { + // If flags cannot be parsed, redact any keys in the error output . + w.FlagSet.SetOutput(sanitizeOutput(w.FlagSet.Output())) + } return true }) ffcomplete.Inject(rootCmd, func(c *ffcli.Command) { c.LongHelp = hidden + c.LongHelp }, usageFunc) - noDupFlagify(rootCmd) + var t testenv.TB + if len(tb) > 0 { + t = tb[0] + } + noDupFlagify(rootCmd, t) return rootCmd } @@ -459,16 +518,116 @@ func countFlags(fs *flag.FlagSet) (n int) { return n } -// colorableOutput returns a colorable writer if stdout is a terminal (not, say, -// redirected to a file or pipe), the Stdout writer is os.Stdout (we're not -// embedding the CLI in wasm or a mobile app), and NO_COLOR is not set (see -// https://no-color.org/). If any of those is not the case, ok is false -// and w is Stdout. -func colorableOutput() (w io.Writer, ok bool) { - if Stdout != os.Stdout || - os.Getenv("NO_COLOR") != "" || - !isatty.IsTerminal(os.Stdout.Fd()) { - return Stdout, false +type commandDoc struct { + Name string + Desc string + Subcommands []commandDoc `json:",omitempty"` + Flags []flagDoc `json:",omitempty"` +} + +type flagDoc struct { + Name string + Desc string +} + +func printJSONDocs(root *ffcli.Command) error { + docs := jsonDocsWalk(root) + return json.NewEncoder(os.Stdout).Encode(docs) +} + +func jsonDocsWalk(cmd *ffcli.Command) *commandDoc { + res := &commandDoc{ + Name: cmd.Name, + } + if cmd.LongHelp != "" { + res.Desc = cmd.LongHelp + } else if cmd.ShortHelp != "" { + res.Desc = cmd.ShortHelp + } else { + res.Desc = cmd.ShortUsage + } + if strings.HasPrefix(res.Desc, hidden) { + return nil } - return colorable.NewColorableStdout(), true + if cmd.FlagSet != nil { + cmd.FlagSet.VisitAll(func(f *flag.Flag) { + if strings.HasPrefix(f.Usage, hidden) { + return + } + res.Flags = append(res.Flags, flagDoc{ + Name: f.Name, + Desc: f.Usage, + }) + }) + } + for _, sub := range cmd.Subcommands { + subj := jsonDocsWalk(sub) + if subj != nil { + res.Subcommands = append(res.Subcommands, *subj) + } + } + return res +} + +func lastSeenFmt(t time.Time) string { + if t.IsZero() { + return "" + } + d := max(time.Since(t), time.Minute) // at least 1 minute + + switch { + case d < time.Hour: + return fmt.Sprintf(", last seen %dm ago", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf(", last seen %dh ago", int(d.Hours())) + default: + return fmt.Sprintf(", last seen %dd ago", int(d.Hours()/24)) + } +} + +var hookFixTailscaledConnectError feature.Hook[func(error) error] // for cliconndiag + +func fixTailscaledConnectError(origErr error) error { + if f, ok := hookFixTailscaledConnectError.GetOk(); ok { + return f(origErr) + } + return origErr +} + +func sanitizeOutput(w io.Writer) io.Writer { + return sanitizeWriter{w} +} + +type sanitizeWriter struct { + w io.Writer +} + +// Write logically replaces /tskey-[A-Za-z0-9-]+/ with /tskey-XXXX.../ in buf +// before writing to the underlying writer. +// +// We avoid the "regexp" package to not bloat the minbox build, and without +// making this a featuretag-omittable protection. +func (w sanitizeWriter) Write(buf []byte) (int, error) { + const prefix = "tskey-" + scrub := buf + for { + i := bytes.Index(scrub, []byte(prefix)) + if i == -1 { + break + } + scrub = scrub[i+len(prefix):] + + for i, b := range scrub { + if (b >= 'a' && b <= 'z') || + (b >= 'A' && b <= 'Z') || + (b >= '0' && b <= '9') || + b == '-' { + scrub[i] = 'X' + } else { + break + } + } + } + + return w.w.Write(buf) } diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index 9aa3693fd92c5..36cffa8aba7b9 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -6,11 +6,14 @@ package cli import ( "bytes" stdcmp "cmp" + "context" "encoding/json" "flag" "fmt" "io" "net/netip" + "os" + "path/filepath" "reflect" "strings" "testing" @@ -20,12 +23,14 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/envknob" "tailscale.com/health/healthmsg" + "tailscale.com/internal/client/tailscale" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" "tailscale.com/tka" "tailscale.com/tstest" "tailscale.com/tstest/deptest" + "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/opt" "tailscale.com/types/persist" @@ -174,6 +179,7 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) { curUser string // os.Getenv("USER") on the client side goos string // empty means "linux" distro distro.Distro + backendState string // empty means "Running" want string }{ @@ -188,6 +194,28 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) { }, want: "", }, + { + name: "bare_up_needs_login_default_prefs", + flags: []string{}, + curPrefs: ipn.NewPrefs(), + backendState: ipn.NeedsLogin.String(), + want: "", + }, + { + name: "bare_up_needs_login_losing_prefs", + flags: []string{}, + curPrefs: &ipn.Prefs{ + // defaults: + ControlURL: ipn.DefaultControlURL, + WantRunning: false, + NetfilterMode: preftype.NetfilterOn, + NoStatefulFiltering: opt.NewBool(true), + // non-default: + CorpDNS: false, + }, + backendState: ipn.NeedsLogin.String(), + want: accidentalUpPrefix + " --accept-dns=false", + }, { name: "losing_hostname", flags: []string{"--accept-dns"}, @@ -620,9 +648,13 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - goos := "linux" - if tt.goos != "" { - goos = tt.goos + goos := stdcmp.Or(tt.goos, "linux") + backendState := stdcmp.Or(tt.backendState, ipn.Running.String()) + // Needs to match the other conditions in checkForAccidentalSettingReverts + tt.curPrefs.Persist = &persist.Persist{ + UserProfile: tailcfg.UserProfile{ + LoginName: "janet", + }, } var upArgs upArgsT flagSet := newUpFlagSet(goos, &upArgs, "up") @@ -638,10 +670,11 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) { curExitNodeIP: tt.curExitNodeIP, distro: tt.distro, user: tt.curUser, + backendState: backendState, } applyImplicitPrefs(newPrefs, tt.curPrefs, upEnv) var got string - if err := checkForAccidentalSettingReverts(newPrefs, tt.curPrefs, upEnv); err != nil { + if _, err := checkForAccidentalSettingReverts(newPrefs, tt.curPrefs, upEnv); err != nil { got = err.Error() } if strings.TrimSpace(got) != tt.want { @@ -737,7 +770,22 @@ func TestPrefsFromUpArgs(t *testing.T) { args: upArgsT{ exitNodeIP: "foo", }, - wantErr: `invalid value "foo" for --exit-node; must be IP or unique node name`, + st: &ipnstate.Status{ + Peer: map[key.NodePublic]*ipnstate.PeerStatus{ + key.NewNode().Public(): { + DNSName: "example.com.", + TailscaleIPs: []netip.Addr{netip.MustParseAddr("1.0.0.2")}, + }, + }, + }, + wantErr: `invalid value "foo" for --exit-node; must be IP or peer hostname`, + }, + { + name: "error_exit_node_not_started", + args: upArgsT{ + exitNodeIP: "foo", + }, + wantErr: `cannot resolve exit node by hostname while Tailscale is starting up; please use its Tailscale IP address instead`, }, { name: "error_exit_node_allow_lan_without_exit_node", @@ -747,11 +795,43 @@ func TestPrefsFromUpArgs(t *testing.T) { wantErr: `--exit-node-allow-lan-access can only be used with --exit-node`, }, { - name: "error_tag_prefix", + name: "error_tag_bad_prefix", args: upArgsT{ - advertiseTags: "foo", + advertiseTags: "notatag:foo", + }, + wantErr: `tag: "notatag:foo": tags must start with 'tag:'`, + }, + { + name: "tag_auto_prefix", + args: upArgsFromOSArgs("linux", "--advertise-tags=foo,bar"), + want: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + WantRunning: true, + CorpDNS: true, + AdvertiseTags: []string{"tag:foo", "tag:bar"}, + NoSNAT: false, + NoStatefulFiltering: "true", + NetfilterMode: preftype.NetfilterOn, + AutoUpdate: ipn.AutoUpdatePrefs{ + Check: true, + }, + }, + }, + { + name: "tag_mixed_prefix", + args: upArgsFromOSArgs("linux", "--advertise-tags=tag:foo,bar"), + want: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + WantRunning: true, + CorpDNS: true, + AdvertiseTags: []string{"tag:foo", "tag:bar"}, + NoSNAT: false, + NoStatefulFiltering: "true", + NetfilterMode: preftype.NetfilterOn, + AutoUpdate: ipn.AutoUpdatePrefs{ + Check: true, + }, }, - wantErr: `tag: "foo": tags must start with 'tag:'`, }, { name: "error_long_hostname", @@ -930,8 +1010,8 @@ func TestPrefFlagMapping(t *testing.T) { } prefType := reflect.TypeFor[ipn.Prefs]() - for i := range prefType.NumField() { - prefName := prefType.Field(i).Name + for field := range prefType.Fields() { + prefName := field.Name if prefHasFlag[prefName] { continue } @@ -964,13 +1044,16 @@ func TestPrefFlagMapping(t *testing.T) { // flag for this. continue case "AdvertiseServices": - // Handled by the tailscale advertise subcommand, we don't want a + // Handled by the tailscale serve subcommand, we don't want a // CLI flag for this. continue case "InternalExitNodePrior": // Used internally by LocalBackend as part of exit node usage toggling. // No CLI flag for this. continue + case "AutoExitNode": + // Handled by tailscale {set,up} --exit-node=auto:any. + continue } t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName) } @@ -1008,13 +1091,10 @@ func TestUpdatePrefs(t *testing.T) { wantErrSubtr string }{ { - name: "bare_up_means_up", - flags: []string{}, - curPrefs: &ipn.Prefs{ - ControlURL: ipn.DefaultControlURL, - WantRunning: false, - Hostname: "foo", - }, + name: "bare_up_means_up", + flags: []string{}, + curPrefs: ipn.NewPrefs(), + wantSimpleUp: false, // user profile not set, so no simple up }, { name: "just_up", @@ -1028,6 +1108,32 @@ func TestUpdatePrefs(t *testing.T) { }, wantSimpleUp: true, }, + { + name: "just_up_needs_login_default_prefs", + flags: []string{}, + curPrefs: ipn.NewPrefs(), + env: upCheckEnv{ + backendState: "NeedsLogin", + }, + wantSimpleUp: false, + }, + { + name: "just_up_needs_login_losing_prefs", + flags: []string{}, + curPrefs: &ipn.Prefs{ + // defaults: + ControlURL: ipn.DefaultControlURL, + WantRunning: false, + NetfilterMode: preftype.NetfilterOn, + // non-default: + CorpDNS: false, + }, + env: upCheckEnv{ + backendState: "NeedsLogin", + }, + wantSimpleUp: false, + wantErrSubtr: "tailscale up --accept-dns=false", + }, { name: "just_edit", flags: []string{}, @@ -1334,6 +1440,27 @@ func TestUpdatePrefs(t *testing.T) { } }, }, + { + name: "auto_exit_node", + flags: []string{"--exit-node=auto:any"}, + curPrefs: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + CorpDNS: true, // enabled by [ipn.NewPrefs] by default + NetfilterMode: preftype.NetfilterOn, // enabled by [ipn.NewPrefs] by default + }, + wantJustEditMP: &ipn.MaskedPrefs{ + WantRunningSet: true, // enabled by default for tailscale up + AutoExitNodeSet: true, + ExitNodeIDSet: true, // we want ExitNodeID cleared + ExitNodeIPSet: true, // same for ExitNodeIP + }, + env: upCheckEnv{backendState: "Running"}, + checkUpdatePrefsMutations: func(t *testing.T, newPrefs *ipn.Prefs) { + if newPrefs.AutoExitNode != ipn.AnyExitNode { + t.Errorf("AutoExitNode: got %q; want %q", newPrefs.AutoExitNode, ipn.AnyExitNode) + } + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1454,13 +1581,13 @@ func TestParseNLArgs(t *testing.T) { parseDisablements: true, }, { - name: "key no votes", + name: "key-no-votes", input: []string{"nlpub:" + strings.Repeat("00", 32)}, parseKeys: true, wantKeys: []tka.Key{{Kind: tka.Key25519, Votes: 1, Public: bytes.Repeat([]byte{0}, 32)}}, }, { - name: "key with votes", + name: "key-with-votes", input: []string{"nlpub:" + strings.Repeat("01", 32) + "?5"}, parseKeys: true, wantKeys: []tka.Key{{Kind: tka.Key25519, Votes: 5, Public: bytes.Repeat([]byte{1}, 32)}}, @@ -1472,13 +1599,13 @@ func TestParseNLArgs(t *testing.T) { wantDisablements: [][]byte{bytes.Repeat([]byte{2}, 32), bytes.Repeat([]byte{3}, 32)}, }, { - name: "disablements not allowed", + name: "disablements-not-allowed", input: []string{"disablement:" + strings.Repeat("02", 32)}, parseKeys: true, wantErr: fmt.Errorf("parsing key 1: key hex string doesn't have expected type prefix tlpub:"), }, { - name: "keys not allowed", + name: "keys-not-allowed", input: []string{"nlpub:" + strings.Repeat("02", 32)}, parseDisablements: true, wantErr: fmt.Errorf("parsing argument 1: expected value with \"disablement:\" or \"disablement-secret:\" prefix, got %q", "nlpub:0202020202020202020202020202020202020202020202020202020202020202"), @@ -1487,7 +1614,7 @@ func TestParseNLArgs(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - keys, disablements, err := parseNLArgs(tc.input, tc.parseKeys, tc.parseDisablements) + keys, disablements, err := parseTLArgs(tc.input, tc.parseKeys, tc.parseDisablements) if (tc.wantErr == nil && err != nil) || (tc.wantErr != nil && err == nil) || (tc.wantErr != nil && err != nil && tc.wantErr.Error() != err.Error()) { @@ -1539,7 +1666,7 @@ func TestNoDups(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cmd := newRootCmd() + cmd := newRootCmd(t) makeQuietContinueOnError(cmd) err := cmd.Parse(tt.args) if got := fmt.Sprint(err); got != tt.want { @@ -1621,6 +1748,78 @@ func TestDocs(t *testing.T) { walk(t, root) } +func TestUpResolves(t *testing.T) { + const testARN = "arn:aws:ssm:us-east-1:123456789012:parameter/my-parameter" + undo := tailscale.HookResolveValueFromParameterStore.SetForTest(func(_ context.Context, valueOrARN string) (string, error) { + if valueOrARN == testARN { + return "resolved-value", nil + } + return valueOrARN, nil + }) + defer undo() + + const content = "file-content" + fpath := filepath.Join(t.TempDir(), "testfile") + if err := os.WriteFile(fpath, []byte(content), 0600); err != nil { + t.Fatal(err) + } + + testCases := []struct { + name string + arg string + want string + }{ + {"parameter_store", testARN, "resolved-value"}, + {"file", "file:" + fpath, "file-content"}, + } + + for _, tt := range testCases { + t.Run(tt.name+"_auth_key", func(t *testing.T) { + args := upArgsT{authKeyOrFile: tt.arg} + got, err := args.getAuthKey(t.Context()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + + t.Run(tt.name+"_client_secret", func(t *testing.T) { + args := upArgsT{clientSecretOrFile: tt.arg} + got, err := args.getClientSecret(t.Context()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + + t.Run(tt.name+"_id_token", func(t *testing.T) { + args := upArgsT{idTokenOrFile: tt.arg} + got, err := args.getIDToken(t.Context()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } + + t.Run("passthrough", func(t *testing.T) { + args := upArgsT{authKeyOrFile: "tskey-abcd1234"} + got, err := args.getAuthKey(t.Context()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "tskey-abcd1234" { + t.Errorf("got %q, want %q", got, "tskey-abcd1234") + } + }) +} + func TestDeps(t *testing.T) { deptest.DepChecker{ GOOS: "linux", @@ -1648,3 +1847,21 @@ func TestDepsNoCapture(t *testing.T) { }.Check(t) } + +func TestSanitizeWriter(t *testing.T) { + buf := new(bytes.Buffer) + w := sanitizeOutput(buf) + + in := []byte(`my auth key is tskey-auth-abc123-def456 and tskey-foo, what's yours?`) + want := []byte(`my auth key is tskey-XXXXXXXXXXXXXXXXXX and tskey-XXX, what's yours?`) + n, err := w.Write(in) + if err != nil { + t.Fatal(err) + } + if n != len(in) { + t.Errorf("unexpected write length %d, want %d", n, len(in)) + } + if got := buf.Bytes(); !bytes.Equal(got, want) { + t.Errorf("unexpected sanitized content\ngot: %q\nwant: %q", got, want) + } +} diff --git a/cmd/tailscale/cli/colorable.go b/cmd/tailscale/cli/colorable.go new file mode 100644 index 0000000000000..6ecd36b1a409f --- /dev/null +++ b/cmd/tailscale/cli/colorable.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_colorable + +package cli + +import ( + "io" + "os" + + "github.com/mattn/go-colorable" + "github.com/mattn/go-isatty" +) + +// colorableOutput returns a colorable writer if stdout is a terminal (not, say, +// redirected to a file or pipe), the Stdout writer is os.Stdout (we're not +// embedding the CLI in wasm or a mobile app), and NO_COLOR is not set (see +// https://no-color.org/). If any of those is not the case, ok is false +// and w is Stdout. +func colorableOutput() (w io.Writer, ok bool) { + if Stdout != os.Stdout || + os.Getenv("NO_COLOR") != "" || + !isatty.IsTerminal(os.Stdout.Fd()) { + return Stdout, false + } + return colorable.NewColorableStdout(), true +} diff --git a/cmd/tailscale/cli/colorable_omit.go b/cmd/tailscale/cli/colorable_omit.go new file mode 100644 index 0000000000000..a821bdbbdc92e --- /dev/null +++ b/cmd/tailscale/cli/colorable_omit.go @@ -0,0 +1,12 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_omit_colorable + +package cli + +import "io" + +func colorableOutput() (w io.Writer, ok bool) { + return Stdout, false +} diff --git a/cmd/tailscale/cli/configure-jetkvm.go b/cmd/tailscale/cli/configure-jetkvm.go new file mode 100644 index 0000000000000..1956ac836fe74 --- /dev/null +++ b/cmd/tailscale/cli/configure-jetkvm.go @@ -0,0 +1,84 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux && !android && arm + +package cli + +import ( + "bytes" + "context" + "errors" + "flag" + "os" + "runtime" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/version/distro" +) + +func init() { + maybeJetKVMConfigureCmd = jetKVMConfigureCmd +} + +func jetKVMConfigureCmd() *ffcli.Command { + if runtime.GOOS != "linux" || distro.Get() != distro.JetKVM { + return nil + } + return &ffcli.Command{ + Name: "jetkvm", + Exec: runConfigureJetKVM, + ShortUsage: "tailscale configure jetkvm", + ShortHelp: "Configure JetKVM to run tailscaled at boot", + LongHelp: strings.TrimSpace(` +This command configures the JetKVM host to run tailscaled at boot. +`), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("jetkvm") + return fs + })(), + } +} + +func runConfigureJetKVM(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unknown arguments") + } + if runtime.GOOS != "linux" || distro.Get() != distro.JetKVM { + return errors.New("only implemented on JetKVM") + } + if err := os.MkdirAll("/userdata/init.d", 0755); err != nil { + return errors.New("unable to create /userdata/init.d") + } + err := os.WriteFile("/userdata/init.d/S22tailscale", bytes.TrimLeft([]byte(` +#!/bin/sh +# /userdata/init.d/S22tailscale +# Start/stop tailscaled + +case "$1" in + start) + /userdata/tailscale/tailscaled > /dev/null 2>&1 & + ;; + stop) + killall tailscaled + ;; + *) + echo "Usage: $0 {start|stop}" + exit 1 + ;; +esac +`), "\n"), 0755) + if err != nil { + return err + } + + if err := os.Symlink("/userdata/tailscale/tailscale", "/bin/tailscale"); err != nil { + if !os.IsExist(err) { + return err + } + } + + printf("Done. Now restart your JetKVM.\n") + return nil +} diff --git a/cmd/tailscale/cli/configure-kube.go b/cmd/tailscale/cli/configure-kube.go index 6bc4e202efd4e..8160025c6858e 100644 --- a/cmd/tailscale/cli/configure-kube.go +++ b/cmd/tailscale/cli/configure-kube.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !ts_omit_kube @@ -9,17 +9,27 @@ import ( "errors" "flag" "fmt" + "net/netip" + "net/url" "os" "path/filepath" "slices" "strings" + "time" "github.com/peterbourgon/ff/v3/ffcli" "k8s.io/client-go/util/homedir" "sigs.k8s.io/yaml" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/util/dnsname" "tailscale.com/version" ) +var configureKubeconfigArgs struct { + http bool // Use HTTP instead of HTTPS (default) for the auth proxy. +} + func configureKubeconfigCmd() *ffcli.Command { return &ffcli.Command{ Name: "kubeconfig", @@ -34,6 +44,7 @@ See: https://tailscale.com/s/k8s-auth-proxy `), FlagSet: (func() *flag.FlagSet { fs := newFlagSet("kubeconfig") + fs.BoolVar(&configureKubeconfigArgs.http, "http", false, "Use HTTP instead of HTTPS to connect to the auth proxy. Ignored if you include a scheme in the hostname argument.") return fs })(), Exec: runConfigureKubeconfig, @@ -70,10 +81,13 @@ func kubeconfigPath() (string, error) { } func runConfigureKubeconfig(ctx context.Context, args []string) error { - if len(args) != 1 { - return errors.New("unknown arguments") + if len(args) != 1 || args[0] == "" { + return flag.ErrHelp + } + hostOrFQDNOrIP, http, err := getInputs(args[0], configureKubeconfigArgs.http) + if err != nil { + return fmt.Errorf("error parsing inputs: %w", err) } - hostOrFQDN := args[0] st, err := localClient.Status(ctx) if err != nil { @@ -82,22 +96,45 @@ func runConfigureKubeconfig(ctx context.Context, args []string) error { if st.BackendState != "Running" { return errors.New("Tailscale is not running") } - targetFQDN, ok := nodeDNSNameFromArg(st, hostOrFQDN) - if !ok { - return fmt.Errorf("no peer found with hostname %q", hostOrFQDN) + dnsCfg, err := getDNSConfig(ctx) + if err != nil { + return err + } + + targetFQDN, err := nodeOrServiceDNSNameFromArg(st, dnsCfg, hostOrFQDNOrIP) + if err != nil { + return err } targetFQDN = strings.TrimSuffix(targetFQDN, ".") var kubeconfig string if kubeconfig, err = kubeconfigPath(); err != nil { return err } - if err = setKubeconfigForPeer(targetFQDN, kubeconfig); err != nil { + scheme := "https://" + if http { + scheme = "http://" + } + if err = setKubeconfigForPeer(scheme, targetFQDN, kubeconfig); err != nil { return err } - printf("kubeconfig configured for %q\n", hostOrFQDN) + printf("kubeconfig configured for %q at URL %q\n", targetFQDN, scheme+targetFQDN) return nil } +func getInputs(arg string, httpArg bool) (string, bool, error) { + u, err := url.Parse(arg) + if err != nil { + return "", false, err + } + + switch u.Scheme { + case "http", "https": + return u.Host, u.Scheme == "http", nil + default: + return arg, httpArg, nil + } +} + // appendOrSetNamed finds a map with a "name" key matching name in dst, and // replaces it with val. If no such map is found, val is appended to dst. func appendOrSetNamed(dst []any, name string, val map[string]any) []any { @@ -116,7 +153,7 @@ func appendOrSetNamed(dst []any, name string, val map[string]any) []any { var errInvalidKubeconfig = errors.New("invalid kubeconfig") -func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { +func updateKubeconfig(cfgYaml []byte, scheme, fqdn string) ([]byte, error) { var cfg map[string]any if len(cfgYaml) > 0 { if err := yaml.Unmarshal(cfgYaml, &cfg); err != nil { @@ -139,7 +176,7 @@ func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { cfg["clusters"] = appendOrSetNamed(clusters, fqdn, map[string]any{ "name": fqdn, "cluster": map[string]string{ - "server": "https://" + fqdn, + "server": scheme + fqdn, }, }) @@ -172,7 +209,7 @@ func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { return yaml.Marshal(cfg) } -func setKubeconfigForPeer(fqdn, filePath string) error { +func setKubeconfigForPeer(scheme, fqdn, filePath string) error { dir := filepath.Dir(filePath) if _, err := os.Stat(dir); err != nil { if !os.IsNotExist(err) { @@ -191,9 +228,85 @@ func setKubeconfigForPeer(fqdn, filePath string) error { if err != nil && !os.IsNotExist(err) { return fmt.Errorf("reading kubeconfig: %w", err) } - b, err = updateKubeconfig(b, fqdn) + b, err = updateKubeconfig(b, scheme, fqdn) if err != nil { return err } return os.WriteFile(filePath, b, 0600) } + +// nodeOrServiceDNSNameFromArg returns the PeerStatus.DNSName value from a peer +// in st that matches the input arg which can be a base name, full DNS name, or +// an IP. If none is found, it looks for a Tailscale Service +func nodeOrServiceDNSNameFromArg(st *ipnstate.Status, dns *tailcfg.DNSConfig, arg string) (string, error) { + // First check for a node DNS name. + if dnsName, ok := nodeDNSNameFromArg(st, arg); ok { + return dnsName, nil + } + + // If not found, check for a Tailscale Service DNS name. + rec, ok := serviceDNSRecordFromDNSConfig(dns, arg) + if !ok { + return "", fmt.Errorf("no peer found for %q", arg) + } + + // Validate we can see a peer advertising the Tailscale Service. + ip, err := netip.ParseAddr(rec.Value) + if err != nil { + return "", fmt.Errorf("error parsing ExtraRecord IP address %q: %w", rec.Value, err) + } + ipPrefix := netip.PrefixFrom(ip, ip.BitLen()) + for _, ps := range st.Peer { + for _, allowedIP := range ps.AllowedIPs.All() { + if allowedIP == ipPrefix { + return rec.Name, nil + } + } + } + + return "", fmt.Errorf("%q is in MagicDNS, but is not currently reachable on any known peer", arg) +} + +func getDNSConfig(ctx context.Context) (*tailcfg.DNSConfig, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return localClient.DNSConfig(ctx) +} + +func serviceDNSRecordFromDNSConfig(dns *tailcfg.DNSConfig, arg string) (rec tailcfg.DNSRecord, ok bool) { + argIP, _ := netip.ParseAddr(arg) + argFQDN, err := dnsname.ToFQDN(arg) + argFQDNValid := err == nil + if !argIP.IsValid() && !argFQDNValid { + return rec, false + } + + for _, rec := range dns.ExtraRecords { + if argIP.IsValid() { + recIP, _ := netip.ParseAddr(rec.Value) + if recIP == argIP { + return rec, true + } + continue + } + + if !argFQDNValid { + continue + } + + recFirstLabel := dnsname.FirstLabel(rec.Name) + if strings.EqualFold(arg, recFirstLabel) { + return rec, true + } + + recFQDN, err := dnsname.ToFQDN(rec.Name) + if err != nil { + continue + } + if strings.EqualFold(argFQDN.WithTrailingDot(), recFQDN.WithTrailingDot()) { + return rec, true + } + } + + return tailcfg.DNSRecord{}, false +} diff --git a/cmd/tailscale/cli/configure-kube_omit.go b/cmd/tailscale/cli/configure-kube_omit.go index 130f2870fab44..946fa2294d5aa 100644 --- a/cmd/tailscale/cli/configure-kube_omit.go +++ b/cmd/tailscale/cli/configure-kube_omit.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build ts_omit_kube diff --git a/cmd/tailscale/cli/configure-kube_test.go b/cmd/tailscale/cli/configure-kube_test.go index d71a9b627e7f0..2c2a05ac0c08f 100644 --- a/cmd/tailscale/cli/configure-kube_test.go +++ b/cmd/tailscale/cli/configure-kube_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !ts_omit_kube @@ -6,6 +6,7 @@ package cli import ( "bytes" + "fmt" "strings" "testing" @@ -16,6 +17,7 @@ func TestKubeconfig(t *testing.T) { const fqdn = "foo.tail-scale.ts.net" tests := []struct { name string + http bool in string want string wantErr error @@ -53,7 +55,28 @@ users: token: unused`, }, { - name: "all configs, clusters, users have been deleted", + name: "empty_http", + http: true, + in: "", + want: `apiVersion: v1 +clusters: +- cluster: + server: http://foo.tail-scale.ts.net + name: foo.tail-scale.ts.net +contexts: +- context: + cluster: foo.tail-scale.ts.net + user: tailscale-auth + name: foo.tail-scale.ts.net +current-context: foo.tail-scale.ts.net +kind: Config +users: +- name: tailscale-auth + user: + token: unused`, + }, + { + name: "all-configs-clusters-users-deleted", in: `apiVersion: v1 clusters: null contexts: null @@ -202,7 +225,11 @@ users: } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := updateKubeconfig([]byte(tt.in), fqdn) + scheme := "https://" + if tt.http { + scheme = "http://" + } + got, err := updateKubeconfig([]byte(tt.in), scheme, fqdn) if err != nil { if err != tt.wantErr { t.Fatalf("updateKubeconfig() error = %v, wantErr %v", err, tt.wantErr) @@ -219,3 +246,30 @@ users: }) } } + +func TestGetInputs(t *testing.T) { + for _, arg := range []string{ + "foo.tail-scale.ts.net", + "foo", + "127.0.0.1", + } { + for _, prefix := range []string{"", "https://", "http://"} { + for _, httpFlag := range []bool{false, true} { + expectedHost := arg + expectedHTTP := (httpFlag && !strings.HasPrefix(prefix, "https://")) || strings.HasPrefix(prefix, "http://") + t.Run(fmt.Sprintf("%s%s_http=%v", prefix, arg, httpFlag), func(t *testing.T) { + host, http, err := getInputs(prefix+arg, httpFlag) + if err != nil { + t.Fatal(err) + } + if host != expectedHost { + t.Errorf("host = %v, want %v", host, expectedHost) + } + if http != expectedHTTP { + t.Errorf("http = %v, want %v", http, expectedHTTP) + } + }) + } + } + } +} diff --git a/cmd/tailscale/cli/configure-synology-cert.go b/cmd/tailscale/cli/configure-synology-cert.go index 663d0c8790456..32f5bbd70593c 100644 --- a/cmd/tailscale/cli/configure-synology-cert.go +++ b/cmd/tailscale/cli/configure-synology-cert.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build linux && !ts_omit_acme && !ts_omit_synology + package cli import ( @@ -14,6 +16,7 @@ import ( "os/exec" "path" "runtime" + "slices" "strings" "github.com/peterbourgon/ff/v3/ffcli" @@ -22,6 +25,10 @@ import ( "tailscale.com/version/distro" ) +func init() { + maybeConfigSynologyCertCmd = synologyConfigureCertCmd +} + func synologyConfigureCertCmd() *ffcli.Command { if runtime.GOOS != "linux" || distro.Get() != distro.Synology { return nil @@ -79,11 +86,8 @@ func runConfigureSynologyCert(ctx context.Context, args []string) error { domain = st.CertDomains[0] } else { var found bool - for _, d := range st.CertDomains { - if d == domain { - found = true - break - } + if slices.Contains(st.CertDomains, domain) { + found = true } if !found { return fmt.Errorf("Domain %q was not one of the valid domain options: %q.", domain, st.CertDomains) diff --git a/cmd/tailscale/cli/configure-synology-cert_test.go b/cmd/tailscale/cli/configure-synology-cert_test.go index 801285e550d9b..d79ceb9d362b8 100644 --- a/cmd/tailscale/cli/configure-synology-cert_test.go +++ b/cmd/tailscale/cli/configure-synology-cert_test.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build linux && !ts_omit_acme + package cli import ( @@ -28,7 +30,7 @@ func Test_listCerts(t *testing.T) { wantErr bool }{ { - name: "normal response", + name: "normal-response", caller: fakeAPICaller{ Data: json.RawMessage(`{ "certificates" : [ @@ -115,12 +117,12 @@ func Test_listCerts(t *testing.T) { }, }, { - name: "call error", + name: "call-error", caller: fakeAPICaller{nil, fmt.Errorf("caller failed")}, wantErr: true, }, { - name: "payload decode error", + name: "payload-decode-error", caller: fakeAPICaller{json.RawMessage("This isn't JSON!"), nil}, wantErr: true, }, diff --git a/cmd/tailscale/cli/configure-synology.go b/cmd/tailscale/cli/configure-synology.go index f0f05f75765b9..4cfd4160e066a 100644 --- a/cmd/tailscale/cli/configure-synology.go +++ b/cmd/tailscale/cli/configure-synology.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/configure.go b/cmd/tailscale/cli/configure.go index acb416755a586..e7a6448e70822 100644 --- a/cmd/tailscale/cli/configure.go +++ b/cmd/tailscale/cli/configure.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -10,6 +10,12 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" ) +var ( + maybeJetKVMConfigureCmd, + maybeConfigSynologyCertCmd, + _ func() *ffcli.Command // non-nil only on Linux/arm for JetKVM +) + func configureCmd() *ffcli.Command { return &ffcli.Command{ Name: "configure", @@ -26,9 +32,11 @@ services on the host to use Tailscale in more ways. Subcommands: nonNilCmds( configureKubeconfigCmd(), synologyConfigureCmd(), - synologyConfigureCertCmd(), + ccall(maybeConfigSynologyCertCmd), ccall(maybeSysExtCmd), ccall(maybeVPNConfigCmd), + ccall(maybeJetKVMConfigureCmd), + ccall(maybeSystrayCmd), ), } } diff --git a/cmd/tailscale/cli/configure_apple-all.go b/cmd/tailscale/cli/configure_apple-all.go index 5f0da9b95420e..95e9259e96cf7 100644 --- a/cmd/tailscale/cli/configure_apple-all.go +++ b/cmd/tailscale/cli/configure_apple-all.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/configure_apple.go b/cmd/tailscale/cli/configure_apple.go index c0d99b90aa2c4..465bc7a47ed2c 100644 --- a/cmd/tailscale/cli/configure_apple.go +++ b/cmd/tailscale/cli/configure_apple.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build darwin diff --git a/cmd/tailscale/cli/configure_linux-all.go b/cmd/tailscale/cli/configure_linux-all.go new file mode 100644 index 0000000000000..2db970eeef497 --- /dev/null +++ b/cmd/tailscale/cli/configure_linux-all.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import "github.com/peterbourgon/ff/v3/ffcli" + +var maybeSystrayCmd func() *ffcli.Command // non-nil only on Linux, see configure_linux.go diff --git a/cmd/tailscale/cli/configure_linux.go b/cmd/tailscale/cli/configure_linux.go new file mode 100644 index 0000000000000..da04449087558 --- /dev/null +++ b/cmd/tailscale/cli/configure_linux.go @@ -0,0 +1,51 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux && !ts_omit_systray + +package cli + +import ( + "context" + "flag" + "fmt" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/systray" +) + +func init() { + maybeSystrayCmd = systrayConfigCmd +} + +var configSystrayArgs struct { + initSystem string + installStartup bool +} + +func systrayConfigCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "systray", + ShortUsage: "tailscale configure systray [options]", + ShortHelp: "[ALPHA] Manage the systray client for Linux", + LongHelp: "[ALPHA] The systray set of commands provides a way to configure the systray application on Linux.", + Exec: configureSystray, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("systray") + fs.StringVar(&configSystrayArgs.initSystem, "enable-startup", "", + "Install startup script for init system. Currently supported systems are [systemd, freedesktop].") + return fs + })(), + } +} + +func configureSystray(_ context.Context, _ []string) error { + if configSystrayArgs.initSystem != "" { + if err := systray.InstallStartupScript(configSystrayArgs.initSystem); err != nil { + fmt.Printf("%s\n\n", err.Error()) + return flag.ErrHelp + } + return nil + } + return flag.ErrHelp +} diff --git a/cmd/tailscale/cli/debug-cachenetmap.go b/cmd/tailscale/cli/debug-cachenetmap.go new file mode 100644 index 0000000000000..735469ee42c3d --- /dev/null +++ b/cmd/tailscale/cli/debug-cachenetmap.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ios && !ts_omit_cachenetmap + +package cli + +import ( + "context" + "errors" + + "github.com/peterbourgon/ff/v3/ffcli" +) + +func init() { + debugClearNetmapCacheCmd = func() *ffcli.Command { + return &ffcli.Command{ + Name: "clear-netmap-cache", + ShortUsage: "tailscale debug clear-netmap-cache", + ShortHelp: "Remove and discard cached network maps (if any)", + Exec: runDebugClearNetmapCache, + } + } +} + +func runDebugClearNetmapCache(ctx context.Context, args []string) error { + if len(args) != 0 { + return errors.New("unexpected arguments") + } + return localClient.DebugAction(ctx, "clear-netmap-cache") +} diff --git a/cmd/tailscale/cli/debug-capture.go b/cmd/tailscale/cli/debug-capture.go index a54066fa614cb..ce282b291a587 100644 --- a/cmd/tailscale/cli/debug-capture.go +++ b/cmd/tailscale/cli/debug-capture.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !ios && !ts_omit_capture diff --git a/cmd/tailscale/cli/debug-peer-relay.go b/cmd/tailscale/cli/debug-peer-relay.go new file mode 100644 index 0000000000000..1b28c3f6bb1a4 --- /dev/null +++ b/cmd/tailscale/cli/debug-peer-relay.go @@ -0,0 +1,77 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ios && !ts_omit_relayserver + +package cli + +import ( + "bytes" + "cmp" + "context" + "fmt" + "net/netip" + "slices" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/net/udprelay/status" +) + +func init() { + debugPeerRelayCmd = mkDebugPeerRelaySessionsCmd +} + +func mkDebugPeerRelaySessionsCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "peer-relay-sessions", + ShortUsage: "tailscale debug peer-relay-sessions", + Exec: runPeerRelaySessions, + ShortHelp: "Print the current set of active peer relay sessions relayed through this node", + } +} + +func runPeerRelaySessions(ctx context.Context, args []string) error { + srv, err := localClient.DebugPeerRelaySessions(ctx) + if err != nil { + return err + } + + var buf bytes.Buffer + f := func(format string, a ...any) { fmt.Fprintf(&buf, format, a...) } + + f("Server port: ") + if srv.UDPPort == nil { + f("not configured (you can configure the port with 'tailscale set --relay-server-port=')") + } else { + f("%d", *srv.UDPPort) + } + f("\n") + f("Sessions count: %d\n", len(srv.Sessions)) + if len(srv.Sessions) == 0 { + Stdout.Write(buf.Bytes()) + return nil + } + + fmtSessionDirection := func(a, z status.ClientInfo) string { + fmtEndpoint := func(ap netip.AddrPort) string { + if ap.IsValid() { + return ap.String() + } + return "" + } + return fmt.Sprintf("%s(%s) --> %s(%s), Packets: %d Bytes: %d", + fmtEndpoint(a.Endpoint), a.ShortDisco, + fmtEndpoint(z.Endpoint), z.ShortDisco, + a.PacketsTx, a.BytesTx) + } + + f("\n") + slices.SortFunc(srv.Sessions, func(s1, s2 status.ServerSession) int { return cmp.Compare(s1.VNI, s2.VNI) }) + for _, s := range srv.Sessions { + f("VNI: %d\n", s.VNI) + f(" %s\n", fmtSessionDirection(s.Client1, s.Client2)) + f(" %s\n", fmtSessionDirection(s.Client2, s.Client1)) + } + Stdout.Write(buf.Bytes()) + return nil +} diff --git a/cmd/tailscale/cli/debug-portmap.go b/cmd/tailscale/cli/debug-portmap.go new file mode 100644 index 0000000000000..a876971ef00b4 --- /dev/null +++ b/cmd/tailscale/cli/debug-portmap.go @@ -0,0 +1,79 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ios && !ts_omit_debugportmapper + +package cli + +import ( + "context" + "flag" + "fmt" + "io" + "net/netip" + "os" + "time" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/local" +) + +func init() { + debugPortmapCmd = mkDebugPortmapCmd +} + +func mkDebugPortmapCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "portmap", + ShortUsage: "tailscale debug portmap", + Exec: debugPortmap, + ShortHelp: "Run portmap debugging", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("portmap") + fs.DurationVar(&debugPortmapArgs.duration, "duration", 5*time.Second, "timeout for port mapping") + fs.StringVar(&debugPortmapArgs.ty, "type", "", `portmap debug type (one of "", "pmp", "pcp", or "upnp")`) + fs.StringVar(&debugPortmapArgs.gatewayAddr, "gateway-addr", "", `override gateway IP (must also pass --self-addr)`) + fs.StringVar(&debugPortmapArgs.selfAddr, "self-addr", "", `override self IP (must also pass --gateway-addr)`) + fs.BoolVar(&debugPortmapArgs.logHTTP, "log-http", false, `print all HTTP requests and responses to the log`) + return fs + })(), + } +} + +var debugPortmapArgs struct { + duration time.Duration + gatewayAddr string + selfAddr string + ty string + logHTTP bool +} + +func debugPortmap(ctx context.Context, args []string) error { + opts := &local.DebugPortmapOpts{ + Duration: debugPortmapArgs.duration, + Type: debugPortmapArgs.ty, + LogHTTP: debugPortmapArgs.logHTTP, + } + if (debugPortmapArgs.gatewayAddr != "") != (debugPortmapArgs.selfAddr != "") { + return fmt.Errorf("if one of --gateway-addr and --self-addr is provided, the other must be as well") + } + if debugPortmapArgs.gatewayAddr != "" { + var err error + opts.GatewayAddr, err = netip.ParseAddr(debugPortmapArgs.gatewayAddr) + if err != nil { + return fmt.Errorf("invalid --gateway-addr: %w", err) + } + opts.SelfAddr, err = netip.ParseAddr(debugPortmapArgs.selfAddr) + if err != nil { + return fmt.Errorf("invalid --self-addr: %w", err) + } + } + rc, err := localClient.DebugPortmap(ctx, opts) + if err != nil { + return err + } + defer rc.Close() + + _, err = io.Copy(os.Stdout, rc) + return err +} diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index ec8a0700dec19..2697ac5d1f8e0 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -6,6 +6,7 @@ package cli import ( "bufio" "bytes" + "cmp" "context" "encoding/binary" "encoding/json" @@ -16,6 +17,7 @@ import ( "log" "net" "net/http" + "net/http/httptrace" "net/http/httputil" "net/netip" "net/url" @@ -27,17 +29,19 @@ import ( "time" "github.com/peterbourgon/ff/v3/ffcli" - "golang.org/x/net/http/httpproxy" - "golang.org/x/net/http2" - "tailscale.com/client/tailscale" "tailscale.com/client/tailscale/apitype" - "tailscale.com/control/controlhttp" + "tailscale.com/control/ts2021" + "tailscale.com/feature" + _ "tailscale.com/feature/condregister/useproxy" + "tailscale.com/health" "tailscale.com/hostinfo" - "tailscale.com/internal/noiseconn" "tailscale.com/ipn" + "tailscale.com/net/ace" + "tailscale.com/net/dnscache" "tailscale.com/net/netmon" + "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" - "tailscale.com/net/tshttpproxy" + "tailscale.com/net/tsdial" "tailscale.com/paths" "tailscale.com/safesocket" "tailscale.com/tailcfg" @@ -48,7 +52,10 @@ import ( ) var ( - debugCaptureCmd func() *ffcli.Command // or nil + debugCaptureCmd func() *ffcli.Command // or nil + debugPortmapCmd func() *ffcli.Command // or nil + debugPeerRelayCmd func() *ffcli.Command // or nil + debugClearNetmapCacheCmd func() *ffcli.Command // or nil ) func debugCmd() *ffcli.Command { @@ -108,6 +115,23 @@ func debugCmd() *ffcli.Command { Exec: runDaemonBusEvents, ShortHelp: "Watch events on the tailscaled bus", }, + { + Name: "daemon-bus-graph", + ShortUsage: "tailscale debug daemon-bus-graph", + Exec: runDaemonBusGraph, + ShortHelp: "Print graph for the tailscaled bus", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("debug-bus-graph") + fs.StringVar(&daemonBusGraphArgs.format, "format", "json", "output format [json/dot]") + return fs + })(), + }, + { + Name: "daemon-bus-queues", + ShortUsage: "tailscale debug daemon-bus-queues", + Exec: runDaemonBusQueues, + ShortHelp: "Print event bus queue depths per client", + }, { Name: "metrics", ShortUsage: "tailscale debug metrics", @@ -166,6 +190,12 @@ func debugCmd() *ffcli.Command { Exec: localAPIAction("rebind"), ShortHelp: "Force a magicsock rebind", }, + { + Name: "rotate-disco-key", + ShortUsage: "tailscale debug rotate-disco-key", + Exec: localAPIAction("rotate-disco-key"), + ShortHelp: "Rotate the discovery key", + }, { Name: "derp-set-on-demand", ShortUsage: "tailscale debug derp-set-on-demand", @@ -239,10 +269,8 @@ func debugCmd() *ffcli.Command { ShortHelp: "Subscribe to IPN message bus", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("watch-ipn") - fs.BoolVar(&watchIPNArgs.netmap, "netmap", true, "include netmap in messages") - fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include initial status") - fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messags") - fs.BoolVar(&watchIPNArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap") + fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include the initial backend State and Prefs in the first message") + fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messages") fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever") return fs })(), @@ -254,7 +282,6 @@ func debugCmd() *ffcli.Command { ShortHelp: "Print the current network map", FlagSet: (func() *flag.FlagSet { fs := newFlagSet("netmap") - fs.BoolVar(&netmapArgs.showPrivateKey, "show-private-key", false, "include node private key in printed netmap") return fs })(), }, @@ -275,6 +302,8 @@ func debugCmd() *ffcli.Command { fs.StringVar(&ts2021Args.host, "host", "controlplane.tailscale.com", "hostname of control plane") fs.IntVar(&ts2021Args.version, "version", int(tailcfg.CurrentCapabilityVersion), "protocol version") fs.BoolVar(&ts2021Args.verbose, "verbose", false, "be extra verbose") + fs.StringVar(&ts2021Args.aceHost, "ace", "", "if non-empty, use this ACE server IP/hostname as a candidate path") + fs.StringVar(&ts2021Args.dialPlanJSONFile, "dial-plan", "", "if non-empty, use this JSON file to configure the dial plan") return fs })(), }, @@ -307,21 +336,7 @@ func debugCmd() *ffcli.Command { ShortHelp: "Test a DERP configuration", }, ccall(debugCaptureCmd), - { - Name: "portmap", - ShortUsage: "tailscale debug portmap", - Exec: debugPortmap, - ShortHelp: "Run portmap debugging", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("portmap") - fs.DurationVar(&debugPortmapArgs.duration, "duration", 5*time.Second, "timeout for port mapping") - fs.StringVar(&debugPortmapArgs.ty, "type", "", `portmap debug type (one of "", "pmp", "pcp", or "upnp")`) - fs.StringVar(&debugPortmapArgs.gatewayAddr, "gateway-addr", "", `override gateway IP (must also pass --self-addr)`) - fs.StringVar(&debugPortmapArgs.selfAddr, "self-addr", "", `override self IP (must also pass --gateway-addr)`) - fs.BoolVar(&debugPortmapArgs.logHTTP, "log-http", false, `print all HTTP requests and responses to the log`) - return fs - })(), - }, + ccall(debugPortmapCmd), { Name: "peer-endpoint-changes", ShortUsage: "tailscale debug peer-endpoint-changes ", @@ -356,6 +371,31 @@ func debugCmd() *ffcli.Command { ShortHelp: "Print Go's runtime/debug.BuildInfo", Exec: runGoBuildInfo, }, + { + Name: "peer-relay-servers", + ShortUsage: "tailscale debug peer-relay-servers", + ShortHelp: "Print the current set of candidate peer relay servers", + Exec: runPeerRelayServers, + }, + { + Name: "test-risk", + ShortUsage: "tailscale debug test-risk", + ShortHelp: "Do a fake risky action", + Exec: runTestRisk, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("test-risk") + fs.StringVar(&testRiskArgs.acceptedRisk, "accept-risk", "", "comma-separated list of accepted risks") + return fs + })(), + }, + { + Name: "statedir", + ShortUsage: "tailscale debug statedir", + ShortHelp: "Print the location of the state directory (if any)", + Exec: runPrintStateDir, + }, + ccall(debugPeerRelayCmd), + ccall(debugClearNetmapCacheCmd), }...), } } @@ -592,20 +632,15 @@ func runPrefs(ctx context.Context, args []string) error { } var watchIPNArgs struct { - netmap bool - initial bool - showPrivateKey bool - rateLimit bool - count int + initial bool + rateLimit bool + count int } func runWatchIPN(ctx context.Context, args []string) error { - var mask ipn.NotifyWatchOpt + mask := ipn.NotifyPeerChanges | ipn.NotifyPeerPatches if watchIPNArgs.initial { - mask = ipn.NotifyInitialState | ipn.NotifyInitialPrefs | ipn.NotifyInitialNetMap - } - if !watchIPNArgs.showPrivateKey { - mask |= ipn.NotifyNoPrivateKeys + mask |= ipn.NotifyInitialState | ipn.NotifyInitialPrefs } if watchIPNArgs.rateLimit { mask |= ipn.NotifyRateLimit @@ -621,38 +656,21 @@ func runWatchIPN(ctx context.Context, args []string) error { if err != nil { return err } - if !watchIPNArgs.netmap { - n.NetMap = nil - } j, _ := json.MarshalIndent(n, "", "\t") fmt.Printf("%s\n", j) } return nil } -var netmapArgs struct { - showPrivateKey bool -} - func runNetmap(ctx context.Context, args []string) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - var mask ipn.NotifyWatchOpt = ipn.NotifyInitialNetMap - if !netmapArgs.showPrivateKey { - mask |= ipn.NotifyNoPrivateKeys - } - watcher, err := localClient.WatchIPNBus(ctx, mask) - if err != nil { - return err - } - defer watcher.Close() - - n, err := watcher.Next() + raw, err := localClient.DebugResultJSON(ctx, "current-netmap") if err != nil { return err } - j, _ := json.MarshalIndent(n.NetMap, "", "\t") + j, _ := json.MarshalIndent(raw, "", "\t") fmt.Printf("%s\n", j) return nil } @@ -769,10 +787,13 @@ func runDaemonLogs(ctx context.Context, args []string) error { } d := json.NewDecoder(logs) for { + type logtail struct { + Time string `json:"client_time"` + } var line struct { - Text string `json:"text"` - Verbose int `json:"v"` - Time string `json:"client_time"` + Text string `json:"text"` + Verbose int `json:"v"` + Logtail logtail `json:"logtail"` } err := d.Decode(&line) if err != nil { @@ -782,8 +803,8 @@ func runDaemonLogs(ctx context.Context, args []string) error { if line.Text == "" || line.Verbose > daemonLogsArgs.verbose { continue } - if daemonLogsArgs.time { - fmt.Printf("%s %s\n", line.Time, line.Text) + if daemonLogsArgs.time && line.Logtail.Time != "" { + fmt.Printf("%s %s\n", line.Logtail.Time, line.Text) } else { fmt.Println(line.Text) } @@ -801,6 +822,59 @@ func runDaemonBusEvents(ctx context.Context, args []string) error { return nil } +var daemonBusGraphArgs struct { + format string +} + +func runDaemonBusGraph(ctx context.Context, args []string) error { + graph, err := localClient.EventBusGraph(ctx) + if err != nil { + return err + } + if format := daemonBusGraphArgs.format; format != "json" && format != "dot" { + return fmt.Errorf("unrecognized output format %q", format) + } + if daemonBusGraphArgs.format == "dot" { + var topics eventbus.DebugTopics + if err := json.Unmarshal(graph, &topics); err != nil { + return fmt.Errorf("unable to parse json: %w", err) + } + fmt.Print(generateDOTGraph(topics.Topics)) + } else { + fmt.Print(string(graph)) + } + return nil +} + +func runDaemonBusQueues(ctx context.Context, args []string) error { + data, err := localClient.EventBusQueues(ctx) + if err != nil { + return err + } + fmt.Print(string(data)) + return nil +} + +// generateDOTGraph generates the DOT graph format based on the events +func generateDOTGraph(topics []eventbus.DebugTopic) string { + var sb strings.Builder + sb.WriteString("digraph event_bus {\n") + + for _, topic := range topics { + // If no subscribers, still ensure the topic is drawn + if len(topic.Subscribers) == 0 { + topic.Subscribers = append(topic.Subscribers, "no-subscribers") + } + for _, subscriber := range topic.Subscribers { + fmt.Fprintf(&sb, "\t%q -> %q [label=%q];\n", + topic.Publisher, subscriber, cmp.Or(topic.Name, "???")) + } + } + + sb.WriteString("}\n") + return sb.String() +} + var metricsArgs struct { watch bool } @@ -905,6 +979,9 @@ var ts2021Args struct { host string // "controlplane.tailscale.com" version int // 27 or whatever verbose bool + aceHost string // if non-empty, FQDN of https ACE server to use ("ace.example.com") + + dialPlanJSONFile string // if non-empty, path to JSON file [tailcfg.ControlDialPlan] JSON } func runTS2021(ctx context.Context, args []string) error { @@ -913,19 +990,22 @@ func runTS2021(ctx context.Context, args []string) error { keysURL := "https://" + ts2021Args.host + "/key?v=" + strconv.Itoa(ts2021Args.version) + keyTransport := netutil.NewDefaultTransport() + if ts2021Args.aceHost != "" { + log.Printf("using ACE server %q", ts2021Args.aceHost) + keyTransport.Proxy = nil + keyTransport.DialContext = (&ace.Dialer{ACEHost: ts2021Args.aceHost}).Dial + } + if ts2021Args.verbose { u, err := url.Parse(keysURL) if err != nil { return err } - envConf := httpproxy.FromEnvironment() - if *envConf == (httpproxy.Config{}) { - log.Printf("HTTP proxy env: (none)") - } else { - log.Printf("HTTP proxy env: %+v", envConf) + if proxyFromEnv, ok := feature.HookProxyFromEnvironment.GetOk(); ok { + proxy, err := proxyFromEnv(&http.Request{URL: u}) + log.Printf("tshttpproxy.ProxyFromEnvironment = (%v, %v)", proxy, err) } - proxy, err := tshttpproxy.ProxyFromEnvironment(&http.Request{URL: u}) - log.Printf("tshttpproxy.ProxyFromEnvironment = (%v, %v)", proxy, err) } machinePrivate := key.NewMachine() var dialer net.Dialer @@ -938,7 +1018,7 @@ func runTS2021(ctx context.Context, args []string) error { if err != nil { return err } - res, err := http.DefaultClient.Do(req) + res, err := keyTransport.RoundTrip(req) if err != nil { log.Printf("Do: %v", err) return err @@ -982,20 +1062,45 @@ func runTS2021(ctx context.Context, args []string) error { return fmt.Errorf("creating netmon: %w", err) } - noiseDialer := &controlhttp.Dialer{ - Hostname: ts2021Args.host, - HTTPPort: "80", - HTTPSPort: "443", - MachineKey: machinePrivate, - ControlKey: keys.PublicKey, - ProtocolVersion: uint16(ts2021Args.version), - Dialer: dialFunc, - Logf: logf, - NetMon: netMon, + var dialPlan *tailcfg.ControlDialPlan + if ts2021Args.dialPlanJSONFile != "" { + b, err := os.ReadFile(ts2021Args.dialPlanJSONFile) + if err != nil { + return fmt.Errorf("reading dial plan JSON file: %w", err) + } + dialPlan = new(tailcfg.ControlDialPlan) + if err := json.Unmarshal(b, dialPlan); err != nil { + return fmt.Errorf("unmarshaling dial plan JSON file: %w", err) + } + } else if ts2021Args.aceHost != "" { + dialPlan = &tailcfg.ControlDialPlan{ + Candidates: []tailcfg.ControlIPCandidate{ + { + ACEHost: ts2021Args.aceHost, + DialTimeoutSec: 10, + }, + }, + } + } + + opts := ts2021.ClientOpts{ + ServerURL: "https://" + ts2021Args.host, + DialPlan: func() *tailcfg.ControlDialPlan { + return dialPlan + }, + Logf: logf, + NetMon: netMon, + PrivKey: machinePrivate, + ServerPubKey: keys.PublicKey, + Dialer: tsdial.NewFromFuncForDebug(logf, dialFunc), + DNSCache: &dnscache.Resolver{}, + HealthTracker: &health.Tracker{}, } + + // TODO: ProtocolVersion: uint16(ts2021Args.version), const tries = 2 for i := range tries { - err := tryConnect(ctx, keys.PublicKey, noiseDialer) + err := tryConnect(ctx, keys.PublicKey, opts) if err != nil { log.Printf("error on attempt %d/%d: %v", i+1, tries, err) continue @@ -1005,53 +1110,37 @@ func runTS2021(ctx context.Context, args []string) error { return nil } -func tryConnect(ctx context.Context, controlPublic key.MachinePublic, noiseDialer *controlhttp.Dialer) error { - conn, err := noiseDialer.Dial(ctx) - log.Printf("controlhttp.Dial = %p, %v", conn, err) - if err != nil { - return err - } - log.Printf("did noise handshake") - - gotPeer := conn.Peer() - if gotPeer != controlPublic { - log.Printf("peer = %v, want %v", gotPeer, controlPublic) - return errors.New("key mismatch") - } +func tryConnect(ctx context.Context, controlPublic key.MachinePublic, opts ts2021.ClientOpts) error { - log.Printf("final underlying conn: %v / %v", conn.LocalAddr(), conn.RemoteAddr()) - - h2Transport, err := http2.ConfigureTransports(&http.Transport{ - IdleConnTimeout: time.Second, + ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + GotConn: func(ci httptrace.GotConnInfo) { + log.Printf("GotConn: %T", ci.Conn) + ncc, ok := ci.Conn.(*ts2021.Conn) + if !ok { + return + } + log.Printf("did noise handshake") + log.Printf("final underlying conn: %v / %v", ncc.LocalAddr(), ncc.RemoteAddr()) + gotPeer := ncc.Peer() + if gotPeer != controlPublic { + log.Fatalf("peer = %v, want %v", gotPeer, controlPublic) + } + }, }) - if err != nil { - return fmt.Errorf("http2.ConfigureTransports: %w", err) - } - - // Now, create a Noise conn over the existing conn. - nc, err := noiseconn.New(conn.Conn, h2Transport, 0, nil) - if err != nil { - return fmt.Errorf("noiseconn.New: %w", err) - } - defer nc.Close() - // Reserve a RoundTrip for the whoami request. - ok, _, err := nc.ReserveNewRequest(ctx) + nc, err := ts2021.NewClient(opts) if err != nil { - return fmt.Errorf("ReserveNewRequest: %w", err) - } - if !ok { - return errors.New("ReserveNewRequest failed") + return fmt.Errorf("NewNoiseClient: %w", err) } // Make a /whoami request to the server to verify that we can actually // communicate over the newly-established connection. - whoamiURL := "http://" + ts2021Args.host + "/machine/whoami" + whoamiURL := "https://" + ts2021Args.host + "/machine/whoami" req, err := http.NewRequestWithContext(ctx, "GET", whoamiURL, nil) if err != nil { return err } - resp, err := nc.RoundTrip(req) + resp, err := nc.Do(req) if err != nil { return fmt.Errorf("RoundTrip whoami request: %w", err) } @@ -1137,44 +1226,6 @@ func runSetExpire(ctx context.Context, args []string) error { return localClient.DebugSetExpireIn(ctx, setExpireArgs.in) } -var debugPortmapArgs struct { - duration time.Duration - gatewayAddr string - selfAddr string - ty string - logHTTP bool -} - -func debugPortmap(ctx context.Context, args []string) error { - opts := &tailscale.DebugPortmapOpts{ - Duration: debugPortmapArgs.duration, - Type: debugPortmapArgs.ty, - LogHTTP: debugPortmapArgs.logHTTP, - } - if (debugPortmapArgs.gatewayAddr != "") != (debugPortmapArgs.selfAddr != "") { - return fmt.Errorf("if one of --gateway-addr and --self-addr is provided, the other must be as well") - } - if debugPortmapArgs.gatewayAddr != "" { - var err error - opts.GatewayAddr, err = netip.ParseAddr(debugPortmapArgs.gatewayAddr) - if err != nil { - return fmt.Errorf("invalid --gateway-addr: %w", err) - } - opts.SelfAddr, err = netip.ParseAddr(debugPortmapArgs.selfAddr) - if err != nil { - return fmt.Errorf("invalid --self-addr: %w", err) - } - } - rc, err := localClient.DebugPortmap(ctx, opts) - if err != nil { - return err - } - defer rc.Close() - - _, err = io.Copy(os.Stdout, rc) - return err -} - func runPeerEndpointChanges(ctx context.Context, args []string) error { st, err := localClient.Status(ctx) if err != nil { @@ -1327,3 +1378,51 @@ func runDebugResolve(ctx context.Context, args []string) error { } return nil } + +func runPeerRelayServers(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unexpected arguments") + } + v, err := localClient.DebugResultJSON(ctx, "peer-relay-servers") + if err != nil { + return err + } + e := json.NewEncoder(os.Stdout) + e.SetIndent("", " ") + e.Encode(v) + return nil +} + +var testRiskArgs struct { + acceptedRisk string +} + +func runTestRisk(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unexpected arguments") + } + if err := presentRiskToUser("test-risk", "This is a test risky action.", testRiskArgs.acceptedRisk); err != nil { + return err + } + fmt.Println("did-test-risky-action") + return nil +} + +func runPrintStateDir(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unexpected arguments") + } + v, err := localClient.DebugResultJSON(ctx, "statedir") + if err != nil { + return err + } + statedir, ok := v.(string) + if ok && statedir != "" { + fmt.Println(statedir) + return nil + } else if ok && statedir == "" { + return errors.New("no statedir is set") + } else { + return fmt.Errorf("got unexpected response from debug API: %v", v) + } +} diff --git a/cmd/tailscale/cli/diag.go b/cmd/tailscale/cli/diag.go index ebf26985fe0bd..8a244ba8817bb 100644 --- a/cmd/tailscale/cli/diag.go +++ b/cmd/tailscale/cli/diag.go @@ -1,7 +1,7 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -//go:build linux || windows || darwin +//go:build (linux || windows || darwin) && !ts_omit_cliconndiag package cli @@ -16,11 +16,15 @@ import ( "tailscale.com/version/distro" ) -// fixTailscaledConnectError is called when the local tailscaled has +func init() { + hookFixTailscaledConnectError.Set(fixTailscaledConnectErrorImpl) +} + +// fixTailscaledConnectErrorImpl is called when the local tailscaled has // been determined unreachable due to the provided origErr value. It // returns either the same error or a better one to help the user // understand why tailscaled isn't running for their platform. -func fixTailscaledConnectError(origErr error) error { +func fixTailscaledConnectErrorImpl(origErr error) error { procs, err := ps.Processes() if err != nil { return fmt.Errorf("failed to connect to local Tailscaled process and failed to enumerate processes while looking for it") diff --git a/cmd/tailscale/cli/diag_other.go b/cmd/tailscale/cli/diag_other.go deleted file mode 100644 index ece10cc79a822..0000000000000 --- a/cmd/tailscale/cli/diag_other.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !linux && !windows && !darwin - -package cli - -import "fmt" - -// The github.com/mitchellh/go-ps package doesn't work on all platforms, -// so just don't diagnose connect failures. - -func fixTailscaledConnectError(origErr error) error { - return fmt.Errorf("failed to connect to local tailscaled process (is it running?); got: %w", origErr) -} diff --git a/cmd/tailscale/cli/dns-query.go b/cmd/tailscale/cli/dns-query.go index 11f64453732fa..2993441b3d2fc 100644 --- a/cmd/tailscale/cli/dns-query.go +++ b/cmd/tailscale/cli/dns-query.go @@ -1,97 +1,169 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli import ( "context" + "encoding/json" + "errors" "flag" "fmt" "net/netip" - "os" "strings" "text/tabwriter" "github.com/peterbourgon/ff/v3/ffcli" "golang.org/x/net/dns/dnsmessage" - "tailscale.com/types/dnstype" + "tailscale.com/cmd/tailscale/cli/jsonoutput" ) +var dnsQueryArgs struct { + json bool +} + var dnsQueryCmd = &ffcli.Command{ Name: "query", - ShortUsage: "tailscale dns query [a|aaaa|cname|mx|ns|opt|ptr|srv|txt]", + ShortUsage: "tailscale dns query [--json] [type]", Exec: runDNSQuery, ShortHelp: "Perform a DNS query", LongHelp: strings.TrimSpace(` The 'tailscale dns query' subcommand performs a DNS query for the specified name using the internal DNS forwarder (100.100.100.100). -By default, the DNS query will request an A record. Another DNS record type can -be specified as the second parameter. +By default, the DNS query will request an A record. Specify the record type as +a second argument after the name (e.g. AAAA, CNAME, MX, NS, PTR, SRV, TXT). The output also provides information about the resolver(s) used to resolve the query. `), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("query") + fs.BoolVar(&dnsQueryArgs.json, "json", false, "output in JSON format") + return fs + })(), } func runDNSQuery(ctx context.Context, args []string) error { - if len(args) < 1 { - return flag.ErrHelp + if len(args) == 0 { + return errors.New("missing required argument: name") + } + if len(args) > 1 { + var flags []string + for _, a := range args[1:] { + if strings.HasPrefix(a, "-") { + flags = append(flags, a) + } + } + if len(flags) > 0 { + return fmt.Errorf("unexpected flags after query name: %s; see 'tailscale dns query --help'", strings.Join(flags, ", ")) + } + if len(args) > 2 { + return fmt.Errorf("unexpected extra arguments: %s", strings.Join(args[2:], " ")) + } } name := args[0] queryType := "A" - if len(args) >= 2 { - queryType = args[1] + if len(args) > 1 { + queryType = strings.ToUpper(args[1]) } - fmt.Printf("DNS query for %q (%s) using internal resolver:\n", name, queryType) - fmt.Println() - bytes, resolvers, err := localClient.QueryDNS(ctx, name, queryType) + + rawBytes, resolvers, err := localClient.QueryDNS(ctx, name, queryType) if err != nil { - fmt.Printf("failed to query DNS: %v\n", err) - return nil + return fmt.Errorf("failed to query DNS: %w", err) } - if len(resolvers) == 1 { - fmt.Printf("Forwarding to resolver: %v\n", makeResolverString(*resolvers[0])) - } else { - fmt.Println("Multiple resolvers available:") - for _, r := range resolvers { - fmt.Printf(" - %v\n", makeResolverString(*r)) - } + data := &jsonoutput.DNSQueryResult{ + Name: name, + QueryType: queryType, + } + + for _, r := range resolvers { + data.Resolvers = append(data.Resolvers, makeDNSResolverInfo(r)) } - fmt.Println() + var p dnsmessage.Parser - header, err := p.Start(bytes) + header, err := p.Start(rawBytes) if err != nil { - fmt.Printf("failed to parse DNS response: %v\n", err) - return err + return fmt.Errorf("failed to parse DNS response: %w", err) } - fmt.Printf("Response code: %v\n", header.RCode.String()) - fmt.Println() + data.ResponseCode = header.RCode.String() + p.SkipAllQuestions() - if header.RCode != dnsmessage.RCodeSuccess { - fmt.Println("No answers were returned.") + + if header.RCode == dnsmessage.RCodeSuccess { + answers, err := p.AllAnswers() + if err != nil { + return fmt.Errorf("failed to parse DNS answers: %w", err) + } + data.Answers = make([]jsonoutput.DNSAnswer, 0, len(answers)) + for _, a := range answers { + data.Answers = append(data.Answers, jsonoutput.DNSAnswer{ + Name: a.Header.Name.String(), + TTL: a.Header.TTL, + Class: a.Header.Class.String(), + Type: a.Header.Type.String(), + Body: makeAnswerBody(a), + }) + } + } + + if dnsQueryArgs.json { + j, err := json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + printf("%s\n", j) return nil } - answers, err := p.AllAnswers() - if err != nil { - fmt.Printf("failed to parse DNS answers: %v\n", err) - return err + printf("%s", formatDNSQueryText(data)) + return nil +} + +func formatDNSQueryText(data *jsonoutput.DNSQueryResult) string { + var sb strings.Builder + + fmt.Fprintf(&sb, "DNS query for %q (%s) using internal resolver:\n", data.Name, data.QueryType) + fmt.Fprintf(&sb, "\n") + if len(data.Resolvers) == 1 { + fmt.Fprintf(&sb, "Forwarding to resolver: %v\n", formatResolverString(data.Resolvers[0])) + } else { + fmt.Fprintf(&sb, "Multiple resolvers available:\n") + for _, r := range data.Resolvers { + fmt.Fprintf(&sb, " - %v\n", formatResolverString(r)) + } } - if len(answers) == 0 { - fmt.Println(" (no answers found)") + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "Response code: %v\n", data.ResponseCode) + fmt.Fprintf(&sb, "\n") + + if data.Answers == nil { + fmt.Fprintf(&sb, "No answers were returned.\n") + return sb.String() } - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + if len(data.Answers) == 0 { + fmt.Fprintf(&sb, " (no answers found)\n") + } + + w := tabwriter.NewWriter(&sb, 0, 0, 2, ' ', 0) fmt.Fprintln(w, "Name\tTTL\tClass\tType\tBody") fmt.Fprintln(w, "----\t---\t-----\t----\t----") - for _, a := range answers { - fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\n", a.Header.Name.String(), a.Header.TTL, a.Header.Class.String(), a.Header.Type.String(), makeAnswerBody(a)) + for _, a := range data.Answers { + fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\n", a.Name, a.TTL, a.Class, a.Type, a.Body) } w.Flush() - fmt.Println() - return nil + fmt.Fprintf(&sb, "\n") + return sb.String() +} + +// formatResolverString formats a jsonoutput.DNSResolverInfo for human-readable text output. +func formatResolverString(r jsonoutput.DNSResolverInfo) string { + if len(r.BootstrapResolution) > 0 { + return fmt.Sprintf("%s (bootstrap: %v)", r.Addr, r.BootstrapResolution) + } + return r.Addr } // makeAnswerBody returns a string with the DNS answer body in a human-readable format. @@ -174,9 +246,3 @@ func makeTXTBody(txt dnsmessage.ResourceBody) string { } return "" } -func makeResolverString(r dnstype.Resolver) string { - if len(r.BootstrapResolution) > 0 { - return fmt.Sprintf("%s (bootstrap: %v)", r.Addr, r.BootstrapResolution) - } - return fmt.Sprintf("%s", r.Addr) -} diff --git a/cmd/tailscale/cli/dns-status.go b/cmd/tailscale/cli/dns-status.go index 8c18622ce45af..91a62f996cc54 100644 --- a/cmd/tailscale/cli/dns-status.go +++ b/cmd/tailscale/cli/dns-status.go @@ -1,10 +1,11 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli import ( "context" + "encoding/json" "flag" "fmt" "maps" @@ -12,13 +13,13 @@ import ( "strings" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/ipn" - "tailscale.com/types/netmap" + "tailscale.com/cmd/tailscale/cli/jsonoutput" + "tailscale.com/types/dnstype" ) var dnsStatusCmd = &ffcli.Command{ Name: "status", - ShortUsage: "tailscale dns status [--all]", + ShortUsage: "tailscale dns status [--all] [--json]", Exec: runDNSStatus, ShortHelp: "Print the current DNS status and configuration", LongHelp: strings.TrimSpace(` @@ -72,17 +73,30 @@ https://tailscale.com/kb/1054/dns. FlagSet: (func() *flag.FlagSet { fs := newFlagSet("status") fs.BoolVar(&dnsStatusArgs.all, "all", false, "outputs advanced debugging information") + fs.BoolVar(&dnsStatusArgs.json, "json", false, "output in JSON format") return fs })(), } // dnsStatusArgs are the arguments for the "dns status" subcommand. var dnsStatusArgs struct { - all bool + all bool + json bool +} + +// makeDNSResolverInfo converts a dnstype.Resolver to a jsonoutput.DNSResolverInfo. +func makeDNSResolverInfo(r *dnstype.Resolver) jsonoutput.DNSResolverInfo { + info := jsonoutput.DNSResolverInfo{Addr: r.Addr} + if r.BootstrapResolution != nil { + info.BootstrapResolution = make([]string, 0, len(r.BootstrapResolution)) + for _, a := range r.BootstrapResolution { + info.BootstrapResolution = append(info.BootstrapResolution, a.String()) + } + } + return info } func runDNSStatus(ctx context.Context, args []string) error { - all := dnsStatusArgs.all s, err := localClient.Status(ctx) if err != nil { return err @@ -92,181 +106,251 @@ func runDNSStatus(ctx context.Context, args []string) error { if err != nil { return err } - enabledStr := "disabled.\n\n(Run 'tailscale set --accept-dns=true' to start sending DNS queries to the Tailscale DNS resolver)" - if prefs.CorpDNS { - enabledStr = "enabled.\n\nTailscale is configured to handle DNS queries on this device.\nRun 'tailscale set --accept-dns=false' to revert to your system default DNS resolver." + + data := &jsonoutput.DNSStatusResult{ + TailscaleDNS: prefs.CorpDNS, } - fmt.Print("\n") - fmt.Println("=== 'Use Tailscale DNS' status ===") - fmt.Print("\n") - fmt.Printf("Tailscale DNS: %s\n", enabledStr) - fmt.Print("\n") - fmt.Println("=== MagicDNS configuration ===") - fmt.Print("\n") - fmt.Println("This is the DNS configuration provided by the coordination server to this device.") - fmt.Print("\n") - if s.CurrentTailnet == nil { - fmt.Println("No tailnet information available; make sure you're logged in to a tailnet.") + + if s.CurrentTailnet != nil { + data.CurrentTailnet = &jsonoutput.DNSTailnetInfo{ + MagicDNSEnabled: s.CurrentTailnet.MagicDNSEnabled, + MagicDNSSuffix: s.CurrentTailnet.MagicDNSSuffix, + SelfDNSName: s.Self.DNSName, + } + + dnsConfig, err := localClient.DNSConfig(ctx) + if err != nil { + return fmt.Errorf("failed to fetch DNS config: %w", err) + } + + for _, r := range dnsConfig.Resolvers { + data.Resolvers = append(data.Resolvers, makeDNSResolverInfo(r)) + } + + data.SplitDNSRoutes = make(map[string][]jsonoutput.DNSResolverInfo) + for k, v := range dnsConfig.Routes { + for _, r := range v { + data.SplitDNSRoutes[k] = append(data.SplitDNSRoutes[k], makeDNSResolverInfo(r)) + } + } + + for _, r := range dnsConfig.FallbackResolvers { + data.FallbackResolvers = append(data.FallbackResolvers, makeDNSResolverInfo(r)) + } + + domains := slices.Clone(dnsConfig.Domains) + slices.Sort(domains) + data.SearchDomains = domains + + for _, a := range dnsConfig.Nameservers { + data.Nameservers = append(data.Nameservers, a.String()) + } + + data.CertDomains = dnsConfig.CertDomains + + for _, er := range dnsConfig.ExtraRecords { + data.ExtraRecords = append(data.ExtraRecords, jsonoutput.DNSExtraRecord{ + Name: er.Name, + Type: er.Type, + Value: er.Value, + }) + } + + data.ExitNodeFilteredSet = dnsConfig.ExitNodeFilteredSet + + osCfg, err := localClient.GetDNSOSConfig(ctx) + if err != nil { + if strings.Contains(err.Error(), "not supported") { + data.SystemDNSError = "not supported on this platform" + } else { + data.SystemDNSError = err.Error() + } + } else if osCfg != nil { + data.SystemDNS = &jsonoutput.DNSSystemConfig{ + Nameservers: osCfg.Nameservers, + SearchDomains: osCfg.SearchDomains, + MatchDomains: osCfg.MatchDomains, + } + } + } + + if dnsStatusArgs.json { + j, err := json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + printf("%s\n", j) return nil - } else if s.CurrentTailnet.MagicDNSEnabled { - fmt.Printf("MagicDNS: enabled tailnet-wide (suffix = %s)", s.CurrentTailnet.MagicDNSSuffix) - fmt.Print("\n\n") - fmt.Printf("Other devices in your tailnet can reach this device at %s\n", s.Self.DNSName) + } + printf("%s", formatDNSStatusText(data, dnsStatusArgs.all)) + return nil +} + +func formatDNSStatusText(data *jsonoutput.DNSStatusResult, all bool) string { + var sb strings.Builder + + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "=== 'Use Tailscale DNS' status ===\n") + fmt.Fprintf(&sb, "\n") + if data.TailscaleDNS { + fmt.Fprintf(&sb, "Tailscale DNS: enabled.\n\nTailscale is configured to handle DNS queries on this device.\nRun 'tailscale set --accept-dns=false' to revert to your system default DNS resolver.\n") } else { - fmt.Printf("MagicDNS: disabled tailnet-wide.\n") + fmt.Fprintf(&sb, "Tailscale DNS: disabled.\n\n(Run 'tailscale set --accept-dns=true' to start sending DNS queries to the Tailscale DNS resolver)\n") + } + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "=== MagicDNS configuration ===\n") + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "This is the DNS configuration provided by the coordination server to this device.\n") + fmt.Fprintf(&sb, "\n") + if data.CurrentTailnet == nil { + fmt.Fprintf(&sb, "No tailnet information available; make sure you're logged in to a tailnet.\n") + return sb.String() } - fmt.Print("\n") - netMap, err := fetchNetMap() - if err != nil { - fmt.Printf("Failed to fetch network map: %v\n", err) - return err + if data.CurrentTailnet.MagicDNSEnabled { + fmt.Fprintf(&sb, "MagicDNS: enabled tailnet-wide (suffix = %s)", data.CurrentTailnet.MagicDNSSuffix) + fmt.Fprintf(&sb, "\n\n") + fmt.Fprintf(&sb, "Other devices in your tailnet can reach this device at %s\n", data.CurrentTailnet.SelfDNSName) + } else { + fmt.Fprintf(&sb, "MagicDNS: disabled tailnet-wide.\n") } - dnsConfig := netMap.DNS - fmt.Println("Resolvers (in preference order):") - if len(dnsConfig.Resolvers) == 0 { - fmt.Println(" (no resolvers configured, system default will be used: see 'System DNS configuration' below)") + fmt.Fprintf(&sb, "\n") + + fmt.Fprintf(&sb, "Resolvers (in preference order):\n") + if len(data.Resolvers) == 0 { + fmt.Fprintf(&sb, " (no resolvers configured, system default will be used: see 'System DNS configuration' below)\n") } - for _, r := range dnsConfig.Resolvers { - fmt.Printf(" - %v", r.Addr) + for _, r := range data.Resolvers { + fmt.Fprintf(&sb, " - %v", r.Addr) if r.BootstrapResolution != nil { - fmt.Printf(" (bootstrap: %v)", r.BootstrapResolution) + fmt.Fprintf(&sb, " (bootstrap: %v)", r.BootstrapResolution) } - fmt.Print("\n") + fmt.Fprintf(&sb, "\n") } - fmt.Print("\n") - fmt.Println("Split DNS Routes:") - if len(dnsConfig.Routes) == 0 { - fmt.Println(" (no routes configured: split DNS disabled)") + fmt.Fprintf(&sb, "\n") + + fmt.Fprintf(&sb, "Split DNS Routes:\n") + if len(data.SplitDNSRoutes) == 0 { + fmt.Fprintf(&sb, " (no routes configured: split DNS disabled)\n") } - for _, k := range slices.Sorted(maps.Keys(dnsConfig.Routes)) { - v := dnsConfig.Routes[k] - for _, r := range v { - fmt.Printf(" - %-30s -> %v", k, r.Addr) + for _, k := range slices.Sorted(maps.Keys(data.SplitDNSRoutes)) { + for _, r := range data.SplitDNSRoutes[k] { + fmt.Fprintf(&sb, " - %-30s -> %v", k, r.Addr) if r.BootstrapResolution != nil { - fmt.Printf(" (bootstrap: %v)", r.BootstrapResolution) + fmt.Fprintf(&sb, " (bootstrap: %v)", r.BootstrapResolution) } - fmt.Print("\n") + fmt.Fprintf(&sb, "\n") } } - fmt.Print("\n") + fmt.Fprintf(&sb, "\n") + if all { - fmt.Println("Fallback Resolvers:") - if len(dnsConfig.FallbackResolvers) == 0 { - fmt.Println(" (no fallback resolvers configured)") + fmt.Fprintf(&sb, "Fallback Resolvers:\n") + if len(data.FallbackResolvers) == 0 { + fmt.Fprintf(&sb, " (no fallback resolvers configured)\n") } - for i, r := range dnsConfig.FallbackResolvers { - fmt.Printf(" %d: %v\n", i, r) + for i, r := range data.FallbackResolvers { + fmt.Fprintf(&sb, " %d: %v", i, r.Addr) + if r.BootstrapResolution != nil { + fmt.Fprintf(&sb, " (bootstrap: %v)", r.BootstrapResolution) + } + fmt.Fprintf(&sb, "\n") } - fmt.Print("\n") + fmt.Fprintf(&sb, "\n") } - fmt.Println("Search Domains:") - if len(dnsConfig.Domains) == 0 { - fmt.Println(" (no search domains configured)") + + fmt.Fprintf(&sb, "Search Domains:\n") + if len(data.SearchDomains) == 0 { + fmt.Fprintf(&sb, " (no search domains configured)\n") } - domains := dnsConfig.Domains - slices.Sort(domains) - for _, r := range domains { - fmt.Printf(" - %v\n", r) + for _, r := range data.SearchDomains { + fmt.Fprintf(&sb, " - %v\n", r) } - fmt.Print("\n") + fmt.Fprintf(&sb, "\n") + if all { - fmt.Println("Nameservers IP Addresses:") - if len(dnsConfig.Nameservers) == 0 { - fmt.Println(" (none were provided)") + fmt.Fprintf(&sb, "Nameservers IP Addresses:\n") + if len(data.Nameservers) == 0 { + fmt.Fprintf(&sb, " (none were provided)\n") } - for _, r := range dnsConfig.Nameservers { - fmt.Printf(" - %v\n", r) + for _, r := range data.Nameservers { + fmt.Fprintf(&sb, " - %v\n", r) } - fmt.Print("\n") - fmt.Println("Certificate Domains:") - if len(dnsConfig.CertDomains) == 0 { - fmt.Println(" (no certificate domains are configured)") + fmt.Fprintf(&sb, "\n") + + fmt.Fprintf(&sb, "Certificate Domains:\n") + if len(data.CertDomains) == 0 { + fmt.Fprintf(&sb, " (no certificate domains are configured)\n") } - for _, r := range dnsConfig.CertDomains { - fmt.Printf(" - %v\n", r) + for _, r := range data.CertDomains { + fmt.Fprintf(&sb, " - %v\n", r) } - fmt.Print("\n") - fmt.Println("Additional DNS Records:") - if len(dnsConfig.ExtraRecords) == 0 { - fmt.Println(" (no extra records are configured)") + fmt.Fprintf(&sb, "\n") + + fmt.Fprintf(&sb, "Additional DNS Records:\n") + if len(data.ExtraRecords) == 0 { + fmt.Fprintf(&sb, " (no extra records are configured)\n") } - for _, er := range dnsConfig.ExtraRecords { + for _, er := range data.ExtraRecords { if er.Type == "" { - fmt.Printf(" - %-50s -> %v\n", er.Name, er.Value) + fmt.Fprintf(&sb, " - %-50s -> %v\n", er.Name, er.Value) } else { - fmt.Printf(" - [%s] %-50s -> %v\n", er.Type, er.Name, er.Value) + fmt.Fprintf(&sb, " - [%s] %-50s -> %v\n", er.Type, er.Name, er.Value) } } - fmt.Print("\n") - fmt.Println("Filtered suffixes when forwarding DNS queries as an exit node:") - if len(dnsConfig.ExitNodeFilteredSet) == 0 { - fmt.Println(" (no suffixes are filtered)") + fmt.Fprintf(&sb, "\n") + + fmt.Fprintf(&sb, "Filtered suffixes when forwarding DNS queries as an exit node:\n") + if len(data.ExitNodeFilteredSet) == 0 { + fmt.Fprintf(&sb, " (no suffixes are filtered)\n") } - for _, s := range dnsConfig.ExitNodeFilteredSet { - fmt.Printf(" - %s\n", s) + for _, s := range data.ExitNodeFilteredSet { + fmt.Fprintf(&sb, " - %s\n", s) } - fmt.Print("\n") + fmt.Fprintf(&sb, "\n") } - fmt.Println("=== System DNS configuration ===") - fmt.Print("\n") - fmt.Println("This is the DNS configuration that Tailscale believes your operating system is using.\nTailscale may use this configuration if 'Override Local DNS' is disabled in the admin console,\nor if no resolvers are provided by the coordination server.") - fmt.Print("\n") - osCfg, err := localClient.GetDNSOSConfig(ctx) - if err != nil { - if strings.Contains(err.Error(), "not supported") { - // avoids showing the HTTP error code which would be odd here - fmt.Println(" (reading the system DNS configuration is not supported on this platform)") + fmt.Fprintf(&sb, "=== System DNS configuration ===\n") + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "This is the DNS configuration that Tailscale believes your operating system is using.\nTailscale may use this configuration if 'Override Local DNS' is disabled in the admin console,\nor if no resolvers are provided by the coordination server.\n") + fmt.Fprintf(&sb, "\n") + + if data.SystemDNSError != "" { + if strings.Contains(data.SystemDNSError, "not supported") { + fmt.Fprintf(&sb, " (reading the system DNS configuration is not supported on this platform)\n") } else { - fmt.Printf(" (failed to read system DNS configuration: %v)\n", err) + fmt.Fprintf(&sb, " (failed to read system DNS configuration: %s)\n", data.SystemDNSError) } - } else if osCfg == nil { - fmt.Println(" (no OS DNS configuration available)") + } else if data.SystemDNS == nil { + fmt.Fprintf(&sb, " (no OS DNS configuration available)\n") } else { - fmt.Println("Nameservers:") - if len(osCfg.Nameservers) == 0 { - fmt.Println(" (no nameservers found, DNS queries might fail\nunless the coordination server is providing a nameserver)") + fmt.Fprintf(&sb, "Nameservers:\n") + if len(data.SystemDNS.Nameservers) == 0 { + fmt.Fprintf(&sb, " (no nameservers found, DNS queries might fail\nunless the coordination server is providing a nameserver)\n") } - for _, ns := range osCfg.Nameservers { - fmt.Printf(" - %v\n", ns) + for _, ns := range data.SystemDNS.Nameservers { + fmt.Fprintf(&sb, " - %v\n", ns) } - fmt.Print("\n") - fmt.Println("Search domains:") - if len(osCfg.SearchDomains) == 0 { - fmt.Println(" (no search domains found)") + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "Search domains:\n") + if len(data.SystemDNS.SearchDomains) == 0 { + fmt.Fprintf(&sb, " (no search domains found)\n") } - for _, sd := range osCfg.SearchDomains { - fmt.Printf(" - %v\n", sd) + for _, sd := range data.SystemDNS.SearchDomains { + fmt.Fprintf(&sb, " - %v\n", sd) } if all { - fmt.Print("\n") - fmt.Println("Match domains:") - if len(osCfg.MatchDomains) == 0 { - fmt.Println(" (no match domains found)") + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "Match domains:\n") + if len(data.SystemDNS.MatchDomains) == 0 { + fmt.Fprintf(&sb, " (no match domains found)\n") } - for _, md := range osCfg.MatchDomains { - fmt.Printf(" - %v\n", md) + for _, md := range data.SystemDNS.MatchDomains { + fmt.Fprintf(&sb, " - %v\n", md) } } } - fmt.Print("\n") - fmt.Println("[this is a preliminary version of this command; the output format may change in the future]") - return nil -} - -func fetchNetMap() (netMap *netmap.NetworkMap, err error) { - w, err := localClient.WatchIPNBus(context.Background(), ipn.NotifyInitialNetMap) - if err != nil { - return nil, err - } - defer w.Close() - notify, err := w.Next() - if err != nil { - return nil, err - } - if notify.NetMap == nil { - return nil, fmt.Errorf("no network map yet available, please try again later") - } - return notify.NetMap, nil + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, "[this is a preliminary version of this command; the output format may change in the future]\n") + return sb.String() } diff --git a/cmd/tailscale/cli/dns.go b/cmd/tailscale/cli/dns.go index 086abefd6b2bf..d8db5d466d6b2 100644 --- a/cmd/tailscale/cli/dns.go +++ b/cmd/tailscale/cli/dns.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/dns_test.go b/cmd/tailscale/cli/dns_test.go new file mode 100644 index 0000000000000..cc01a52702fac --- /dev/null +++ b/cmd/tailscale/cli/dns_test.go @@ -0,0 +1,65 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "strings" + "testing" +) + +func TestRunDNSQueryArgs(t *testing.T) { + tests := []struct { + name string + args []string + wantErr string + }{ + { + name: "no_args", + args: []string{}, + wantErr: "missing required argument: name", + }, + { + name: "flag_after_name", + args: []string{"example.com", "--json"}, + wantErr: "unexpected flags after query name: --json", + }, + { + name: "flag_after_name_and_type", + args: []string{"example.com", "AAAA", "--json"}, + wantErr: "unexpected flags after query name: --json", + }, + { + name: "extra_args_after_type", + args: []string{"example.com", "AAAA", "extra"}, + wantErr: "unexpected extra arguments: extra", + }, + { + name: "multiple_extra_args", + args: []string{"example.com", "AAAA", "extra1", "extra2"}, + wantErr: "unexpected extra arguments: extra1 extra2", + }, + { + name: "non_flag_then_flag", + args: []string{"example.com", "AAAA", "foo", "--json"}, + wantErr: "unexpected flags after query name: --json", + }, + { + name: "multiple_misplaced_flags", + args: []string{"example.com", "--json", "--verbose"}, + wantErr: "unexpected flags after query name: --json, --verbose", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := runDNSQuery(context.Background(), tt.args) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error = %q, want it to contain %q", err.Error(), tt.wantErr) + } + }) + } +} diff --git a/cmd/tailscale/cli/down.go b/cmd/tailscale/cli/down.go index 224198a98deb5..6fecbd76cec12 100644 --- a/cmd/tailscale/cli/down.go +++ b/cmd/tailscale/cli/down.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/drive.go b/cmd/tailscale/cli/drive.go index 929852b4c5a32..280ff3172fb92 100644 --- a/cmd/tailscale/cli/drive.go +++ b/cmd/tailscale/cli/drive.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_drive && !ts_mac_gui + package cli import ( @@ -20,43 +22,49 @@ const ( driveListUsage = "tailscale drive list" ) -var driveCmd = &ffcli.Command{ - Name: "drive", - ShortHelp: "Share a directory with your tailnet", - ShortUsage: strings.Join([]string{ - driveShareUsage, - driveRenameUsage, - driveUnshareUsage, - driveListUsage, - }, "\n"), - LongHelp: buildShareLongHelp(), - UsageFunc: usageFuncNoDefaultValues, - Subcommands: []*ffcli.Command{ - { - Name: "share", - ShortUsage: driveShareUsage, - Exec: runDriveShare, - ShortHelp: "[ALPHA] Create or modify a share", - }, - { - Name: "rename", - ShortUsage: driveRenameUsage, - ShortHelp: "[ALPHA] Rename a share", - Exec: runDriveRename, - }, - { - Name: "unshare", - ShortUsage: driveUnshareUsage, - ShortHelp: "[ALPHA] Remove a share", - Exec: runDriveUnshare, - }, - { - Name: "list", - ShortUsage: driveListUsage, - ShortHelp: "[ALPHA] List current shares", - Exec: runDriveList, +func init() { + maybeDriveCmd = driveCmd +} + +func driveCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "drive", + ShortHelp: "Share a directory with your tailnet", + ShortUsage: strings.Join([]string{ + driveShareUsage, + driveRenameUsage, + driveUnshareUsage, + driveListUsage, + }, "\n"), + LongHelp: buildShareLongHelp(), + UsageFunc: usageFuncNoDefaultValues, + Subcommands: []*ffcli.Command{ + { + Name: "share", + ShortUsage: driveShareUsage, + Exec: runDriveShare, + ShortHelp: "[ALPHA] Create or modify a share", + }, + { + Name: "rename", + ShortUsage: driveRenameUsage, + ShortHelp: "[ALPHA] Rename a share", + Exec: runDriveRename, + }, + { + Name: "unshare", + ShortUsage: driveUnshareUsage, + ShortHelp: "[ALPHA] Remove a share", + Exec: runDriveUnshare, + }, + { + Name: "list", + ShortUsage: driveListUsage, + ShortHelp: "[ALPHA] List current shares", + Exec: runDriveList, + }, }, - }, + } } // runDriveShare is the entry point for the "tailscale drive share" command. diff --git a/cmd/tailscale/cli/drive_macgui.go b/cmd/tailscale/cli/drive_macgui.go new file mode 100644 index 0000000000000..8a4594f86c4c9 --- /dev/null +++ b/cmd/tailscale/cli/drive_macgui.go @@ -0,0 +1,33 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_drive && ts_mac_gui + +package cli + +import ( + "context" + "errors" + + "github.com/peterbourgon/ff/v3/ffcli" +) + +func init() { + maybeDriveCmd = driveCmdStub +} + +func driveCmdStub() *ffcli.Command { + return &ffcli.Command{ + Name: "drive", + ShortHelp: "Share a directory with your tailnet", + ShortUsage: "tailscale drive [...any]", + LongHelp: hidden + "Taildrive allows you to share directories with other machines on your tailnet.", + Exec: func(_ context.Context, args []string) error { + return errors.New( + "Taildrive CLI commands are not supported when using the macOS GUI app. " + + "Please use the Tailscale menu bar icon to configure Taildrive in Settings.\n\n" + + "See https://tailscale.com/docs/features/taildrive", + ) + }, + } +} diff --git a/cmd/tailscale/cli/drive_macgui_test.go b/cmd/tailscale/cli/drive_macgui_test.go new file mode 100644 index 0000000000000..11f72b13a578a --- /dev/null +++ b/cmd/tailscale/cli/drive_macgui_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_drive && ts_mac_gui + +package cli + +import ( + "bytes" + "context" + "flag" + "strings" + "testing" + + "github.com/peterbourgon/ff/v3/ffcli" +) + +// In macOS GUI builds, the `drive` command should not appear in +// the help text generated by ffcli. +func TestDriveCommandHiddenInHelpText(t *testing.T) { + root := newRootCmd() + + var buf bytes.Buffer + root.FlagSet = flag.NewFlagSet("tailscale", flag.ContinueOnError) + root.FlagSet.SetOutput(&buf) + + ffcli.DefaultUsageFunc(root) + + output := buf.String() + + if strings.Contains(output, "drive") { + t.Errorf("found hidden command 'drive' in help output:\n%q", output) + } +} + +// Running the drive command always prints an error pointing you to +// the GUI app, regardless of input. +func TestDriveCommandPrintsError(t *testing.T) { + commands := [][]string{ + {"drive"}, + {"drive", "share", "myfile.txt", "/path/to/myfile.txt"}, + {"drive", "rename", "oldname.txt", "newname.txt"}, + {"drive", "unshare", "myfile.txt"}, + {"drive", "list"}, + } + + for _, args := range commands { + root := newRootCmd() + + if err := root.Parse(args); err != nil { + t.Errorf("unable to parse args %q, got err %v", args, err) + continue + } + + t.Logf("running `tailscale drive %q`", strings.Join(args, " ")) + err := root.Run(context.Background()) + if err == nil { + t.Error("expected error, but got nil", args) + } + + expectedText := "Taildrive CLI commands are not supported when using the macOS GUI app." + if !strings.Contains(err.Error(), expectedText) { + t.Errorf("error was not expected: want %q, got %q", expectedText, err.Error()) + } + } +} diff --git a/cmd/tailscale/cli/exitnode.go b/cmd/tailscale/cli/exitnode.go index ad7a8ccee5b42..7ba4859d79463 100644 --- a/cmd/tailscale/cli/exitnode.go +++ b/cmd/tailscale/cli/exitnode.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -131,14 +131,14 @@ func runExitNodeList(ctx context.Context, args []string) error { for _, country := range filteredPeers.Countries { for _, city := range country.Cities { for _, peer := range city.Peers { - fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", peer.TailscaleIPs[0], strings.Trim(peer.DNSName, "."), country.Name, city.Name, peerStatus(peer)) + fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", peer.TailscaleIPs[0], strings.Trim(peer.DNSName, "."), cmp.Or(country.Name, "-"), cmp.Or(city.Name, "-"), peerStatus(peer)) } } } fmt.Fprintln(w) fmt.Fprintln(w) fmt.Fprintln(w, "# To view the complete list of exit nodes for a country, use `tailscale exit-node list --filter=` followed by the country name.") - fmt.Fprintln(w, "# To use an exit node, use `tailscale set --exit-node=` followed by the hostname or IP.") + fmt.Fprintln(w, "# To use an exit node, use `tailscale set --exit-node=` followed by the IP or hostname.") if hasAnyExitNodeSuggestions(peers) { fmt.Fprintln(w, "# To have Tailscale suggest an exit node, use `tailscale exit-node suggest`.") } @@ -173,11 +173,13 @@ func hasAnyExitNodeSuggestions(peers []*ipnstate.PeerStatus) bool { // a peer. If there is no notable state, a - is returned. func peerStatus(peer *ipnstate.PeerStatus) string { if !peer.Active { + lastseen := lastSeenFmt(peer.LastSeen) + if peer.ExitNode { - return "selected but offline" + return "selected but offline" + lastseen } if !peer.Online { - return "offline" + return "offline" + lastseen } } @@ -202,23 +204,16 @@ type filteredCity struct { Peers []*ipnstate.PeerStatus } -const noLocationData = "-" - -var noLocation = &tailcfg.Location{ - Country: noLocationData, - CountryCode: noLocationData, - City: noLocationData, - CityCode: noLocationData, -} - // filterFormatAndSortExitNodes filters and sorts exit nodes into // alphabetical order, by country, city and then by priority if // present. +// // If an exit node has location data, and the country has more than // one city, an `Any` city is added to the country that contains the // highest priority exit node within that country. +// // For exit nodes without location data, their country fields are -// defined as '-' to indicate that the data is not available. +// defined as the empty string to indicate that the data is not available. func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) filteredExitNodes { // first get peers into some fixed order, as code below doesn't break ties // and our input comes from a random range-over-map. @@ -229,7 +224,10 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) countries := make(map[string]*filteredCountry) cities := make(map[string]*filteredCity) for _, ps := range peers { - loc := cmp.Or(ps.Location, noLocation) + loc := ps.Location + if loc == nil { + loc = &tailcfg.Location{} + } if filterBy != "" && !strings.EqualFold(loc.Country, filterBy) { continue @@ -259,7 +257,7 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) } for _, country := range filteredExitNodes.Countries { - if country.Name == noLocationData { + if country.Name == "" { // Countries without location data should not // be filtered further. continue diff --git a/cmd/tailscale/cli/exitnode_test.go b/cmd/tailscale/cli/exitnode_test.go index 9d569a45a4615..d7906b929ff57 100644 --- a/cmd/tailscale/cli/exitnode_test.go +++ b/cmd/tailscale/cli/exitnode_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -14,7 +14,7 @@ import ( ) func TestFilterFormatAndSortExitNodes(t *testing.T) { - t.Run("without filter", func(t *testing.T) { + t.Run("without-filter", func(t *testing.T) { ps := []*ipnstate.PeerStatus{ { HostName: "everest-1", @@ -74,10 +74,10 @@ func TestFilterFormatAndSortExitNodes(t *testing.T) { want := filteredExitNodes{ Countries: []*filteredCountry{ { - Name: noLocationData, + Name: "", Cities: []*filteredCity{ { - Name: noLocationData, + Name: "", Peers: []*ipnstate.PeerStatus{ ps[5], }, @@ -139,7 +139,7 @@ func TestFilterFormatAndSortExitNodes(t *testing.T) { } }) - t.Run("with country filter", func(t *testing.T) { + t.Run("with-country-filter", func(t *testing.T) { ps := []*ipnstate.PeerStatus{ { HostName: "baker-1", @@ -273,14 +273,20 @@ func TestSortByCountryName(t *testing.T) { Name: "Zimbabwe", }, { - Name: noLocationData, + Name: "", }, } sortByCountryName(fc) - if fc[0].Name != noLocationData { - t.Fatalf("sortByCountryName did not order countries by alphabetical order, got %v, want %v", fc[0].Name, noLocationData) + want := []string{"", "Albania", "Sweden", "Zimbabwe"} + var got []string + for _, c := range fc { + got = append(got, c.Name) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("sortByCountryName did not order countries by alphabetical order (-want +got):\n%s", diff) } } @@ -296,13 +302,19 @@ func TestSortByCityName(t *testing.T) { Name: "Squamish", }, { - Name: noLocationData, + Name: "", }, } sortByCityName(fc) - if fc[0].Name != noLocationData { - t.Fatalf("sortByCityName did not order cities by alphabetical order, got %v, want %v", fc[0].Name, noLocationData) + want := []string{"", "Goteborg", "Kingston", "Squamish"} + var got []string + for _, c := range fc { + got = append(got, c.Name) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("sortByCityName did not order countries by alphabetical order (-want +got):\n%s", diff) } } diff --git a/cmd/tailscale/cli/ffcomplete/complete.go b/cmd/tailscale/cli/ffcomplete/complete.go index fbd5b9d62823d..7d280f691a407 100644 --- a/cmd/tailscale/cli/ffcomplete/complete.go +++ b/cmd/tailscale/cli/ffcomplete/complete.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 && !ts_omit_completion diff --git a/cmd/tailscale/cli/ffcomplete/complete_omit.go b/cmd/tailscale/cli/ffcomplete/complete_omit.go index bafc059e7b71d..06efa63fcd3a7 100644 --- a/cmd/tailscale/cli/ffcomplete/complete_omit.go +++ b/cmd/tailscale/cli/ffcomplete/complete_omit.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 && ts_omit_completion diff --git a/cmd/tailscale/cli/ffcomplete/ffcomplete.go b/cmd/tailscale/cli/ffcomplete/ffcomplete.go index 4b8207ec60a0c..e6af2515ff26f 100644 --- a/cmd/tailscale/cli/ffcomplete/ffcomplete.go +++ b/cmd/tailscale/cli/ffcomplete/ffcomplete.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package ffcomplete diff --git a/cmd/tailscale/cli/ffcomplete/internal/complete.go b/cmd/tailscale/cli/ffcomplete/internal/complete.go index b6c39dc837215..911972518d331 100644 --- a/cmd/tailscale/cli/ffcomplete/internal/complete.go +++ b/cmd/tailscale/cli/ffcomplete/internal/complete.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package internal contains internal code for the ffcomplete package. diff --git a/cmd/tailscale/cli/ffcomplete/internal/complete_test.go b/cmd/tailscale/cli/ffcomplete/internal/complete_test.go index 7e36b1bcd1437..2bba72283b044 100644 --- a/cmd/tailscale/cli/ffcomplete/internal/complete_test.go +++ b/cmd/tailscale/cli/ffcomplete/internal/complete_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package internal_test @@ -196,7 +196,6 @@ func TestComplete(t *testing.T) { // Run the tests. for _, test := range tests { - test := test name := strings.Join(test.args, "âŖ") if test.showFlags { name += "+flags" diff --git a/cmd/tailscale/cli/ffcomplete/scripts.go b/cmd/tailscale/cli/ffcomplete/scripts.go index 8218683afa349..bccebed7feec1 100644 --- a/cmd/tailscale/cli/ffcomplete/scripts.go +++ b/cmd/tailscale/cli/ffcomplete/scripts.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 && !ts_omit_completion && !ts_omit_completion_scripts diff --git a/cmd/tailscale/cli/ffcomplete/scripts_omit.go b/cmd/tailscale/cli/ffcomplete/scripts_omit.go index b5d520c3fe1d9..4c082d9d1ed06 100644 --- a/cmd/tailscale/cli/ffcomplete/scripts_omit.go +++ b/cmd/tailscale/cli/ffcomplete/scripts_omit.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 && !ts_omit_completion && ts_omit_completion_scripts diff --git a/cmd/tailscale/cli/file.go b/cmd/tailscale/cli/file.go index 6f3aa40b5a806..946e5f2cf951e 100644 --- a/cmd/tailscale/cli/file.go +++ b/cmd/tailscale/cli/file.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !ts_omit_taildrop @@ -19,7 +19,9 @@ import ( "os" "path" "path/filepath" + "slices" "strings" + "sync" "sync/atomic" "time" "unicode/utf8" @@ -30,9 +32,9 @@ import ( "tailscale.com/client/tailscale/apitype" "tailscale.com/cmd/tailscale/cli/ffcomplete" "tailscale.com/envknob" + "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/net/tsaddr" - "tailscale.com/syncs" "tailscale.com/tailcfg" tsrate "tailscale.com/tstime/rate" "tailscale.com/util/quarantine" @@ -77,14 +79,16 @@ var fileCpCmd = &ffcli.Command{ fs.StringVar(&cpArgs.name, "name", "", "alternate filename to use, especially useful when is \"-\" (stdin)") fs.BoolVar(&cpArgs.verbose, "verbose", false, "verbose output") fs.BoolVar(&cpArgs.targets, "targets", false, "list possible file cp targets") + fs.DurationVar(&cpArgs.updateInterval, "update-interval", 250*time.Millisecond, "how often to repaint the progress line; zero or negative disables progress display entirely") return fs })(), } var cpArgs struct { - name string - verbose bool - targets bool + name string + verbose bool + targets bool + updateInterval time.Duration } func runCp(ctx context.Context, args []string) error { @@ -118,22 +122,61 @@ func runCp(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("can't send to %s: %v", target, err) } - if isOffline { - fmt.Fprintf(Stderr, "# warning: %s is offline\n", target) - } if len(files) > 1 { if cpArgs.name != "" { return errors.New("can't use --name= with multiple files") } - for _, fileArg := range files { - if fileArg == "-" { - return errors.New("can't use '-' as STDIN file when providing filename arguments") + if slices.Contains(files, "-") { + return errors.New("can't use '-' as STDIN file when providing filename arguments") + } + } + + // outFiles tracks per-name push state, populated by a goroutine subscribed + // to the IPN bus. tailscaled's OutgoingFile.Sent is the bytes-pulled-toward- + // peerAPI signal; it stays at 0 until the peerAPI request body is actually + // being read, which is what we want both for the progress display and for + // disarming the offline warning. The CLI's local-side bytes counter would + // say "100% sent" the moment net/http buffers a small body into the local + // unix-socket conn to tailscaled, well before the peer has heard a thing. + type pushState struct { + sent atomic.Int64 + warnTimer *time.Timer // disarmed on first byte sent to peerAPI; nil after + } + var ( + outMu sync.Mutex + outFiles = map[string]*pushState{} // keyed by file name + ) + + busCtx, cancelBus := context.WithCancel(ctx) + defer cancelBus() + go watchOutgoingFiles(busCtx, stableID, func(name string, sent int64) { + outMu.Lock() + ps := outFiles[name] + outMu.Unlock() + if ps == nil { + return + } + // Only ever advance ps.sent forward. Bus updates can arrive late + // (after the success path below has already written contentLength + // to ps.sent for an instant final-100% paint), so we'd otherwise + // regress the count and the progress printer would compute a + // negative delta on its next tick. + for { + old := ps.sent.Load() + if sent <= old { + return + } + if ps.sent.CompareAndSwap(old, sent) { + if old == 0 && ps.warnTimer != nil { + ps.warnTimer.Stop() + } + return } } - } + }) - for _, fileArg := range files { + for i, fileArg := range files { var fileContents *countingReader var name = cpArgs.name var contentLength int64 = -1 @@ -176,16 +219,57 @@ func runCp(ctx context.Context, args []string) error { log.Printf("sending %q to %v/%v/%v ...", name, target, ip, stableID) } - var group syncs.WaitGroup + // Register this file with the watcher and, for the first file only, + // arm a timer that warns the user if no bytes have flowed to peerAPI + // after a few seconds. The watcher disarms it on first byte; PushFile + // returning also disarms it (cleanup, below). We don't gate on the + // netmap's Online bit (which can lag reality), but we do use it to + // pick between two warning messages. + ps := &pushState{} + if i == 0 { + ps.warnTimer = time.AfterFunc(3*time.Second, func() { + // vtRestartLine clears whatever (possibly progress) was on + // the current line, then we print the warning + \n so the + // next progress redraw lands on a fresh line below. + const vtRestartLine = "\r\x1b[K" + if isOffline { + fmt.Fprintf(Stderr, "%s# warning: %s is reportedly offline; trying anyway\n", vtRestartLine, target) + } else { + fmt.Fprintf(Stderr, "%s# warning: %s is not replying; trying anyway\n", vtRestartLine, target) + } + }) + } + outMu.Lock() + outFiles[name] = ps + outMu.Unlock() + + var group sync.WaitGroup ctxProgress, cancelProgress := context.WithCancel(ctx) defer cancelProgress() - if isatty.IsTerminal(os.Stderr.Fd()) { - group.Go(func() { progressPrinter(ctxProgress, name, fileContents.n.Load, contentLength) }) + if cpArgs.updateInterval > 0 && isatty.IsTerminal(os.Stderr.Fd()) { + group.Go(func() { + progressPrinter(ctxProgress, name, ps.sent.Load, contentLength, cpArgs.updateInterval) + }) } err := localClient.PushFile(ctx, stableID, contentLength, name, fileContents) + if err == nil { + // PushFile can finish faster than the IPN bus delivers a final + // OutgoingFile update, leaving the progress display stuck at 0%. + // Synthesize a "fully done" count before stopping the printer so + // its final paint shows 100%. For stdin (contentLength == -1) we + // don't know the size, so fall back to the local read count. + if contentLength >= 0 { + ps.sent.Store(contentLength) + } else { + ps.sent.Store(fileContents.n.Load()) + } + } cancelProgress() group.Wait() // wait for progress printer to stop before reporting the error + if ps.warnTimer != nil { + ps.warnTimer.Stop() + } if err != nil { return err } @@ -196,15 +280,71 @@ func runCp(ctx context.Context, args []string) error { return nil } -func progressPrinter(ctx context.Context, name string, contentCount func() int64, contentLength int64) { +// watchOutgoingFiles subscribes to the IPN bus and invokes onUpdate once +// per OutgoingFile event for files going to peer. It runs until ctx is +// done (which runCp does on return) and is best-effort: if the bus +// subscription fails for any reason, onUpdate simply isn't called and the +// caller's progress display stays at 0 — exactly the right degradation, +// since the warning timer will then fire on its normal 3-second deadline. +func watchOutgoingFiles(ctx context.Context, peer tailcfg.StableNodeID, onUpdate func(name string, sent int64)) { + // NotifyPeerChanges opts in to per-peer add/remove notifications so the + // bus stays responsive without us also subscribing to the full NetMap, + // which we don't read here. + w, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialOutgoingFiles|ipn.NotifyPeerChanges) + if err != nil { + return + } + defer w.Close() + for { + n, err := w.Next() + if err != nil { + return + } + for _, of := range n.OutgoingFiles { + if of.PeerID != peer { + continue + } + // tailscaled keeps Finished entries in its OutgoingFiles map + // across PushFile calls (see feature/taildrop/ext.go), so a + // re-send of the same filename will see both the old completed + // (Sent == DeclaredSize) entry and the new in-progress one. + // Without this filter the watcher's monotonic CAS would latch + // onto the old entry's max value and the new transfer would + // appear stuck at 100% from the first bus tick. + if of.Finished { + continue + } + onUpdate(of.Name, of.Sent) + } + } +} + +// progressPrinter repaints a single-line transfer progress display every +// interval. interval must be > 0; runCp's caller gates on the +// --update-interval flag and skips invoking us when it's <= 0. +// +// It returns when ctx is done OR when it detects the transfer is stuck — +// "stuck" being: contentCount has equalled contentLength with a near-zero +// rate for >2 seconds. The stuck case prints a final newline so subsequent +// output (e.g. an error from PushFile) lands on a fresh line below the +// frozen progress line, instead of being painted over by it. +func progressPrinter(ctx context.Context, name string, contentCount func() int64, contentLength int64, interval time.Duration) { var rateValueFast, rateValueSlow tsrate.Value - rateValueFast.HalfLife = 1 * time.Second // fast response for rate measurement - rateValueSlow.HalfLife = 10 * time.Second // slow response for ETA measurement + // tailscaled emits OutgoingFile.Sent updates at ~1 Hz, so most printer + // ticks see no delta. With too short a half-life the displayed rate + // roughly halves between updates and doubles back when one arrives, + // looking jumpy. 5s keeps the swing under ~15% while still settling + // within a few seconds of a real change. + rateValueFast.HalfLife = 5 * time.Second // smoothed rate for display + rateValueSlow.HalfLife = 10 * time.Second // even slower, for ETA measurement var prevContentCount int64 print := func() { currContentCount := contentCount() - rateValueFast.Add(float64(currContentCount - prevContentCount)) - rateValueSlow.Add(float64(currContentCount - prevContentCount)) + // Clamp so a regression (which shouldn't happen, but tsrate.Value.Add + // panics on a negative count) can't take down the CLI. + delta := max(currContentCount-prevContentCount, 0) + rateValueFast.Add(float64(delta)) + rateValueSlow.Add(float64(delta)) prevContentCount = currContentCount const vtRestartLine = "\r\x1b[K" @@ -216,16 +356,23 @@ func progressPrinter(ctx context.Context, name string, contentCount func() int64 if contentLength >= 0 { currContentCount = min(currContentCount, contentLength) // cap at 100% ratioRemain := float64(currContentCount) / float64(contentLength) - bytesRemain := float64(contentLength - currContentCount) - secsRemain := bytesRemain / rateValueSlow.Rate() - secs := int(min(max(0, secsRemain), 99*60*60+59+60+59)) + etaStr := "ETA -" + if rate := rateValueSlow.Rate(); rate > 0 { + bytesRemain := float64(contentLength - currContentCount) + secsRemain := bytesRemain / rate + secs := int(min(max(0, secsRemain), 99*60*60+59+60+59)) + etaStr = fmt.Sprintf("ETA %02d:%02d:%02d", secs/60/60, (secs/60)%60, secs%60) + } fmt.Fprintf(os.Stderr, " %s %s", leftPad(fmt.Sprintf("%0.2f%%", 100.0*ratioRemain), len("100.00%")), - fmt.Sprintf("ETA %02d:%02d:%02d", secs/60/60, (secs/60)%60, secs%60)) + etaStr) } } - tc := time.NewTicker(250 * time.Millisecond) + const stuckAfter = 2 * time.Second + var fullStartedAt time.Time // when we first observed currCount==contentLength with ~zero rate + + tc := time.NewTicker(interval) defer tc.Stop() print() for { @@ -236,6 +383,24 @@ func progressPrinter(ctx context.Context, name string, contentCount func() int64 return case <-tc.C: print() + if contentLength < 0 { + continue + } + currCount := contentCount() + rate := rateValueFast.Rate() + if currCount >= contentLength && rate < 1 { + if fullStartedAt.IsZero() { + fullStartedAt = time.Now() + } else if time.Since(fullStartedAt) >= stuckAfter { + // Transfer is stuck at 100% with no movement. Stop + // repainting so we don't keep clobbering anything the + // rest of runCp prints (warnings, errors). + fmt.Fprintln(os.Stderr) + return + } + } else { + fullStartedAt = time.Time{} + } } } } @@ -329,7 +494,10 @@ peerLoop: return "", isOffline, errors.New("cannot send files: missing required Taildrop capability") case ipnstate.TaildropTargetOffline: - return "", isOffline, errors.New("cannot send files: peer is offline") + // Don't gate on the server-reported Online bit (which lags reality + // and isn't always accurate). runCp probes reachability itself with + // TSMP pings. + return foundPeer.ID, isOffline, nil case ipnstate.TaildropTargetNoPeerInfo: return "", isOffline, errors.New("cannot send files: invalid or unrecognized peer") @@ -452,10 +620,10 @@ var fileGetCmd = &ffcli.Command{ Exec: runFileGet, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("get") - fs.BoolVar(&getArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty") - fs.BoolVar(&getArgs.loop, "loop", false, "run get in a loop, receiving files as they come in") - fs.BoolVar(&getArgs.verbose, "verbose", false, "verbose output") - fs.Var(&getArgs.conflict, "conflict", "`behavior`"+` when a conflicting (same-named) file already exists in the target directory. + fs.BoolVar(&fileGetArgs.wait, "wait", false, "wait for a file to arrive if inbox is empty") + fs.BoolVar(&fileGetArgs.loop, "loop", false, "run get in a loop, receiving files as they come in") + fs.BoolVar(&fileGetArgs.verbose, "verbose", false, "verbose output") + fs.Var(&fileGetArgs.conflict, "conflict", "`behavior`"+` when a conflicting (same-named) file already exists in the target directory. skip: skip conflicting files: leave them in the taildrop inbox and print an error. get any non-conflicting files overwrite: overwrite existing file rename: write to a new number-suffixed filename`) @@ -464,7 +632,7 @@ var fileGetCmd = &ffcli.Command{ })(), } -var getArgs = struct { +var fileGetArgs = struct { wait bool loop bool verbose bool @@ -526,7 +694,7 @@ func receiveFile(ctx context.Context, wf apitype.WaitingFile, dir string) (targe return "", 0, fmt.Errorf("opening inbox file %q: %w", wf.Name, err) } defer rc.Close() - f, err := openFileOrSubstitute(dir, wf.Name, getArgs.conflict) + f, err := openFileOrSubstitute(dir, wf.Name, fileGetArgs.conflict) if err != nil { return "", 0, err } @@ -552,10 +720,10 @@ func runFileGetOneBatch(ctx context.Context, dir string) []error { errs = append(errs, fmt.Errorf("getting WaitingFiles: %w", err)) break } - if len(wfs) != 0 || !(getArgs.wait || getArgs.loop) { + if len(wfs) != 0 || !(fileGetArgs.wait || fileGetArgs.loop) { break } - if getArgs.verbose { + if fileGetArgs.verbose { printf("waiting for file...") } if err := waitForFile(ctx); err != nil { @@ -576,7 +744,7 @@ func runFileGetOneBatch(ctx context.Context, dir string) []error { errs = append(errs, err) continue } - if getArgs.verbose { + if fileGetArgs.verbose { printf("wrote %v as %v (%d bytes)\n", wf.Name, writtenFile, size) } if err = localClient.DeleteWaitingFile(ctx, wf.Name); err != nil { @@ -588,7 +756,7 @@ func runFileGetOneBatch(ctx context.Context, dir string) []error { if deleted == 0 && len(wfs) > 0 { // persistently stuck files are basically an error errs = append(errs, fmt.Errorf("moved %d/%d files", deleted, len(wfs))) - } else if getArgs.verbose { + } else if fileGetArgs.verbose { printf("moved %d/%d files\n", deleted, len(wfs)) } return errs @@ -608,7 +776,7 @@ func runFileGet(ctx context.Context, args []string) error { if fi, err := os.Stat(dir); err != nil || !fi.IsDir() { return fmt.Errorf("%q is not a directory", dir) } - if getArgs.loop { + if fileGetArgs.loop { for { errs := runFileGetOneBatch(ctx, dir) for _, err := range errs { @@ -640,7 +808,7 @@ func runFileGet(ctx context.Context, args []string) error { } func wipeInbox(ctx context.Context) error { - if getArgs.wait { + if fileGetArgs.wait { return errors.New("can't use --wait with /dev/null target") } wfs, err := localClient.WaitingFiles(ctx) @@ -649,7 +817,7 @@ func wipeInbox(ctx context.Context) error { } deleted := 0 for _, wf := range wfs { - if getArgs.verbose { + if fileGetArgs.verbose { log.Printf("deleting %v ...", wf.Name) } if err := localClient.DeleteWaitingFile(ctx, wf.Name); err != nil { @@ -657,7 +825,7 @@ func wipeInbox(ctx context.Context) error { } deleted++ } - if getArgs.verbose { + if fileGetArgs.verbose { log.Printf("deleted %d files", deleted) } return nil diff --git a/cmd/tailscale/cli/funnel.go b/cmd/tailscale/cli/funnel.go index f4a1c6bfdb3b8..f16f571e09508 100644 --- a/cmd/tailscale/cli/funnel.go +++ b/cmd/tailscale/cli/funnel.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_serve + package cli import ( @@ -16,6 +18,10 @@ import ( "tailscale.com/tailcfg" ) +func init() { + maybeFunnelCmd = funnelCmd +} + var funnelCmd = func() *ffcli.Command { se := &serveEnv{lc: &localClient} // previously used to serve legacy newFunnelCommand unless useWIPCode is true @@ -174,3 +180,42 @@ func printFunnelWarning(sc *ipn.ServeConfig) { fmt.Fprintf(Stderr, " run: `tailscale serve --help` to see how to configure handlers\n") } } + +func init() { + hookPrintFunnelStatus.Set(printFunnelStatus) +} + +// printFunnelStatus prints the status of the funnel, if it's running. +// It prints nothing if the funnel is not running. +func printFunnelStatus(ctx context.Context) { + sc, err := localClient.GetServeConfig(ctx) + if err != nil { + outln() + printf("# Funnel:\n") + printf("# - Unable to get Funnel status: %v\n", err) + return + } + if !sc.IsFunnelOn() { + return + } + outln() + printf("# Funnel on:\n") + for hp, on := range sc.AllowFunnel { + if !on { // if present, should be on + continue + } + sni, portStr, _ := net.SplitHostPort(string(hp)) + p, _ := strconv.ParseUint(portStr, 10, 16) + isTCP := sc.IsTCPForwardingOnPort(uint16(p), noService) + url := "https://" + if isTCP { + url = "tcp://" + } + url += sni + if isTCP || p != 443 { + url += ":" + portStr + } + printf("# - %s\n", url) + } + outln() +} diff --git a/cmd/tailscale/cli/get.go b/cmd/tailscale/cli/get.go new file mode 100644 index 0000000000000..f9cf3b1c36ee0 --- /dev/null +++ b/cmd/tailscale/cli/get.go @@ -0,0 +1,238 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "strings" + "text/tabwriter" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/net/tsaddr" + "tailscale.com/types/views" +) + +var getCmd = &ffcli.Command{ + Name: "get", + ShortUsage: "tailscale get [flags] [setting-name | all]", + ShortHelp: "Show current preference values", + LongHelp: `"tailscale get" shows the current value of one or all preferences. + +With no argument or "all", all preferences are shown. +With a specific setting name, only that value is shown. + +The setting names are the same flag names accepted by "tailscale set".`, + FlagSet: getFlags, + Exec: runGet, +} + +type getArgsT struct { + json bool + setFlags bool +} + +var getArgs getArgsT + +var getFlags = newGetFlagSet(&getArgs) + +func newGetFlagSet(args *getArgsT) *flag.FlagSet { + fs := newFlagSet("get") + fs.BoolVar(&args.json, "json", false, "output as JSON") + fs.BoolVar(&args.setFlags, "set-flags", false, "output as \"tailscale set\" flag arguments") + return fs +} + +// getSetting is a single preference name-value pair. +type getSetting struct { + name string + value any +} + +func runGet(ctx context.Context, args []string) error { + prefs, err := localClient.GetPrefs(ctx) + if err != nil { + return err + } + st, err := localClient.Status(ctx) + if err != nil { + return err + } + + settings, wantAll, err := selectSettings(prefs, st, effectiveGOOS(), args) + if err != nil { + return err + } + + switch { + case getArgs.json: + return getOutputJSON(settings) + case getArgs.setFlags: + return getOutputSetFlags(settings) + case !wantAll: + // Single value: just print the raw value. + outln(fmt.Sprint(settings[0].value)) + return nil + default: + return getOutputTable(settings) + } +} + +// selectSettings validates args and returns the settings to display. +// wantAll reports whether the caller asked for all settings (no arg or "all"). +func selectSettings(prefs *ipn.Prefs, st *ipnstate.Status, goos string, args []string) (settings []getSetting, wantAll bool, err error) { + if len(args) > 1 { + return nil, false, fmt.Errorf("too many arguments: %q", args) + } + wantAll = len(args) == 0 || args[0] == "all" + if wantAll { + return getSettingsFromPrefs(prefs, st, goos, false), true, nil + } + wantName := args[0] + // When querying a specific name, include hidden flags. + for _, s := range getSettingsFromPrefs(prefs, st, goos, true) { + if s.name == wantName { + return []getSetting{s}, false, nil + } + } + return nil, false, fmt.Errorf("unknown setting %q; see \"tailscale set --help\" for valid settings", wantName) +} + +// getSettingsFromPrefs returns get-able settings derived from prefs, +// using the same flag names as "tailscale set". +// If includeHidden is false, flags with hidden usage strings are omitted. +func getSettingsFromPrefs(prefs *ipn.Prefs, st *ipnstate.Status, goos string, includeHidden bool) []getSetting { + // Use the set command's flag set to get the canonical ordered list + // of flag names and to determine OS applicability. + var dummy setArgsT + fs := newSetFlagSet(goos, &dummy) + + var settings []getSetting + fs.VisitAll(func(f *flag.Flag) { + if preflessFlag(f.Name) { + return + } + if !includeHidden && strings.HasPrefix(f.Usage, hidden) { + return + } + v := prefValue(f.Name, prefs, st) + settings = append(settings, getSetting{name: f.Name, value: v}) + }) + return settings +} + +// prefValue returns the current value of the preference corresponding to +// the given "tailscale set" flag name. +func prefValue(flagName string, prefs *ipn.Prefs, st *ipnstate.Status) any { + switch flagName { + case "accept-routes": + return prefs.RouteAll + case "accept-dns": + return prefs.CorpDNS + case "exit-node": + if prefs.AutoExitNode.IsSet() { + return ipn.AutoExitNodePrefix + string(prefs.AutoExitNode) + } + ip := exitNodeIP(prefs, st) + if ip.IsValid() { + return ip.String() + } + return "" + case "exit-node-allow-lan-access": + return prefs.ExitNodeAllowLANAccess + case "shields-up": + return prefs.ShieldsUp + case "ssh": + return prefs.RunSSH + case "hostname": + return prefs.Hostname + case "advertise-routes": + var sb strings.Builder + for i, r := range tsaddr.WithoutExitRoutes(views.SliceOf(prefs.AdvertiseRoutes)).All() { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(r.String()) + } + return sb.String() + case "advertise-exit-node": + return tsaddr.ContainsExitRoutes(views.SliceOf(prefs.AdvertiseRoutes)) + case "advertise-connector": + return prefs.AppConnector.Advertise + case "nickname": + return prefs.ProfileName + case "update-check": + return prefs.AutoUpdate.Check + case "auto-update": + return prefs.AutoUpdate.Apply.EqualBool(true) + case "report-posture": + return prefs.PostureChecking + case "webclient": + return prefs.RunWebClient + case "operator": + return prefs.OperatorUser + case "snat-subnet-routes": + return !prefs.NoSNAT + case "stateful-filtering": + val, ok := prefs.NoStatefulFiltering.Get() + if ok && val { + return false + } + return true + case "netfilter-mode": + return prefs.NetfilterMode.String() + case "unattended": + return prefs.ForceDaemon + case "sync": + return prefs.Sync.EqualBool(true) + case "relay-server-port": + if prefs.RelayServerPort != nil { + return fmt.Sprint(*prefs.RelayServerPort) + } + return "" + case "relay-server-static-endpoints": + parts := make([]string, len(prefs.RelayServerStaticEndpoints)) + for i, ep := range prefs.RelayServerStaticEndpoints { + parts[i] = ep.String() + } + return strings.Join(parts, ",") + default: + return nil + } +} + +func getOutputTable(settings []getSetting) error { + w := tabwriter.NewWriter(Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintf(w, "NAME\tVALUE\n") + for _, s := range settings { + fmt.Fprintf(w, "%s\t%v\n", s.name, s.value) + } + return w.Flush() +} + +func getOutputJSON(settings []getSetting) error { + m := make(map[string]any, len(settings)) + for _, s := range settings { + m[s.name] = s.value + } + j, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + outln(string(j)) + return nil +} + +func getOutputSetFlags(settings []getSetting) error { + var parts []string + for _, s := range settings { + parts = append(parts, fmtFlagValueArg(s.name, s.value)) + } + outln(strings.Join(parts, " ")) + return nil +} diff --git a/cmd/tailscale/cli/get_test.go b/cmd/tailscale/cli/get_test.go new file mode 100644 index 0000000000000..7b9f13ea3a085 --- /dev/null +++ b/cmd/tailscale/cli/get_test.go @@ -0,0 +1,596 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "bytes" + "encoding/json" + "flag" + "io" + "net/netip" + "reflect" + "strings" + "testing" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/safesocket" + "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/types/key" + "tailscale.com/types/opt" + "tailscale.com/types/preftype" +) + +func TestPrefValue(t *testing.T) { + port := uint16(41641) + peerKey := key.NewNode().Public() + exitPeerID := tailcfg.StableNodeID("exit-peer") + exitPeerIP := netip.MustParseAddr("100.64.0.5") + + stWithExitPeer := &ipnstate.Status{ + Peer: map[key.NodePublic]*ipnstate.PeerStatus{ + peerKey: { + ID: exitPeerID, + TailscaleIPs: []netip.Addr{exitPeerIP}, + }, + }, + } + + tests := []struct { + name string + flag string + prefs *ipn.Prefs + st *ipnstate.Status + want any + }{ + // Simple boolean prefs. + { + name: "accept-routes-true", + flag: "accept-routes", + prefs: &ipn.Prefs{RouteAll: true}, + want: true, + }, + { + name: "accept-routes-false", + flag: "accept-routes", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "accept-dns", + flag: "accept-dns", + prefs: &ipn.Prefs{CorpDNS: true}, + want: true, + }, + { + name: "exit-node-allow-lan-access", + flag: "exit-node-allow-lan-access", + prefs: &ipn.Prefs{ExitNodeAllowLANAccess: true}, + want: true, + }, + { + name: "shields-up", + flag: "shields-up", + prefs: &ipn.Prefs{ShieldsUp: true}, + want: true, + }, + { + name: "ssh", + flag: "ssh", + prefs: &ipn.Prefs{RunSSH: true}, + want: true, + }, + { + name: "advertise-connector", + flag: "advertise-connector", + prefs: &ipn.Prefs{AppConnector: ipn.AppConnectorPrefs{Advertise: true}}, + want: true, + }, + { + name: "update-check", + flag: "update-check", + prefs: &ipn.Prefs{AutoUpdate: ipn.AutoUpdatePrefs{Check: true}}, + want: true, + }, + { + name: "report-posture", + flag: "report-posture", + prefs: &ipn.Prefs{PostureChecking: true}, + want: true, + }, + { + name: "webclient", + flag: "webclient", + prefs: &ipn.Prefs{RunWebClient: true}, + want: true, + }, + { + name: "unattended", + flag: "unattended", + prefs: &ipn.Prefs{ForceDaemon: true}, + want: true, + }, + + // Simple string prefs. + { + name: "hostname", + flag: "hostname", + prefs: &ipn.Prefs{Hostname: "myhost"}, + want: "myhost", + }, + { + name: "nickname", + flag: "nickname", + prefs: &ipn.Prefs{ProfileName: "work"}, + want: "work", + }, + { + name: "operator", + flag: "operator", + prefs: &ipn.Prefs{OperatorUser: "alice"}, + want: "alice", + }, + + // exit-node has three branches. + { + name: "exit-node/auto", + flag: "exit-node", + prefs: &ipn.Prefs{AutoExitNode: ipn.AnyExitNode}, + want: "auto:any", + }, + { + name: "exit-node/by-ip", + flag: "exit-node", + prefs: &ipn.Prefs{ExitNodeIP: netip.MustParseAddr("100.64.0.1")}, + want: "100.64.0.1", + }, + { + name: "exit-node/by-id-resolves-via-status", + flag: "exit-node", + prefs: &ipn.Prefs{ExitNodeID: exitPeerID}, + st: stWithExitPeer, + want: exitPeerIP.String(), + }, + { + name: "exit-node/empty", + flag: "exit-node", + prefs: &ipn.Prefs{}, + want: "", + }, + + // advertise-routes filters out exit routes, comma-joins. + { + name: "advertise-routes/multiple", + flag: "advertise-routes", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/24"), + netip.MustParsePrefix("192.168.0.0/16"), + }}, + want: "10.0.0.0/24,192.168.0.0/16", + }, + { + name: "advertise-routes/excludes-exit-routes", + flag: "advertise-routes", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/24"), + netip.MustParsePrefix("0.0.0.0/0"), + netip.MustParsePrefix("::/0"), + }}, + want: "10.0.0.0/24", + }, + { + name: "advertise-routes/empty", + flag: "advertise-routes", + prefs: &ipn.Prefs{}, + want: "", + }, + + // advertise-exit-node derives from AdvertiseRoutes. + { + name: "advertise-exit-node/true", + flag: "advertise-exit-node", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/0"), + netip.MustParsePrefix("::/0"), + }}, + want: true, + }, + { + name: "advertise-exit-node/false-empty", + flag: "advertise-exit-node", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "advertise-exit-node/false-only-subnet", + flag: "advertise-exit-node", + prefs: &ipn.Prefs{AdvertiseRoutes: []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/24"), + }}, + want: false, + }, + + // auto-update and sync use opt.Bool.EqualBool(true). + { + name: "auto-update/unset-is-false", + flag: "auto-update", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "auto-update/explicit-true", + flag: "auto-update", + prefs: &ipn.Prefs{AutoUpdate: ipn.AutoUpdatePrefs{Apply: opt.NewBool(true)}}, + want: true, + }, + { + name: "auto-update/explicit-false", + flag: "auto-update", + prefs: &ipn.Prefs{AutoUpdate: ipn.AutoUpdatePrefs{Apply: opt.NewBool(false)}}, + want: false, + }, + { + name: "sync/unset-is-false", + flag: "sync", + prefs: &ipn.Prefs{}, + want: false, + }, + { + name: "sync/explicit-true", + flag: "sync", + prefs: &ipn.Prefs{Sync: opt.NewBool(true)}, + want: true, + }, + + // snat-subnet-routes is inverted. + { + name: "snat-subnet-routes/default-true", + flag: "snat-subnet-routes", + prefs: &ipn.Prefs{}, + want: true, + }, + { + name: "snat-subnet-routes/false-when-no-snat", + flag: "snat-subnet-routes", + prefs: &ipn.Prefs{NoSNAT: true}, + want: false, + }, + + // stateful-filtering: the inversion of NoStatefulFiltering, defaulting on. + { + name: "stateful-filtering/unset-is-true", + flag: "stateful-filtering", + prefs: &ipn.Prefs{}, + want: true, + }, + { + name: "stateful-filtering/explicit-disabled-no-stateful", + flag: "stateful-filtering", + prefs: &ipn.Prefs{NoStatefulFiltering: opt.NewBool(true)}, + want: false, + }, + { + name: "stateful-filtering/explicit-enabled-no-stateful", + flag: "stateful-filtering", + prefs: &ipn.Prefs{NoStatefulFiltering: opt.NewBool(false)}, + want: true, + }, + + // netfilter-mode renders via String(). + { + name: "netfilter-mode/off", + flag: "netfilter-mode", + prefs: &ipn.Prefs{NetfilterMode: preftype.NetfilterOff}, + want: "off", + }, + { + name: "netfilter-mode/on", + flag: "netfilter-mode", + prefs: &ipn.Prefs{NetfilterMode: preftype.NetfilterOn}, + want: "on", + }, + + // relay-server-port: nil pointer vs explicit. + { + name: "relay-server-port/unset", + flag: "relay-server-port", + prefs: &ipn.Prefs{}, + want: "", + }, + { + name: "relay-server-port/set", + flag: "relay-server-port", + prefs: &ipn.Prefs{RelayServerPort: &port}, + want: "41641", + }, + + // relay-server-static-endpoints: empty vs joined. + { + name: "relay-server-static-endpoints/empty", + flag: "relay-server-static-endpoints", + prefs: &ipn.Prefs{}, + want: "", + }, + { + name: "relay-server-static-endpoints/multiple", + flag: "relay-server-static-endpoints", + prefs: &ipn.Prefs{RelayServerStaticEndpoints: []netip.AddrPort{ + netip.MustParseAddrPort("192.0.2.1:40000"), + netip.MustParseAddrPort("[2001:db8::1]:40000"), + }}, + want: "192.0.2.1:40000,[2001:db8::1]:40000", + }, + + // Unknown flag returns nil. This guards against the default branch + // silently producing nil for a flag that should have been wired up. + { + name: "unknown-flag", + flag: "no-such-flag", + prefs: &ipn.Prefs{}, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := tt.st + if st == nil { + st = &ipnstate.Status{} + } + got := prefValue(tt.flag, tt.prefs, st) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("prefValue(%q) = %v (%T), want %v (%T)", + tt.flag, got, got, tt.want, tt.want) + } + }) + } +} + +// TestPrefValueCoversAllSetFlags is the load-bearing guard: every flag +// that "tailscale set" exposes must have a corresponding prefValue case, +// or "tailscale get" silently returns nil for it. It iterates the set +// command's flag set across the platforms whose flag sets differ, so +// OS-conditional flags (snat-subnet-routes, netfilter-mode, unattended, +// operator, ...) are all covered. +func TestPrefValueCoversAllSetFlags(t *testing.T) { + for _, goos := range []string{"linux", "darwin", "windows"} { + t.Run(goos, func(t *testing.T) { + var dummy setArgsT + fs := newSetFlagSet(goos, &dummy) + fs.VisitAll(func(f *flag.Flag) { + if preflessFlag(f.Name) { + return + } + if got := prefValue(f.Name, &ipn.Prefs{}, &ipnstate.Status{}); got == nil { + t.Errorf("prefValue(%q) returned nil; add a case for it in prefValue", f.Name) + } + }) + }) + } +} + +func TestGetSettingsFromPrefsHiddenFlag(t *testing.T) { + prefs := &ipn.Prefs{} + st := &ipnstate.Status{} + + visible := getSettingsFromPrefs(prefs, st, "linux", false) + if containsSetting(visible, "sync") { + t.Error("expected hidden flag --sync to be excluded when includeHidden=false") + } + if !containsSetting(visible, "accept-dns") { + t.Error("expected visible flag --accept-dns to be included") + } + + withHidden := getSettingsFromPrefs(prefs, st, "linux", true) + if !containsSetting(withHidden, "sync") { + t.Error("expected hidden flag --sync to be included when includeHidden=true") + } + + // Ordering must match the set flag set's VisitAll order. + var wantOrder []string + var dummy setArgsT + newSetFlagSet("linux", &dummy).VisitAll(func(f *flag.Flag) { + if preflessFlag(f.Name) { + return + } + wantOrder = append(wantOrder, f.Name) + }) + var gotOrder []string + for _, s := range withHidden { + gotOrder = append(gotOrder, s.name) + } + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Errorf("setting order = %v, want %v", gotOrder, wantOrder) + } +} + +func TestSelectSettings(t *testing.T) { + prefs := &ipn.Prefs{Hostname: "h", CorpDNS: true} + st := &ipnstate.Status{} + const goos = "linux" + + t.Run("empty-args-returns-all-visible", func(t *testing.T) { + got, wantAll, err := selectSettings(prefs, st, goos, nil) + if err != nil { + t.Fatal(err) + } + if !wantAll { + t.Error("wantAll = false; want true") + } + if containsSetting(got, "sync") { + t.Error("hidden flag --sync leaked into all-settings result") + } + if !containsSetting(got, "hostname") { + t.Error("missing --hostname in all-settings result") + } + }) + + t.Run("all-arg-same-as-empty", func(t *testing.T) { + empty, _, err := selectSettings(prefs, st, goos, nil) + if err != nil { + t.Fatal(err) + } + allArg, wantAll, err := selectSettings(prefs, st, goos, []string{"all"}) + if err != nil { + t.Fatal(err) + } + if !wantAll { + t.Error("wantAll = false; want true for explicit \"all\"") + } + if !reflect.DeepEqual(empty, allArg) { + t.Errorf("\"all\" produced %v, empty produced %v", allArg, empty) + } + }) + + t.Run("specific-visible-flag", func(t *testing.T) { + got, wantAll, err := selectSettings(prefs, st, goos, []string{"hostname"}) + if err != nil { + t.Fatal(err) + } + if wantAll { + t.Error("wantAll = true; want false for specific name") + } + if len(got) != 1 || got[0].name != "hostname" || got[0].value != "h" { + t.Errorf("got %+v, want [{hostname h}]", got) + } + }) + + t.Run("specific-hidden-flag", func(t *testing.T) { + // Hidden flags must be reachable by exact name. + got, _, err := selectSettings(prefs, st, goos, []string{"sync"}) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].name != "sync" { + t.Errorf("got %+v, want [{sync ...}]", got) + } + }) + + t.Run("unknown-flag-errors", func(t *testing.T) { + _, _, err := selectSettings(prefs, st, goos, []string{"no-such-flag"}) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "unknown setting") || !strings.Contains(err.Error(), "no-such-flag") { + t.Errorf("error %q missing expected substrings", err) + } + }) + + t.Run("too-many-args-errors", func(t *testing.T) { + _, _, err := selectSettings(prefs, st, goos, []string{"hostname", "ssh"}) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "too many arguments") { + t.Errorf("error %q missing \"too many arguments\"", err) + } + }) + + t.Run("os-conditional-flag-on-wrong-goos", func(t *testing.T) { + // "netfilter-mode" is registered only on linux. Asking for it + // on darwin should produce an "unknown setting" error. + _, _, err := selectSettings(prefs, st, "darwin", []string{"netfilter-mode"}) + if err == nil || !strings.Contains(err.Error(), "unknown setting") { + t.Errorf("got err=%v, want \"unknown setting\"", err) + } + // And operator is peer-creds-only. + if safesocket.GOOSUsesPeerCreds("windows") { + t.Skip("operator is exposed on windows") + } + _, _, err = selectSettings(prefs, st, "windows", []string{"operator"}) + if err == nil || !strings.Contains(err.Error(), "unknown setting") { + t.Errorf("got err=%v, want \"unknown setting\"", err) + } + }) +} + +func TestGetOutputJSON(t *testing.T) { + var buf bytes.Buffer + tstest.Replace[io.Writer](t, &Stdout, &buf) + + settings := []getSetting{ + {name: "accept-dns", value: true}, + {name: "hostname", value: "myhost"}, + {name: "advertise-routes", value: "10.0.0.0/24"}, + {name: "shields-up", value: false}, + } + if err := getOutputJSON(settings); err != nil { + t.Fatal(err) + } + + var got map[string]any + if err := json.Unmarshal(buf.Bytes(), &got); err != nil { + t.Fatalf("output is not valid JSON: %v\noutput: %s", err, buf.String()) + } + want := map[string]any{ + "accept-dns": true, + "hostname": "myhost", + "advertise-routes": "10.0.0.0/24", + "shields-up": false, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestGetOutputTable(t *testing.T) { + var buf bytes.Buffer + tstest.Replace[io.Writer](t, &Stdout, &buf) + + settings := []getSetting{ + {name: "accept-dns", value: true}, + {name: "hostname", value: "myhost"}, + } + if err := getOutputTable(settings); err != nil { + t.Fatal(err) + } + + out := buf.String() + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if len(lines) != 3 { + t.Fatalf("got %d lines, want 3:\n%s", len(lines), out) + } + if !strings.HasPrefix(lines[0], "NAME") || !strings.Contains(lines[0], "VALUE") { + t.Errorf("header line = %q, want NAME ... VALUE", lines[0]) + } + if !strings.HasPrefix(lines[1], "accept-dns") || !strings.HasSuffix(lines[1], "true") { + t.Errorf("row 1 = %q", lines[1]) + } + if !strings.HasPrefix(lines[2], "hostname") || !strings.HasSuffix(lines[2], "myhost") { + t.Errorf("row 2 = %q", lines[2]) + } +} + +func TestGetOutputSetFlags(t *testing.T) { + var buf bytes.Buffer + tstest.Replace[io.Writer](t, &Stdout, &buf) + + settings := []getSetting{ + {name: "ssh", value: true}, + {name: "shields-up", value: false}, + {name: "hostname", value: "myhost"}, + {name: "advertise-routes", value: ""}, + } + if err := getOutputSetFlags(settings); err != nil { + t.Fatal(err) + } + + got := strings.TrimSpace(buf.String()) + // true → bare flag; false → --flag=false; empty string → --flag=; other → --flag=value + want := "--ssh --shields-up=false --hostname=myhost --advertise-routes=" + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// containsSetting reports whether settings contains a setting with the given name. +func containsSetting(settings []getSetting, name string) bool { + for _, s := range settings { + if s.name == name { + return true + } + } + return false +} diff --git a/cmd/tailscale/cli/id-token.go b/cmd/tailscale/cli/id-token.go index a4d02c95a82c1..e2707ee84ca42 100644 --- a/cmd/tailscale/cli/id-token.go +++ b/cmd/tailscale/cli/id-token.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/ip.go b/cmd/tailscale/cli/ip.go index 8379329120436..6ff1cf867a827 100644 --- a/cmd/tailscale/cli/ip.go +++ b/cmd/tailscale/cli/ip.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -9,30 +9,34 @@ import ( "flag" "fmt" "net/netip" + "slices" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" ) var ipCmd = &ffcli.Command{ Name: "ip", - ShortUsage: "tailscale ip [-1] [-4] [-6] [peer hostname or ip address]", + ShortUsage: "tailscale ip [-1] [-4] [-6] [peer or service hostname or ip address]", ShortHelp: "Show Tailscale IP addresses", - LongHelp: "Show Tailscale IP addresses for peer. Peer defaults to the current machine.", + LongHelp: "Show Tailscale IP addresses for peer or service. Peer defaults to the current machine.", Exec: runIP, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("ip") fs.BoolVar(&ipArgs.want1, "1", false, "only print one IP address") fs.BoolVar(&ipArgs.want4, "4", false, "only print IPv4 address") fs.BoolVar(&ipArgs.want6, "6", false, "only print IPv6 address") + fs.StringVar(&ipArgs.assert, "assert", "", "assert that one of the node's IP(s) matches this IP address") return fs })(), } var ipArgs struct { - want1 bool - want4 bool - want6 bool + want1 bool + want4 bool + want6 bool + assert string } func runIP(ctx context.Context, args []string) error { @@ -62,16 +66,34 @@ func runIP(ctx context.Context, args []string) error { return err } ips := st.TailscaleIPs + if ipArgs.assert != "" { + for _, ip := range ips { + if ip.String() == ipArgs.assert { + return nil + } + } + return fmt.Errorf("assertion failed: IP %q not found among %v", ipArgs.assert, ips) + } if of != "" { ip, _, err := tailscaleIPFromArg(ctx, of) if err != nil { return err } peer, ok := peerMatchingIP(st, ip) - if !ok { - return fmt.Errorf("no peer found with IP %v", ip) + if ok { + ips = peer.TailscaleIPs + } else { + // No peer matched; check if the IP belongs to a service. + serviceIPs, err := serviceAddrsMatchingIP(ctx, ip) + if err != nil { + return err + } + if serviceIPs != nil { + ips = serviceIPs + } else { + return fmt.Errorf("no peer or service found with IP %v", ip) + } } - ips = peer.TailscaleIPs } if len(ips) == 0 { return fmt.Errorf("no current Tailscale IPs; state: %v", st.BackendState) @@ -98,23 +120,44 @@ func runIP(ctx context.Context, args []string) error { return nil } +// serviceAddrsMatchingIP checks whether ipStr matches a service's VIP address +// and returns the service's addresses if so. +func serviceAddrsMatchingIP(ctx context.Context, ipStr string) ([]netip.Addr, error) { + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return nil, nil + } + services, err := localClient.GetServices(ctx) + if err != nil { + return nil, err + } + return allIPsForServiceWithIP(services, ip), nil +} + +// allIPsForServiceWithIP returns the Addrs of the service whose VIP addresses +// contain ip, or nil if no service matches. +func allIPsForServiceWithIP(services map[tailcfg.ServiceName]tailcfg.ServiceDetails, ip netip.Addr) []netip.Addr { + for _, svc := range services { + if slices.Contains(svc.Addrs, ip) { + return svc.Addrs + } + } + return nil +} + func peerMatchingIP(st *ipnstate.Status, ipStr string) (ps *ipnstate.PeerStatus, ok bool) { ip, err := netip.ParseAddr(ipStr) if err != nil { return } for _, ps = range st.Peer { - for _, pip := range ps.TailscaleIPs { - if ip == pip { - return ps, true - } + if slices.Contains(ps.TailscaleIPs, ip) { + return ps, true } } if ps := st.Self; ps != nil { - for _, pip := range ps.TailscaleIPs { - if ip == pip { - return ps, true - } + if slices.Contains(ps.TailscaleIPs, ip) { + return ps, true } } return nil, false diff --git a/cmd/tailscale/cli/ip_test.go b/cmd/tailscale/cli/ip_test.go new file mode 100644 index 0000000000000..9e5d31166ee36 --- /dev/null +++ b/cmd/tailscale/cli/ip_test.go @@ -0,0 +1,199 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "net/netip" + "testing" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/key" +) + +func TestPeerMatchingIP(t *testing.T) { + st := &ipnstate.Status{ + Self: &ipnstate.PeerStatus{ + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1"), netip.MustParseAddr("fd7a:115c:a1e0::1")}, + }, + Peer: map[key.NodePublic]*ipnstate.PeerStatus{ + key.NewNode().Public(): { + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.2"), netip.MustParseAddr("fd7a:115c:a1e0::2")}, + }, + key.NewNode().Public(): { + TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.3")}, + }, + }, + } + + tests := []struct { + name string + ipStr string + wantOK bool + wantIPs []netip.Addr + }{ + { + name: "match_self_v4", + ipStr: "100.64.0.1", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1"), netip.MustParseAddr("fd7a:115c:a1e0::1")}, + }, + { + name: "match_self_v6", + ipStr: "fd7a:115c:a1e0::1", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1"), netip.MustParseAddr("fd7a:115c:a1e0::1")}, + }, + { + name: "match_peer_v4", + ipStr: "100.64.0.2", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.2"), netip.MustParseAddr("fd7a:115c:a1e0::2")}, + }, + { + name: "match_peer_single_ip", + ipStr: "100.64.0.3", + wantOK: true, + wantIPs: []netip.Addr{netip.MustParseAddr("100.64.0.3")}, + }, + { + name: "no_match", + ipStr: "100.64.0.99", + wantOK: false, + }, + { + name: "invalid_ip", + ipStr: "not-an-ip", + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ps, ok := peerMatchingIP(st, tt.ipStr) + if ok != tt.wantOK { + t.Fatalf("peerMatchingIP(%q) ok = %v, want %v", tt.ipStr, ok, tt.wantOK) + } + if ok { + if len(ps.TailscaleIPs) != len(tt.wantIPs) { + t.Fatalf("got %d IPs, want %d", len(ps.TailscaleIPs), len(tt.wantIPs)) + } + for i, ip := range ps.TailscaleIPs { + if ip != tt.wantIPs[i] { + t.Errorf("IP[%d] = %v, want %v", i, ip, tt.wantIPs[i]) + } + } + } + }) + } +} + +func TestAllIPsForServiceWithIP(t *testing.T) { + services := map[tailcfg.ServiceName]tailcfg.ServiceDetails{ + "svc:web": { + Name: "svc:web", + Addrs: []netip.Addr{ + netip.MustParseAddr("100.100.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + }, + }, + "svc:api": { + Name: "svc:api", + Addrs: []netip.Addr{netip.MustParseAddr("100.100.0.2")}, + }, + } + // Services should have at most 2 addrs (one v4, one v6), but + // we handle more gracefully if the server ever returns them. + multiAddr := map[tailcfg.ServiceName]tailcfg.ServiceDetails{ + "svc:multi": { + Name: "svc:multi", + Addrs: []netip.Addr{ + netip.MustParseAddr("100.100.0.3"), + netip.MustParseAddr("100.100.0.4"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::4"), + }, + }, + } + + tests := []struct { + name string + services map[tailcfg.ServiceName]tailcfg.ServiceDetails + ip netip.Addr + wantIPs []netip.Addr + }{ + { + name: "match_service_v4", + services: services, + ip: netip.MustParseAddr("100.100.0.1"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + }, + }, + { + name: "match_service_v6", + services: services, + ip: netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.1"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::1"), + }, + }, + { + name: "match_single_addr_service", + services: services, + ip: netip.MustParseAddr("100.100.0.2"), + wantIPs: []netip.Addr{netip.MustParseAddr("100.100.0.2")}, + }, + { + name: "match_service_multiple_addrs_v4", + services: multiAddr, + ip: netip.MustParseAddr("100.100.0.3"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.3"), + netip.MustParseAddr("100.100.0.4"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::4"), + }, + }, + { + name: "match_service_multiple_addrs_v6", + services: multiAddr, + ip: netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + wantIPs: []netip.Addr{ + netip.MustParseAddr("100.100.0.3"), + netip.MustParseAddr("100.100.0.4"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::3"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12::4"), + }, + }, + { + name: "no_match", + services: services, + ip: netip.MustParseAddr("100.100.0.99"), + wantIPs: nil, + }, + { + name: "empty_services", + services: nil, + ip: netip.MustParseAddr("100.100.0.1"), + wantIPs: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := allIPsForServiceWithIP(tt.services, tt.ip) + if len(got) != len(tt.wantIPs) { + t.Fatalf("allIPsForServiceWithIP(%v) returned %d IPs, want %d", tt.ip, len(got), len(tt.wantIPs)) + } + for i, ip := range got { + if ip != tt.wantIPs[i] { + t.Errorf("IP[%d] = %v, want %v", i, ip, tt.wantIPs[i]) + } + } + }) + } +} diff --git a/cmd/tailscale/cli/jsonoutput/dns.go b/cmd/tailscale/cli/jsonoutput/dns.go new file mode 100644 index 0000000000000..a03113f0ea0c0 --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/dns.go @@ -0,0 +1,121 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput + +// DNSResolverInfo is the JSON form of [tailscale.com/types/dnstype.Resolver]. +type DNSResolverInfo struct { + // Addr is a plain IP, IP:port, DoH URL, or HTTP-over-WireGuard URL. + Addr string + + // BootstrapResolution is optional pre-resolved IPs for DoT/DoH + // resolvers whose address is not already an IP. + BootstrapResolution []string `json:",omitempty"` +} + +// DNSExtraRecord is the JSON form of [tailscale.com/tailcfg.DNSRecord]. +type DNSExtraRecord struct { + Name string + Type string `json:",omitempty"` // empty means A or AAAA, depending on Value + Value string // typically an IP address +} + +// DNSSystemConfig is the OS DNS configuration as observed by Tailscale, +// mirroring [tailscale.com/net/dns.OSConfig]. +type DNSSystemConfig struct { + Nameservers []string `json:",omitzero"` + SearchDomains []string `json:",omitzero"` + + // MatchDomains are DNS suffixes restricting which queries use + // these Nameservers. Empty means Nameservers is the primary + // resolver. + MatchDomains []string `json:",omitzero"` +} + +// DNSTailnetInfo describes MagicDNS configuration for the tailnet, +// combining [tailscale.com/ipn/ipnstate.TailnetStatus] +// and [tailscale.com/ipn/ipnstate.PeerStatus]. +type DNSTailnetInfo struct { + // MagicDNSEnabled is whether MagicDNS is enabled for the + // tailnet. The device may still not use it if + // --accept-dns=false. + MagicDNSEnabled bool + + // MagicDNSSuffix is the tailnet's MagicDNS suffix + // (e.g. "tail1234.ts.net"), without surrounding dots. + MagicDNSSuffix string `json:",omitempty"` + + // SelfDNSName is this device's FQDN + // (e.g. "host.tail1234.ts.net."), with trailing dot. + SelfDNSName string `json:",omitempty"` +} + +// DNSStatusResult is the full DNS status collected from the local +// Tailscale daemon. It is the output of: +// +// $ tailscale dns status --json +type DNSStatusResult struct { + // TailscaleDNS is whether the Tailscale DNS configuration is + // installed on this device (the --accept-dns setting). + TailscaleDNS bool + + // CurrentTailnet describes MagicDNS configuration for the tailnet. + CurrentTailnet *DNSTailnetInfo `json:",omitzero"` // nil if not connected + + // Resolvers are the DNS resolvers, in preference order. If + // empty, the system defaults are used. + Resolvers []DNSResolverInfo `json:",omitzero"` + + // SplitDNSRoutes maps domain suffixes to dedicated resolvers. + // An empty resolver slice means the suffix is handled by + // Tailscale's built-in resolver (100.100.100.100). + SplitDNSRoutes map[string][]DNSResolverInfo `json:",omitzero"` + + // FallbackResolvers are like Resolvers but only used when + // split DNS needs explicit default resolvers. + FallbackResolvers []DNSResolverInfo `json:",omitzero"` + + SearchDomains []string `json:",omitzero"` + + // Nameservers are nameserver IPs. + // + // Deprecated: old protocol versions only. Use Resolvers. + Nameservers []string `json:",omitzero"` + + // CertDomains are FQDNs for which the coordination server + // provisions TLS certificates via dns-01 ACME challenges. + CertDomains []string `json:",omitzero"` + + // ExtraRecords contains extra DNS records in the MagicDNS config. + ExtraRecords []DNSExtraRecord `json:",omitzero"` + + // ExitNodeFilteredSet are DNS suffixes this node won't resolve + // when acting as an exit node DNS proxy. Period-prefixed + // entries are suffix matches; others are exact. Always + // lowercase, no trailing dots. + ExitNodeFilteredSet []string `json:",omitzero"` + + SystemDNS *DNSSystemConfig `json:",omitzero"` // nil if unavailable + SystemDNSError string `json:",omitempty"` +} + +// DNSAnswer is a single DNS resource record from a query response. +type DNSAnswer struct { + Name string + TTL uint32 + Class string // e.g. "ClassINET" + Type string // e.g. "TypeA", "TypeAAAA" + Body string // human-readable record data +} + +// DNSQueryResult is the result of a DNS query via the Tailscale +// internal forwarder (100.100.100.100). It is the output of: +// +// $ tailscale dns query --json NAME +type DNSQueryResult struct { + Name string + QueryType string // e.g. "A", "AAAA" + Resolvers []DNSResolverInfo `json:",omitzero"` + ResponseCode string // e.g. "RCodeSuccess", "RCodeNameError" + Answers []DNSAnswer `json:",omitzero"` +} diff --git a/cmd/tailscale/cli/jsonoutput/example_dnsqueryresult_test.go b/cmd/tailscale/cli/jsonoutput/example_dnsqueryresult_test.go new file mode 100644 index 0000000000000..f4f38d74cb57e --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/example_dnsqueryresult_test.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + + "tailscale.com/cmd/tailscale/cli/jsonoutput" +) + +func ExampleDNSQueryResult() { + cmd := exec.Command("tailscale", "dns", "query", "--json", "hello.ts.net") + out, err := cmd.Output() + if err != nil { + if err, ok := errors.AsType[*exec.ExitError](err); ok { + fmt.Fprintf(os.Stderr, "%s", err.Stderr) + } + panic(err) + } + + var dnsQuery jsonoutput.DNSQueryResult + if err := json.Unmarshal(out, &dnsQuery); err != nil { + panic(err) + } + fmt.Printf("{type: %s, name: %q}\n", dnsQuery.QueryType, dnsQuery.Name) +} diff --git a/cmd/tailscale/cli/jsonoutput/example_dnsstatusresult_test.go b/cmd/tailscale/cli/jsonoutput/example_dnsstatusresult_test.go new file mode 100644 index 0000000000000..ca53c60026be4 --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/example_dnsstatusresult_test.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + + "tailscale.com/cmd/tailscale/cli/jsonoutput" +) + +func ExampleDNSStatusResult() { + cmd := exec.Command("tailscale", "dns", "status", "--json") + out, err := cmd.Output() + if err != nil { + if err, ok := errors.AsType[*exec.ExitError](err); ok { + fmt.Fprintf(os.Stderr, "%s", err.Stderr) + } + panic(err) + } + + var dnsStatus jsonoutput.DNSStatusResult + if err := json.Unmarshal(out, &dnsStatus); err != nil { + panic(err) + } + fmt.Printf("{accept-dns: %t, resolvers: %q}\n", dnsStatus.TailscaleDNS, dnsStatus.Resolvers) +} diff --git a/cmd/tailscale/cli/jsonoutput/example_responseenvelope_test.go b/cmd/tailscale/cli/jsonoutput/example_responseenvelope_test.go new file mode 100644 index 0000000000000..b48edef6dc7ee --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/example_responseenvelope_test.go @@ -0,0 +1,33 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "encoding/json" + "fmt" + + "tailscale.com/cmd/tailscale/cli/jsonoutput" +) + +type Hello struct { + jsonoutput.ResponseEnvelope + Greeting string +} + +func ExampleResponseEnvelope() { + hi := Hello{ + ResponseEnvelope: jsonoutput.ResponseEnvelope{SchemaVersion: "1"}, + Greeting: "Hello, world", + } + out, err := json.MarshalIndent(hi, "", " ") + if err != nil { + panic(err) + } + fmt.Printf("%s\n", out) + // Output: + // { + // "SchemaVersion": "1", + // "Greeting": "Hello, world" + // } +} diff --git a/cmd/tailscale/cli/jsonoutput/example_schemaversion_test.go b/cmd/tailscale/cli/jsonoutput/example_schemaversion_test.go new file mode 100644 index 0000000000000..5d5246a83bc2f --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/example_schemaversion_test.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "flag" + "fmt" + + "tailscale.com/cmd/tailscale/cli/jsonoutput" +) + +var args struct { + json jsonoutput.SchemaVersion +} + +func ExampleSchemaVersion() { + fs := flag.NewFlagSet("ExampleSchemaVersion", flag.ExitOnError) + fs.Var(&args.json, "json", "output in JSON format") + + fs.Parse([]string{"-json=2"}) + fmt.Printf(`{set: %t, value: %d}`, args.json.IsSet, args.json.Version) + // Output: + // {set: true, value: 2} +} diff --git a/cmd/tailscale/cli/jsonoutput/jsonoutput.go b/cmd/tailscale/cli/jsonoutput/jsonoutput.go new file mode 100644 index 0000000000000..41b605eea6b3c --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/jsonoutput.go @@ -0,0 +1,104 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package jsonoutput provides stable and versioned JSON serialisation for CLI output. +// This allows us to provide stable output to scripts/clients, but also make +// breaking changes to the output when it's useful. +// +// Historically we only used a boolean -json flag, so changing the output +// could break scripts that rely on the existing format. +// +// This package provides a [SchemaVersion] flag type that allows callers +// to pass either a boolean or a version number and get a consistent output. +// We'll bump the version when we make a breaking change +// that's likely to break scripts that rely on the existing output, +// e.g. if we remove a field or change the type/format. +// Passing just the boolean flag will always return 1, to preserve +// compatibility with scripts written before we versioned our output. +// +// This package also provides [ResponseEnvelope] which is used to provide the +// set of fields common to all versioned JSON output. +package jsonoutput + +import ( + "errors" + "flag" + "io" + "strconv" + "strings" +) + +var _ flag.Value = &SchemaVersion{} + +// SchemaVersion implements the [flag.Value] interface, +// tracking whether the flag has been set or cleared, and its value when set. +type SchemaVersion struct { + // IsSet tracks if the flag was set or cleared. + // This flag is true when set by -name or -name=true or -name=INT, + // otherwise it is false when cleared by -name=false. + IsSet bool + + // Version tracks the desired schema version, as set by the -name=INT flag. + // The version defaults to 1 when implicitly set by -name or -name=true. + Version int +} + +// String returns the default value which is printed in the CLI help text. +func (v *SchemaVersion) String() string { + if v.IsSet { + return strconv.Itoa(v.Version) + } + return strconv.FormatBool(false) +} + +// Set is called when the user passes the flag as a command-line argument. +func (v *SchemaVersion) Set(s string) error { + // Delegate to a FlagSet to parse this as both a BoolVar and an IntVar. + // This is less efficient than copying the implementation from the standard library + // but this design makes it likelier that Set will inherit any upstream fixes. + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.BoolVar(&v.IsSet, "bool", false, "") + fs.IntVar(&v.Version, "int", 0, "") + fs.SetOutput(io.Discard) // silence + + // First, try to parse as an IntVar to handle -flag=INT. + // This order is important because -bool=0 will parse as false. + if err := fs.Parse([]string{"-int=" + s}); err == nil { + v.IsSet = true + return nil + } + // If that fails, parse as a BoolVar to handle -flag and -flag=false. + // This is checked last for compatibility with the boolean -json flag. + if err := fs.Parse([]string{"-bool=" + s}); err != nil { + // Unwrap the header added by FlagSet.failf: + // `invalid boolean value "invalid" for -bool: ` + bits := strings.SplitN(err.Error(), ": ", 2) + return errors.New(bits[len(bits)-1]) + } + // If the user doesn't supply a schema version, default to 1. + // This ensures that any existing scripts will continue to get their + // current output. + if v.IsSet { + v.Version = 1 + } else { + v.Version = 0 // if unset, zero out the Version + } + return nil +} + +// IsBoolFlag reports that this [flag.Value] can be set without an argument. +// This is the magic interface that makes -name equivalent to -name=true +// rather than using the next command-line argument. +func (v *SchemaVersion) IsBoolFlag() bool { + return true +} + +// ResponseEnvelope is a set of fields common to all versioned JSON output. +type ResponseEnvelope struct { + // SchemaVersion is the version of the JSON output, e.g. "1", "2", "3" + SchemaVersion string + + // ResponseWarning tells a user if a newer version of the JSON output + // is available. + ResponseWarning string `json:"_WARNING,omitzero"` +} diff --git a/cmd/tailscale/cli/jsonoutput/jsonoutput_test.go b/cmd/tailscale/cli/jsonoutput/jsonoutput_test.go new file mode 100644 index 0000000000000..595fe2a12faaf --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/jsonoutput_test.go @@ -0,0 +1,164 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package jsonoutput_test + +import ( + "flag" + "math" + "testing" + + gcmp "github.com/google/go-cmp/cmp" + "github.com/kballard/go-shellquote" + + "tailscale.com/cmd/tailscale/cli/jsonoutput" +) + +func TestSchemaVersion(t *testing.T) { + for _, tc := range []struct { + name string + args string + want jsonoutput.SchemaVersion + wantErr string + wantStr string + }{ + { + name: "none", + want: jsonoutput.SchemaVersion{IsSet: false, Version: 0}, + wantStr: "false", + }, + { + name: "default", + args: "-got", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 1}, + wantStr: "1", + }, + { + name: "true", + args: "-got=true", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 1}, + wantStr: "1", + }, + { + name: "false", + args: "-got=false", + want: jsonoutput.SchemaVersion{IsSet: false, Version: 0}, + wantStr: "false", + }, + { + // Test that -got=0 isn’t interpreted as -bool=0, i.e. false. + name: "zero_not_false", + args: "-got=0", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 0}, + wantStr: "0", + }, + { + name: "one", + args: "-got=1", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 1}, + wantStr: "1", + }, + { + name: "two", + args: "-got=2", + want: jsonoutput.SchemaVersion{IsSet: true, Version: 2}, + wantStr: "2", + }, + { + name: "max", + args: "-got=2147483647", + want: jsonoutput.SchemaVersion{IsSet: true, Version: math.MaxInt32}, + wantStr: "2147483647", + }, + { + name: "min", + args: "-got=-2147483648", + want: jsonoutput.SchemaVersion{IsSet: true, Version: math.MinInt32}, + wantStr: "-2147483648", + }, + { + name: "invalid", + args: "-got=invalid", + wantErr: `invalid boolean value "invalid" for -got: parse error`, + }, + { + name: "float", + args: "-got=1.3", + wantErr: `invalid boolean value "1.3" for -got: parse error`, + }, + { + name: "space", + args: "-got=' '", + wantErr: `invalid boolean value " " for -got: parse error`, + }, + { + name: "trailing_space", + args: "-got='1 '", + wantErr: `invalid boolean value "1 " for -got: parse error`, + }, + } { + args, err := shellquote.Split(tc.args) + if err != nil { + t.Fatalf("broken args %q: %v", tc.args, err) + } + + // Test both Set and String methods. + t.Run(tc.name, func(t *testing.T) { + var got jsonoutput.SchemaVersion + fs := flag.NewFlagSet("name", flag.ContinueOnError) + fs.Var(&got, "got", "usage") + + err = fs.Parse(args) + if err != nil && tc.wantErr == "" { + t.Errorf("parse error: %v", err) + } else if err != nil && err.Error() != tc.wantErr { + t.Errorf("parse error mismatch: %q, want %q", err, tc.wantErr) + } else if err == nil && tc.wantErr != "" { + t.Errorf("parse error: %v, want %q", err, tc.wantErr) + } + + if len(fs.Args()) != 0 { + t.Errorf("unexpected positional arguments: %q", fs.Args()) + } + + if diff := gcmp.Diff(tc.want, got); diff != "" { + t.Errorf("parse mismatch: -want +got\n%s", diff) + } + + if s := got.String(); s != tc.wantStr && tc.wantStr != "" { + t.Errorf("string %q, want %q", s, tc.wantStr) + } + }) + + if tc.args == "" { + continue // nothing to clobber + } + if tc.wantErr != "" { + continue // clobbering will just trigger another error + } + + // The last -got flag will clobber all previous -got flags. + t.Run(tc.name+"/clobber", func(t *testing.T) { + var got jsonoutput.SchemaVersion + fs := flag.NewFlagSet("name", flag.ContinueOnError) + fs.Var(&got, "got", "usage") + + sentinel := []string{"-got=-1"} + if err := fs.Parse(append(sentinel, args...)); err != nil { + t.Errorf("parse error: %v", err) + } + + if got.Version == -1 { + t.Errorf("sentinel detected: flag didn’t clobber") + } + + if len(fs.Args()) != 0 { + t.Errorf("unexpected positional arguments: %q", fs.Args()) + } + + if diff := gcmp.Diff(tc.want, got); diff != "" { + t.Errorf("parse mismatch: -want +got\n%s", diff) + } + }) + } +} diff --git a/cmd/tailscale/cli/jsonoutput/tailnet-lock-log.go b/cmd/tailscale/cli/jsonoutput/tailnet-lock-log.go new file mode 100644 index 0000000000000..97af3bed78be1 --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/tailnet-lock-log.go @@ -0,0 +1,209 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_tailnetlock + +package jsonoutput + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tka" +) + +// PrintTailnetLockLogJSONV1 prints the stored TKA state as a JSON object to the CLI, +// in a stable "v1" format. +// +// This format includes: +// +// - the AUM hash as a base32-encoded string +// - the raw AUM as base64-encoded bytes +// - the expanded AUM, which prints named fields for consumption by other tools +func PrintTailnetLockLogJSONV1(out io.Writer, updates []ipnstate.TailnetLockUpdate) error { + messages := make([]logMessageV1, len(updates)) + + for i, update := range updates { + var aum tka.AUM + if err := aum.Unserialize(update.Raw); err != nil { + return fmt.Errorf("decoding: %w", err) + } + + h := aum.Hash() + + if !bytes.Equal(h[:], update.Hash[:]) { + return fmt.Errorf("incorrect AUM hash: got %v, want %v", h, update) + } + + messages[i] = toLogMessageV1(aum, update) + } + + result := struct { + ResponseEnvelope + Messages []logMessageV1 + }{ + ResponseEnvelope: ResponseEnvelope{ + SchemaVersion: "1", + }, + Messages: messages, + } + + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(result) +} + +// toLogMessageV1 converts a [tka.AUM] and [ipnstate.TailnetLockUpdate] to the +// JSON output returned by the CLI. +func toLogMessageV1(aum tka.AUM, update ipnstate.TailnetLockUpdate) logMessageV1 { + expandedAUM := expandedAUMV1{} + expandedAUM.MessageKind = aum.MessageKind.String() + if len(aum.PrevAUMHash) > 0 { + expandedAUM.PrevAUMHash = aum.PrevAUMHash.String() + } + if key := aum.Key; key != nil { + expandedAUM.Key = toTKAKeyV1(key) + } + if keyID := aum.KeyID; keyID != nil { + expandedAUM.KeyID = fmt.Sprintf("tlpub:%x", keyID) + } + if state := aum.State; state != nil { + expandedState := expandedStateV1{} + if h := state.LastAUMHash; h != nil { + expandedState.LastAUMHash = h.String() + } + for _, secret := range state.DisablementValues { + expandedState.DisablementValues = append(expandedState.DisablementValues, fmt.Sprintf("%x", secret)) + } + for _, key := range state.Keys { + expandedState.Keys = append(expandedState.Keys, toTKAKeyV1(&key)) + } + expandedState.StateID1 = state.StateID1 + expandedState.StateID2 = state.StateID2 + expandedAUM.State = expandedState + } + if votes := aum.Votes; votes != nil { + expandedAUM.Votes = *votes + } + expandedAUM.Meta = aum.Meta + for _, signature := range aum.Signatures { + expandedAUM.Signatures = append(expandedAUM.Signatures, expandedSignatureV1{ + KeyID: fmt.Sprintf("tlpub:%x", signature.KeyID), + Signature: base64.URLEncoding.EncodeToString(signature.Signature), + }) + } + + return logMessageV1{ + Hash: aum.Hash().String(), + AUM: expandedAUM, + Raw: base64.URLEncoding.EncodeToString(update.Raw), + } +} + +// toTKAKeyV1 converts a [tka.Key] to the JSON output returned +// by the CLI. +func toTKAKeyV1(key *tka.Key) tkaKeyV1 { + return tkaKeyV1{ + Kind: key.Kind.String(), + Votes: key.Votes, + Public: fmt.Sprintf("tlpub:%x", key.Public), + Meta: key.Meta, + } +} + +// logMessageV1 is the JSON representation of an AUM as both raw bytes and +// in its expanded form, and the CLI output is a list of these entries. +type logMessageV1 struct { + // The BLAKE2s digest of the CBOR-encoded AUM. This is printed as a + // base32-encoded string, e.g. KCEâ€ĻXZQ + Hash string + + // The expanded form of the AUM, which presents the fields in a more + // accessible format than doing a CBOR decoding. + AUM expandedAUMV1 + + // The raw bytes of the CBOR-encoded AUM, encoded as base64. + // This is useful for verifying the AUM hash. + Raw string +} + +// expandedAUMV1 is the expanded version of a [tka.AUM], designed so external tools +// can read the AUM without knowing our CBOR definitions. +type expandedAUMV1 struct { + MessageKind string + PrevAUMHash string `json:"PrevAUMHash,omitzero"` + + // Key encodes a public key to be added to the key authority. + // This field is used for AddKey AUMs. + Key tkaKeyV1 `json:"Key,omitzero"` + + // KeyID references a public key which is part of the key authority. + // This field is used for RemoveKey and UpdateKey AUMs. + KeyID string `json:"KeyID,omitzero"` + + // State describes the full state of the key authority. + // This field is used for Checkpoint AUMs. + State expandedStateV1 `json:"State,omitzero"` + + // Votes and Meta describe properties of a key in the key authority. + // These fields are used for UpdateKey AUMs. + Votes uint `json:"Votes,omitzero"` + Meta map[string]string `json:"Meta,omitzero"` + + // Signatures lists the signatures over this AUM. + Signatures []expandedSignatureV1 `json:"Signatures,omitzero"` +} + +// tkaKeyV1 is the expanded version of a [tka.Key], which describes +// the public components of a key known to tailnet-lock. +type tkaKeyV1 struct { + Kind string `json:"Kind,omitzero"` + + // Votes describes the weight applied to signatures using this key. + Votes uint + + // Public encodes the public key of the key as a hex string. + Public string + + // Meta describes arbitrary metadata about the key. This could be + // used to store the name of the key, for instance. + Meta map[string]string `json:"Meta,omitzero"` +} + +// expandedStateV1 is the expanded version of a [tka.State], which describes +// Tailnet Key Authority state at an instant in time. +type expandedStateV1 struct { + // LastAUMHash is the blake2s digest of the last-applied AUM. + LastAUMHash string `json:"LastAUMHash,omitzero"` + + // DisablementValues are KDF-derived values used to verify that a caller + // possesses a valid DisablementSecret. These values are used during the + // Tailnet Lock deactivation process. + // + // These are safe to share publicly or store in the clear. They cannot be + // used to derive the original DisablementSecret. + DisablementValues []string + + // Keys are the public keys of either: + // + // 1. The signing nodes currently trusted by the TKA. + // 2. Ephemeral keys that were used to generate pre-signed auth keys. + Keys []tkaKeyV1 + + // StateID's are nonce's, generated on enablement and fixed for + // the lifetime of the Tailnet Key Authority. + StateID1 uint64 + StateID2 uint64 +} + +// expandedSignatureV1 is the expanded form of a [tka.Signature], which +// describes a signature over an AUM. This signature can be verified +// using the key referenced by KeyID. +type expandedSignatureV1 struct { + KeyID string + Signature string +} diff --git a/cmd/tailscale/cli/jsonoutput/tailnet-lock-status.go b/cmd/tailscale/cli/jsonoutput/tailnet-lock-status.go new file mode 100644 index 0000000000000..b3bbd30868614 --- /dev/null +++ b/cmd/tailscale/cli/jsonoutput/tailnet-lock-status.go @@ -0,0 +1,249 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_tailnetlock + +package jsonoutput + +import ( + "encoding/base64" + jsonv1 "encoding/json" + "fmt" + "io" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tka" +) + +// PrintTailnetLockStatusJSONV1 prints the current Tailnet Lock status +// as a JSON object to the CLI, in a stable "v1" format. +func PrintTailnetLockStatusJSONV1(out io.Writer, status *ipnstate.TailnetLockStatus) error { + responseEnvelope := ResponseEnvelope{ + SchemaVersion: "1", + } + + var result any + if status.Enabled { + result = struct { + ResponseEnvelope + tailnetLockEnabledStatusV1 + }{ + ResponseEnvelope: responseEnvelope, + tailnetLockEnabledStatusV1: toTailnetLockEnabledStatusV1(status), + } + } else { + result = struct { + ResponseEnvelope + tailnetLockDisabledStatusV1 + }{ + ResponseEnvelope: responseEnvelope, + tailnetLockDisabledStatusV1: toTailnetLockDisabledStatusV1(status), + } + } + + enc := jsonv1.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(result) +} + +func toTailnetLockDisabledStatusV1(status *ipnstate.TailnetLockStatus) tailnetLockDisabledStatusV1 { + out := tailnetLockDisabledStatusV1{ + tailnetLockStatusV1Base: tailnetLockStatusV1Base{ + Enabled: status.Enabled, + }, + } + if !status.PublicKey.IsZero() { + out.PublicKey = status.PublicKey.CLIString() + } + if nk := status.NodeKey; nk != nil { + out.NodeKey = nk.String() + } + return out +} + +func toTailnetLockEnabledStatusV1(status *ipnstate.TailnetLockStatus) tailnetLockEnabledStatusV1 { + out := tailnetLockEnabledStatusV1{ + tailnetLockStatusV1Base: tailnetLockStatusV1Base{ + Enabled: status.Enabled, + }, + } + + if status.Head != nil { + var head tka.AUMHash + h := status.Head + copy(head[:], h[:]) + out.Head = head.String() + } + if !status.PublicKey.IsZero() { + out.PublicKey = status.PublicKey.CLIString() + } + if nk := status.NodeKey; nk != nil { + out.NodeKey = nk.String() + } + out.NodeKeySigned = status.NodeKeySigned + if sig := status.NodeKeySignature; sig != nil { + out.NodeKeySignature = toTKANodeKeySignatureV1(sig) + } + for _, key := range status.TrustedKeys { + out.TrustedKeys = append(out.TrustedKeys, ipnTKAKeytoTKAKeyV1(&key)) + } + for _, vp := range status.VisiblePeers { + out.VisiblePeers = append(out.VisiblePeers, tkaTrustedPeerV1{ + tkaPeerV1: toTKAPeerV1(vp), + NodeKeySignature: toTKANodeKeySignatureV1(&vp.NodeKeySignature), + }) + } + for _, fp := range status.FilteredPeers { + out.FilteredPeers = append(out.FilteredPeers, toTKAPeerV1(fp)) + } + out.StateID = status.StateID + + return out +} + +// toTKAKeyV1 converts an [ipnstate.TKAKey] to the JSON output returned +// by the CLI. +func ipnTKAKeytoTKAKeyV1(key *ipnstate.TKAKey) tkaKeyV1 { + return tkaKeyV1{ + Kind: key.Kind, + Votes: key.Votes, + Public: key.Key.CLIString(), + Meta: key.Metadata, + } +} + +type tailnetLockStatusV1Base struct { + // Enabled is true if Tailnet Lock is enabled. + Enabled bool + + // PublicKey describes the node's tailnet-lock public key. + PublicKey string `json:"PublicKey,omitzero"` + + // NodeKey describes the node's current node-key. This field is not + // populated if the node is not operating (i.e. waiting for a login). + NodeKey string `json:"NodeKey,omitzero"` +} + +// tailnetLockDisabledStatusV1 is the JSON representation of the Tailnet Lock status +// when Tailnet Lock is disabled. +type tailnetLockDisabledStatusV1 struct { + tailnetLockStatusV1Base +} + +// tailnetLockEnabledStatusV1 is the JSON representation of the Tailnet Lock status. +type tailnetLockEnabledStatusV1 struct { + tailnetLockStatusV1Base + + // Head describes the AUM hash of the leaf AUM. + Head string `json:"Head,omitzero"` + + // NodeKeySigned is true if our node is authorized by Tailnet Lock. + NodeKeySigned bool + + // NodeKeySignature is the current signature of this node's key. + NodeKeySignature *tkaNodeKeySignatureV1 + + // TrustedKeys describes the keys currently trusted to make changes + // to tailnet-lock. + TrustedKeys []tkaKeyV1 + + // VisiblePeers describes peers which are visible in the netmap that + // have valid Tailnet Lock signatures signatures. + VisiblePeers []tkaTrustedPeerV1 + + // FilteredPeers describes peers which were removed from the netmap + // (i.e. no connectivity) because they failed Tailnet Lock + // checks. + FilteredPeers []tkaPeerV1 + + // StateID is a nonce associated with the Tailnet Lock authority, + // generated upon enablement. This field is empty if Tailnet Lock + // is disabled. + StateID uint64 `json:"State,omitzero"` +} + +// tkaPeerV1 is the JSON representation of an [ipnstate.TKAPeer], which describes +// a peer and its Tailnet Lock details. +type tkaPeerV1 struct { + // Stable ID, i.e. [tailcfg.StableNodeID] + ID string + + // DNS name + DNSName string + + // Tailscale IP(s) assigned to this node + TailscaleIPs []string + + // The node's public key + NodeKey string +} + +// tkaPeerV1 is the JSON representation of a trusted [ipnstate.TKAPeer], which +// has a node key signature. +type tkaTrustedPeerV1 struct { + tkaPeerV1 + + // The node's key signature + NodeKeySignature *tkaNodeKeySignatureV1 `json:"NodeKeySignature,omitzero"` +} + +func toTKAPeerV1(peer *ipnstate.TKAPeer) tkaPeerV1 { + out := tkaPeerV1{ + DNSName: peer.Name, + ID: string(peer.StableID), + } + for _, ip := range peer.TailscaleIPs { + out.TailscaleIPs = append(out.TailscaleIPs, ip.String()) + } + out.NodeKey = peer.NodeKey.String() + + return out +} + +// tkaNodeKeySignatureV1 is the JSON representation of a [tka.NodeKeySignature], +// which describes a signature that authorizes a specific node key. +type tkaNodeKeySignatureV1 struct { + // SigKind identifies the variety of signature. + SigKind string + + // PublicKey identifies the key.NodePublic which is being authorized. + // SigCredential signatures do not use this field. + PublicKey string `json:"PublicKey,omitzero"` + + // KeyID identifies which key in the tailnet key authority should + // be used to verify this signature. Only set for SigDirect and + // SigCredential signature kinds. + KeyID string `json:"KeyID,omitzero"` + + // Signature is the packed (R, S) ed25519 signature over all other + // fields of the structure. + Signature string + + // Nested describes a NodeKeySignature which authorizes the node-key + // used as Pubkey. Only used for SigRotation signatures. + Nested *tkaNodeKeySignatureV1 `json:"Nested,omitzero"` + + // WrappingPubkey specifies the ed25519 public key which must be used + // to sign a Signature which embeds this one. + WrappingPublicKey string `json:"WrappingPublicKey,omitzero"` +} + +func toTKANodeKeySignatureV1(sig *tka.NodeKeySignature) *tkaNodeKeySignatureV1 { + out := tkaNodeKeySignatureV1{ + SigKind: sig.SigKind.String(), + } + if len(sig.Pubkey) > 0 { + out.PublicKey = fmt.Sprintf("tlpub:%x", sig.Pubkey) + } + if len(sig.KeyID) > 0 { + out.KeyID = fmt.Sprintf("tlpub:%x", sig.KeyID) + } + out.Signature = base64.URLEncoding.EncodeToString(sig.Signature) + if sig.Nested != nil { + out.Nested = toTKANodeKeySignatureV1(sig.Nested) + } + if len(sig.WrappingPubkey) > 0 { + out.WrappingPublicKey = fmt.Sprintf("tlpub:%x", sig.WrappingPubkey) + } + return &out +} diff --git a/cmd/tailscale/cli/licenses.go b/cmd/tailscale/cli/licenses.go index bede827edf693..35d636aa2eb84 100644 --- a/cmd/tailscale/cli/licenses.go +++ b/cmd/tailscale/cli/licenses.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/login.go b/cmd/tailscale/cli/login.go index fb5b786920660..bdf97c70f8e1d 100644 --- a/cmd/tailscale/cli/login.go +++ b/cmd/tailscale/cli/login.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/logout.go b/cmd/tailscale/cli/logout.go index 0c2007a66ab1b..90843edc2e299 100644 --- a/cmd/tailscale/cli/logout.go +++ b/cmd/tailscale/cli/logout.go @@ -1,16 +1,22 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli import ( "context" + "flag" "fmt" "strings" "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/tailscale/apitype" ) +var logoutArgs struct { + reason string +} + var logoutCmd = &ffcli.Command{ Name: "logout", ShortUsage: "tailscale logout", @@ -22,11 +28,17 @@ the current node key, forcing a future use of it to cause a reauthentication. `), Exec: runLogout, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("logout") + fs.StringVar(&logoutArgs.reason, "reason", "", "reason for the logout, if required by a policy") + return fs + })(), } func runLogout(ctx context.Context, args []string) error { if len(args) > 0 { return fmt.Errorf("too many non-flag arguments: %q", args) } + ctx = apitype.RequestReasonKey.WithValue(ctx, logoutArgs.reason) return localClient.Logout(ctx) } diff --git a/cmd/tailscale/cli/maybe_syspolicy.go b/cmd/tailscale/cli/maybe_syspolicy.go new file mode 100644 index 0000000000000..a66c1a65df5e4 --- /dev/null +++ b/cmd/tailscale/cli/maybe_syspolicy.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_syspolicy + +package cli + +import _ "tailscale.com/feature/syspolicy" diff --git a/cmd/tailscale/cli/metrics.go b/cmd/tailscale/cli/metrics.go index dbdedd5a61037..d16ce76d2725f 100644 --- a/cmd/tailscale/cli/metrics.go +++ b/cmd/tailscale/cli/metrics.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/nc.go b/cmd/tailscale/cli/nc.go index 4ea62255412ea..34490ec212557 100644 --- a/cmd/tailscale/cli/nc.go +++ b/cmd/tailscale/cli/nc.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/netcheck.go b/cmd/tailscale/cli/netcheck.go index 3cf05a3b7987f..52d66d5160189 100644 --- a/cmd/tailscale/cli/netcheck.go +++ b/cmd/tailscale/cli/netcheck.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -10,21 +10,34 @@ import ( "fmt" "io" "log" + "math" "net/http" + "net/netip" "sort" "strings" "time" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/envknob" + "tailscale.com/feature/buildfeatures" "tailscale.com/ipn" "tailscale.com/net/netcheck" "tailscale.com/net/netmon" - "tailscale.com/net/portmapper" + "tailscale.com/net/portmapper/portmappertype" "tailscale.com/net/tlsdial" "tailscale.com/tailcfg" + "tailscale.com/tstime" "tailscale.com/types/logger" "tailscale.com/util/eventbus" + "tailscale.com/util/set" + + // The "netcheck" command also wants the portmapper linked. + // + // TODO: make that subcommand either hit LocalAPI for that info, or use a + // tailscaled subcommand, to avoid making the CLI also link in the portmapper. + // For now (2025-09-15), keep doing what we've done for the past five years and + // keep linking it here. + _ "tailscale.com/feature/condregister/portmapper" ) var netcheckCmd = &ffcli.Command{ @@ -32,19 +45,25 @@ var netcheckCmd = &ffcli.Command{ ShortUsage: "tailscale netcheck", ShortHelp: "Print an analysis of local network conditions", Exec: runNetcheck, - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("netcheck") - fs.StringVar(&netcheckArgs.format, "format", "", `output format; empty (for human-readable), "json" or "json-line"`) - fs.DurationVar(&netcheckArgs.every, "every", 0, "if non-zero, do an incremental report with the given frequency") - fs.BoolVar(&netcheckArgs.verbose, "verbose", false, "verbose logs") - return fs - })(), + FlagSet: netcheckFlagSet, } +var netcheckFlagSet = func() *flag.FlagSet { + fs := newFlagSet("netcheck") + fs.StringVar(&netcheckArgs.format, "format", "", `output format; empty (for human-readable), "json" or "json-line"`) + fs.DurationVar(&netcheckArgs.every, "every", 0, "if non-zero, do an incremental report with the given frequency") + fs.BoolVar(&netcheckArgs.verbose, "verbose", false, "verbose logs") + fs.StringVar(&netcheckArgs.bindAddress, "bind-address", "", "send and receive connectivity probes using this locally bound IP address; default: OS-assigned") + fs.IntVar(&netcheckArgs.bindPort, "bind-port", 0, "send and receive connectivity probes using this UDP port; default: OS-assigned") + return fs +}() + var netcheckArgs struct { - format string - every time.Duration - verbose bool + format string + every time.Duration + verbose bool + bindAddress string + bindPort int } func runNetcheck(ctx context.Context, args []string) error { @@ -56,13 +75,18 @@ func runNetcheck(ctx context.Context, args []string) error { return err } - // Ensure that we close the portmapper after running a netcheck; this - // will release any port mappings created. - pm := portmapper.NewClient(portmapper.Config{ - Logf: logf, - NetMon: netMon, + var pm portmappertype.Client + if buildfeatures.HasPortMapper { + // Ensure that we close the portmapper after running a netcheck; this + // will release any port mappings created. + pm = portmappertype.HookNewPortMapper.Get()(logf, bus, netMon, nil, nil) + defer pm.Close() + } + + flagsProvided := set.Set[string]{} + netcheckFlagSet.Visit(func(f *flag.Flag) { + flagsProvided.Add(f.Name) }) - defer pm.Close() c := &netcheck.Client{ NetMon: netMon, @@ -80,7 +104,17 @@ func runNetcheck(ctx context.Context, args []string) error { fmt.Fprintln(Stderr, "# Warning: this JSON format is not yet considered a stable interface") } - if err := c.Standalone(ctx, envknob.String("TS_DEBUG_NETCHECK_UDP_BIND")); err != nil { + bind, err := createNetcheckBindString( + netcheckArgs.bindAddress, + flagsProvided.Contains("bind-address"), + netcheckArgs.bindPort, + flagsProvided.Contains("bind-port"), + envknob.String("TS_DEBUG_NETCHECK_UDP_BIND")) + if err != nil { + return err + } + + if err := c.Standalone(ctx, bind); err != nil { fmt.Fprintln(Stderr, "netcheck: UDP test failure:", err) } @@ -110,7 +144,7 @@ func runNetcheck(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("netcheck: %w", err) } - if err := printReport(dm, report); err != nil { + if err := printNetCheckReport(dm, report); err != nil { return err } if netcheckArgs.every == 0 { @@ -120,7 +154,7 @@ func runNetcheck(ctx context.Context, args []string) error { } } -func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { +func printNetCheckReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { var j []byte var err error switch netcheckArgs.format { @@ -142,7 +176,7 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { } printf("\nReport:\n") - printf("\t* Time: %v\n", report.Now.Format(time.RFC3339Nano)) + printf("\t* Time: %v\n", report.Now.Local().Format(tstime.DateSpTimeNanoZ)) printf("\t* UDP: %v\n", report.UDP) if report.GlobalV4.IsValid() { printf("\t* IPv4: yes, %s\n", report.GlobalV4) @@ -171,7 +205,11 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { printf("\t* Nearest DERP: unknown (no response to latency probes)\n") } else { if report.PreferredDERP != 0 { - printf("\t* Nearest DERP: %v\n", dm.Regions[report.PreferredDERP].RegionName) + if region, ok := dm.Regions[report.PreferredDERP]; ok { + printf("\t* Nearest DERP: %v\n", region.RegionName) + } else { + printf("\t* Nearest DERP: %v (region not found in map)\n", report.PreferredDERP) + } } else { printf("\t* Nearest DERP: [none]\n") } @@ -209,6 +247,9 @@ func printReport(dm *tailcfg.DERPMap, report *netcheck.Report) error { } func portMapping(r *netcheck.Report) string { + if !buildfeatures.HasPortMapper { + return "binary built without portmapper support" + } if !r.AnyPortMappingChecked() { return "not checked" } @@ -249,3 +290,44 @@ func prodDERPMap(ctx context.Context, httpc *http.Client) (*tailcfg.DERPMap, err } return &derpMap, nil } + +// createNetcheckBindString determines the netcheck socket bind "address:port" string based +// on the CLI args and environment variable values used to invoke the netcheck CLI. +// Arguments cliAddressIsSet and cliPortIsSet explicitly indicate whether the +// corresponding cliAddress and cliPort were set in CLI args, instead of relying +// on in-band sentinel values. +func createNetcheckBindString(cliAddress string, cliAddressIsSet bool, cliPort int, cliPortIsSet bool, envBind string) (string, error) { + // Default to port number 0 but overwrite with a valid CLI value, if set. + var port uint16 = 0 + if cliPortIsSet { + // 0 is valid, results in OS picking port. + if cliPort >= 0 && cliPort <= math.MaxUint16 { + port = uint16(cliPort) + } else { + return "", fmt.Errorf("invalid bind port number: %d", cliPort) + } + } + + // Use CLI address, if set. + if cliAddressIsSet { + addr, err := netip.ParseAddr(cliAddress) + if err != nil { + return "", fmt.Errorf("invalid bind address: %q", cliAddress) + } + return netip.AddrPortFrom(addr, port).String(), nil + } else { + // No CLI address set, but port is set. + if cliPortIsSet { + return fmt.Sprintf(":%d", port), nil + } + } + + // Fall back to the environment variable. + // Intentionally skipping input validation here to avoid breaking legacy usage method. + if envBind != "" { + return envBind, nil + } + + // OS picks both address and port. + return ":0", nil +} diff --git a/cmd/tailscale/cli/netcheck_test.go b/cmd/tailscale/cli/netcheck_test.go new file mode 100644 index 0000000000000..b2c2bceb39dc9 --- /dev/null +++ b/cmd/tailscale/cli/netcheck_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "testing" +) + +func TestCreateBindStr(t *testing.T) { + // Test all combinations of CLI arg address, CLI arg port, and env var string + // as inputs to create netcheck bind string. + tests := []struct { + name string + cliAddress string + cliAddressIsSet bool + cliPort int + cliPortIsSet bool + envBind string + want string + wantError string + }{ + { + name: "noAddr-noPort-noEnv", + want: ":0", + }, + { + name: "yesAddrv4-noPort-noEnv", + cliAddress: "100.123.123.123", + cliAddressIsSet: true, + want: "100.123.123.123:0", + }, + { + name: "yesAddrv6-noPort-noEnv", + cliAddress: "dead::beef", + cliAddressIsSet: true, + want: "[dead::beef]:0", + }, + { + name: "yesAddr-yesPort-noEnv", + cliAddress: "100.123.123.123", + cliAddressIsSet: true, + cliPort: 456, + cliPortIsSet: true, + want: "100.123.123.123:456", + }, + { + name: "yesAddr-yesPort-yesEnv", + cliAddress: "100.123.123.123", + cliAddressIsSet: true, + cliPort: 456, + cliPortIsSet: true, + envBind: "55.55.55.55:789", + want: "100.123.123.123:456", + }, + { + name: "noAddr-yesPort-noEnv", + cliPort: 456, + cliPortIsSet: true, + want: ":456", + }, + { + name: "noAddr-yesPort-yesEnv", + cliPort: 456, + cliPortIsSet: true, + envBind: "55.55.55.55:789", + want: ":456", + }, + { + name: "noAddr-noPort-yesEnv", + envBind: "55.55.55.55:789", + want: "55.55.55.55:789", + }, + { + name: "badAddr-noPort-noEnv-1", + cliAddress: "678.678.678.678", + cliAddressIsSet: true, + wantError: `invalid bind address: "678.678.678.678"`, + }, + { + name: "badAddr-noPort-noEnv-2", + cliAddress: "lorem ipsum", + cliAddressIsSet: true, + wantError: `invalid bind address: "lorem ipsum"`, + }, + { + name: "noAddr-badPort-noEnv", + cliPort: -1, + cliPortIsSet: true, + wantError: "invalid bind port number: -1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := createNetcheckBindString(tt.cliAddress, tt.cliAddressIsSet, tt.cliPort, tt.cliPortIsSet, tt.envBind) + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + if gotErrStr != tt.wantError { + t.Errorf("got error %q; want error %q", gotErrStr, tt.wantError) + } + if got != tt.want { + t.Errorf("got result %q; want result %q", got, tt.want) + } + }) + } +} diff --git a/cmd/tailscale/cli/open_browser.go b/cmd/tailscale/cli/open_browser.go new file mode 100644 index 0000000000000..a006b9765da7a --- /dev/null +++ b/cmd/tailscale/cli/open_browser.go @@ -0,0 +1,12 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_webbrowser + +package cli + +import "github.com/toqueteos/webbrowser" + +func init() { + hookOpenURL.Set(webbrowser.Open) +} diff --git a/cmd/tailscale/cli/ping.go b/cmd/tailscale/cli/ping.go index 3a909f30dee86..1e8bbd23f15e8 100644 --- a/cmd/tailscale/cli/ping.go +++ b/cmd/tailscale/cli/ping.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -16,7 +16,7 @@ import ( "time" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/client/tailscale" + "tailscale.com/client/local" "tailscale.com/cmd/tailscale/cli/ffcomplete" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" @@ -128,7 +128,7 @@ func runPing(ctx context.Context, args []string) error { for { n++ ctx, cancel := context.WithTimeout(ctx, pingArgs.timeout) - pr, err := localClient.PingWithOpts(ctx, netip.MustParseAddr(ip), pingType(), tailscale.PingOpts{Size: pingArgs.size}) + pr, err := localClient.PingWithOpts(ctx, netip.MustParseAddr(ip), pingType(), local.PingOpts{Size: pingArgs.size}) cancel() if err != nil { if errors.Is(err, context.DeadlineExceeded) { @@ -152,7 +152,9 @@ func runPing(ctx context.Context, args []string) error { } latency := time.Duration(pr.LatencySeconds * float64(time.Second)).Round(time.Millisecond) via := pr.Endpoint - if pr.DERPRegionID != 0 { + if pr.PeerRelay != "" { + via = fmt.Sprintf("peer-relay(%s)", pr.PeerRelay) + } else if pr.DERPRegionID != 0 { via = fmt.Sprintf("DERP(%s)", pr.DERPRegionCode) } if via == "" { diff --git a/cmd/tailscale/cli/risks.go b/cmd/tailscale/cli/risks.go index 9b03025a83a0b..6f3ebf37bbebe 100644 --- a/cmd/tailscale/cli/risks.go +++ b/cmd/tailscale/cli/risks.go @@ -1,21 +1,14 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli import ( - "context" "errors" "flag" - "fmt" - "os" - "os/signal" - "runtime" "strings" - "syscall" - "time" - "tailscale.com/ipn" + "tailscale.com/util/prompt" "tailscale.com/util/testenv" ) @@ -23,7 +16,6 @@ var ( riskTypes []string riskLoseSSH = registerRiskType("lose-ssh") riskMacAppConnector = registerRiskType("mac-app-connector") - riskStrictRPFilter = registerRiskType("linux-strict-rp-filter") riskAll = registerRiskType("all") ) @@ -47,7 +39,7 @@ func registerAcceptRiskFlag(f *flag.FlagSet, acceptedRisks *string) { // isRiskAccepted reports whether riskType is in the comma-separated list of // risks in acceptedRisks. func isRiskAccepted(riskType, acceptedRisks string) bool { - for _, r := range strings.Split(acceptedRisks, ",") { + for r := range strings.SplitSeq(acceptedRisks, ",") { if r == riskType || r == riskAll { return true } @@ -57,11 +49,6 @@ func isRiskAccepted(riskType, acceptedRisks string) bool { var errAborted = errors.New("aborted, no changes made") -// riskAbortTimeSeconds is the number of seconds to wait after displaying the -// risk message before continuing with the operation. -// It is used by the presentRiskToUser function below. -const riskAbortTimeSeconds = 5 - // presentRiskToUser displays the risk message and waits for the user to cancel. // It returns errorAborted if the user aborts. In tests it returns errAborted // immediately unless the risk has been explicitly accepted. @@ -75,36 +62,9 @@ func presentRiskToUser(riskType, riskMessage, acceptedRisks string) error { outln(riskMessage) printf("To skip this warning, use --accept-risk=%s\n", riskType) - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, syscall.SIGINT) - var msgLen int - for left := riskAbortTimeSeconds; left > 0; left-- { - msg := fmt.Sprintf("\rContinuing in %d seconds...", left) - msgLen = len(msg) - printf("%s", msg) - select { - case <-interrupt: - printf("\r%s\r", strings.Repeat("x", msgLen+1)) - return errAborted - case <-time.After(time.Second): - continue - } - } - printf("\r%s\r", strings.Repeat(" ", msgLen)) - return errAborted -} - -// checkExitNodeRisk checks if the user is using an exit node on Linux and -// whether reverse path filtering is enabled. If so, it presents a risk message. -func checkExitNodeRisk(ctx context.Context, prefs *ipn.Prefs, acceptedRisks string) error { - if runtime.GOOS != "linux" { + if prompt.YesNo("Continue?", false) { return nil } - if !prefs.ExitNodeIP.IsValid() && prefs.ExitNodeID == "" { - return nil - } - if err := localClient.CheckReversePathFiltering(ctx); err != nil { - return presentRiskToUser(riskStrictRPFilter, err.Error(), acceptedRisks) - } - return nil + + return errAborted } diff --git a/cmd/tailscale/cli/routecheck.go b/cmd/tailscale/cli/routecheck.go new file mode 100644 index 0000000000000..679c174be6c5c --- /dev/null +++ b/cmd/tailscale/cli/routecheck.go @@ -0,0 +1,123 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_routecheck + +package cli + +import ( + "cmp" + "context" + "flag" + "fmt" + "slices" + "strings" + "text/tabwriter" + "time" + + jsonv2 "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + "github.com/peterbourgon/ff/v3/ffcli" + + "tailscale.com/net/routecheck" + "tailscale.com/tstime" +) + +func init() { + maybeRoutecheckCmd = routecheckCmd +} + +var routecheckCmd = func() *ffcli.Command { + return &ffcli.Command{ + Name: "routecheck", + ShortUsage: "tailscale routecheck", + ShortHelp: "Print a reachability report for routes with multiple paths", + LongHelp: hidden + `"tailscale routecheck" is an experimental feature; it is not a stable interface`, + Exec: runRoutecheck, + FlagSet: routecheckFlagSet, + } +} + +var routecheckFlagSet = func() *flag.FlagSet { + fs := newFlagSet("routecheck") + fs.BoolVar(&routecheckArgs.probe, "probe", false, "probe now to generate a new reachability report") + fs.StringVar(&routecheckArgs.format, "format", "", `output format: empty (for human-readable), "json" or "json-line"`) + return fs +}() + +var routecheckArgs struct { + probe bool + format string +} + +func runRoutecheck(ctx context.Context, args []string) error { + routeCheck := localClient.RouteCheck + if routecheckArgs.probe { + routeCheck = localClient.RouteCheckProbe + } + rp, err := routeCheck(ctx) + if err != nil { + return fmt.Errorf("routecheck: %w", err) + } + if err := printRouteCheckReport(rp); err != nil { + return err + } + return nil +} + +func printRouteCheckReport(rp *routecheck.Report) error { + var enc *jsontext.Encoder + switch routecheckArgs.format { + case "": + case "json": + enc = jsontext.NewEncoder(Stdout, jsontext.WithIndent("\t")) + case "json-line": + enc = jsontext.NewEncoder(Stdout, jsontext.Multiline(false)) + default: + return fmt.Errorf("unknown output format %q", routecheckArgs.format) + } + + if rp == nil { + return fmt.Errorf("routecheck: report unavailable") + } + routes := rp.RoutablePrefixes() + + // Don’t render prefixes that only have one router: + for pfx, nodes := range routes { + if len(nodes) <= 1 { + delete(routes, pfx) + } + } + + if enc != nil { + out := struct { + Done time.Time `json:"done"` + Routes routecheck.RoutablePrefixes `json:"routes"` + }{ + Done: rp.Done, + Routes: routes, + } + if err := jsonv2.MarshalEncode(enc, out); err != nil { + return err + } + if _, err := Stdout.Write([]byte("\n")); err != nil { + return err + } + return nil + } + + w := tabwriter.NewWriter(Stdout, 10, 5, 5, ' ', 0) + defer w.Flush() + fmt.Fprintf(w, "\nReachable routers at %s:\n", rp.Done.Local().Format(tstime.DateSpTimeZ)) + fmt.Fprintf(w, "\n %s\t%s\t%s", "PREFIX", "IP", "HOSTNAME") + for prefix, nodes := range routes.Sorted() { + slices.SortFunc(nodes, func(a, b routecheck.Node) int { + return cmp.Compare(a.Name, b.Name) // order by hostname + }) + for _, n := range nodes { + fmt.Fprintf(w, "\n %s\t%s\t%s", prefix, n.Addr, strings.TrimSuffix(n.Name, ".")) + } + } + fmt.Fprintln(w) + return nil +} diff --git a/cmd/tailscale/cli/serve_legacy.go b/cmd/tailscale/cli/serve_legacy.go index 96629b5ad45ef..a8a248e1e6cb0 100644 --- a/cmd/tailscale/cli/serve_legacy.go +++ b/cmd/tailscale/cli/serve_legacy.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_serve + package cli import ( @@ -23,7 +25,7 @@ import ( "strings" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/client/tailscale" + "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" @@ -31,10 +33,14 @@ import ( "tailscale.com/version" ) +func init() { + maybeServeCmd = serveCmd +} + var serveCmd = func() *ffcli.Command { se := &serveEnv{lc: &localClient} // previously used to serve legacy newFunnelCommand unless useWIPCode is true - // change is limited to make a revert easier and full cleanup to come after the relase. + // change is limited to make a revert easier and full cleanup to come after the release. // TODO(tylersmalley): cleanup and removal of newServeLegacyCommand as of 2023-10-16 return newServeV2Command(se, serve) } @@ -139,8 +145,11 @@ type localServeClient interface { GetServeConfig(context.Context) (*ipn.ServeConfig, error) SetServeConfig(context.Context, *ipn.ServeConfig) error QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error) - WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*tailscale.IPNBusWatcher, error) + WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*local.IPNBusWatcher, error) IncrementCounter(ctx context.Context, name string, delta int) error + GetPrefs(ctx context.Context) (*ipn.Prefs, error) + EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) + CheckSOMarkInUse(ctx context.Context) (bool, error) } // serveEnv is the environment the serve command runs within. All I/O should be @@ -154,17 +163,21 @@ type serveEnv struct { json bool // output JSON (status only for now) // v2 specific flags - bg bool // background mode - setPath string // serve path - https uint // HTTP port - http uint // HTTP port - tcp uint // TCP port - tlsTerminatedTCP uint // a TLS terminated TCP port - subcmd serveMode // subcommand - yes bool // update without prompt + bg bgBoolFlag // background mode + setPath string // serve path + https uint // HTTP port + http uint // HTTP port + tcp uint // TCP port + tlsTerminatedTCP uint // a TLS terminated TCP port + proxyProtocol uint // PROXY protocol version (1 or 2) + subcmd serveMode // subcommand + yes bool // update without prompt + service tailcfg.ServiceName // service name + tun bool // redirect traffic to OS for service + allServices bool // apply config file to all services + acceptAppCaps []tailcfg.PeerCapability // app capabilities to forward lc localServeClient // localClient interface, specific to serve - // optional stuff for tests: testFlagOut io.Writer testStdout io.Writer @@ -354,12 +367,12 @@ func (e *serveEnv) handleWebServe(ctx context.Context, srvPort uint16, useTLS bo if err != nil { return err } - if sc.IsTCPForwardingOnPort(srvPort) { + if sc.IsTCPForwardingOnPort(srvPort, noService) { fmt.Fprintf(Stderr, "error: cannot serve web; already serving TCP\n") return errHelp } - sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS) + sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS, noService.String()) if !reflect.DeepEqual(cursc, sc) { if err := e.lc.SetServeConfig(ctx, sc); err != nil { @@ -411,11 +424,11 @@ func (e *serveEnv) handleWebServeRemove(ctx context.Context, srvPort uint16, mou if err != nil { return err } - if sc.IsTCPForwardingOnPort(srvPort) { + if sc.IsTCPForwardingOnPort(srvPort, noService) { return errors.New("cannot remove web handler; currently serving TCP") } hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort)))) - if !sc.WebHandlerExists(hp, mount) { + if !sc.WebHandlerExists(noService, hp, mount) { return errors.New("error: handler does not exist") } sc.RemoveWebHandler(dnsName, srvPort, []string{mount}, false) @@ -550,16 +563,16 @@ func (e *serveEnv) handleTCPServe(ctx context.Context, srcType string, srcPort u fwdAddr := "127.0.0.1:" + dstPortStr - if sc.IsServingWeb(srcPort) { - return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) - } - dnsName, err := e.getSelfDNSName(ctx) if err != nil { return err } - sc.SetTCPForwarding(srcPort, fwdAddr, terminateTLS, dnsName) + if sc.IsServingWeb(srcPort, noService) { + return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) + } + + sc.SetTCPForwarding(srcPort, fwdAddr, terminateTLS, 0 /* proxy proto */, dnsName) if !reflect.DeepEqual(cursc, sc) { if err := e.lc.SetServeConfig(ctx, sc); err != nil { @@ -581,11 +594,11 @@ func (e *serveEnv) handleTCPServeRemove(ctx context.Context, src uint16) error { if sc == nil { sc = new(ipn.ServeConfig) } - if sc.IsServingWeb(src) { + if sc.IsServingWeb(src, noService) { return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", src) } - if ph := sc.GetTCPPortHandler(src); ph != nil { - sc.RemoveTCPForwarding(src) + if ph := sc.GetTCPPortHandler(src, noService); ph != nil { + sc.RemoveTCPForwarding(noService, src) return e.lc.SetServeConfig(ctx, sc) } return errors.New("error: serve config does not exist") @@ -615,7 +628,7 @@ func (e *serveEnv) runServeStatus(ctx context.Context, args []string) error { return nil } printFunnelStatus(ctx) - if sc == nil || (len(sc.TCP) == 0 && len(sc.Web) == 0 && len(sc.AllowFunnel) == 0) { + if isServeConfigEmpty(sc) { printf("No serve config\n") return nil } @@ -623,18 +636,8 @@ func (e *serveEnv) runServeStatus(ctx context.Context, args []string) error { if err != nil { return err } - if sc.IsTCPForwardingAny() { - if err := printTCPStatusTree(ctx, sc, st); err != nil { - return err - } - printf("\n") - } - for hp := range sc.Web { - err := e.printWebStatusTree(sc, hp) - if err != nil { - return err - } - printf("\n") + if err := printServeStatusTrees(sc, st); err != nil { + return err } printFunnelWarning(sc) return nil @@ -647,15 +650,15 @@ func printTCPStatusTree(ctx context.Context, sc *ipn.ServeConfig, st *ipnstate.S continue } hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(p)))) - tlsStatus := "TLS over TCP" - if h.TerminateTLS != "" { - tlsStatus = "TLS terminated" - } fStatus := "tailnet only" if sc.AllowFunnel[hp] { fStatus = "Funnel on" } - printf("|-- tcp://%s (%s, %s)\n", hp, tlsStatus, fStatus) + if h.TerminateTLS != "" { + printf("|-- tcp://%s (TLS-terminated TCP, %s)\n", hp, fStatus) + } else { + printf("|-- tcp://%s (%s)\n", hp, fStatus) + } for _, a := range st.TailscaleIPs { ipp := net.JoinHostPort(a.String(), strconv.Itoa(int(p))) printf("|-- tcp://%s\n", ipp) @@ -665,64 +668,75 @@ func printTCPStatusTree(ctx context.Context, sc *ipn.ServeConfig, st *ipnstate.S return nil } -func (e *serveEnv) printWebStatusTree(sc *ipn.ServeConfig, hp ipn.HostPort) error { - // No-op if no serve config - if sc == nil { +// printWebStatusTree renders one Web entry (the URL line plus its handler +// tree) for either a node-level serve or a service-level serve. When +// svcName is the empty string, the entry is treated as node-level. Service +// entries include the service name in the URL annotation. +// +// funnel and https are computed by the caller from the parent ServeConfig +// so this function does not need a reference to it. +func printWebStatusTree(wsc *ipn.WebServerConfig, hp ipn.HostPort, funnel, https bool, svcName tailcfg.ServiceName) error { + if wsc == nil { return nil } - fStatus := "tailnet only" - if sc.AllowFunnel[hp] { - fStatus = "Funnel on" - } host, portStr, _ := net.SplitHostPort(string(hp)) - - port, err := parseServePort(portStr) - if err != nil { - return fmt.Errorf("invalid port %q: %w", portStr, err) - } - scheme := "https" - if sc.IsServingHTTP(port) { + if !https { scheme = "http" } - portPart := ":" + portStr if scheme == "http" && portStr == "80" || scheme == "https" && portStr == "443" { portPart = "" } - if scheme == "http" { - hostname, _, _ := strings.Cut(host, ".") - printf("%s://%s%s (%s)\n", scheme, hostname, portPart, fStatus) - } - printf("%s://%s%s (%s)\n", scheme, host, portPart, fStatus) - srvTypeAndDesc := func(h *ipn.HTTPHandler) (string, string) { - switch { - case h.Path != "": - return "path", h.Path - case h.Proxy != "": - return "proxy", h.Proxy - case h.Text != "": - return "text", "\"" + elipticallyTruncate(h.Text, 20) + "\"" + + fStatus := "tailnet only" + if funnel { + fStatus = "Funnel on" + } + if svcName != "" { + printf("%s://%s%s (%s) (%s)\n", scheme, host, portPart, fStatus, svcName) + } else { + if scheme == "http" { + hostname, _, _ := strings.Cut(host, ".") + printf("%s://%s%s (%s)\n", scheme, hostname, portPart, fStatus) } - return "", "" + printf("%s://%s%s (%s)\n", scheme, host, portPart, fStatus) } - mounts := slicesx.MapKeys(sc.Web[hp].Handlers) + mounts := slicesx.MapKeys(wsc.Handlers) + if len(mounts) == 0 { + return nil + } sort.Slice(mounts, func(i, j int) bool { return len(mounts[i]) < len(mounts[j]) }) maxLen := len(mounts[len(mounts)-1]) for _, m := range mounts { - h := sc.Web[hp].Handlers[m] - t, d := srvTypeAndDesc(h) + h := wsc.Handlers[m] + t, d := serveHandlerDesc(h) printf("%s %s%s %-5s %s\n", "|--", m, strings.Repeat(" ", maxLen-len(m)), t, d) } return nil } +// serveHandlerDesc returns the type label and description for an +// HTTPHandler, matching the format used by the Web tree printer for +// node-level and service-level serves. +func serveHandlerDesc(h *ipn.HTTPHandler) (string, string) { + switch { + case h.Path != "": + return "path", h.Path + case h.Proxy != "": + return "proxy", h.Proxy + case h.Text != "": + return "text", "\"" + elipticallyTruncate(h.Text, 20) + "\"" + } + return "", "" +} + func elipticallyTruncate(s string, max int) string { if len(s) <= max { return s @@ -835,10 +849,10 @@ func (e *serveEnv) enableFeatureInteractive(ctx context.Context, feature string, e.lc.IncrementCounter(ctx, fmt.Sprintf("%s_enablement_lost_connection", feature), 1) return err } - if nm := n.NetMap; nm != nil && nm.SelfNode.Valid() { + if self := n.SelfChange; self != nil { gotAll := true for _, c := range caps { - if !nm.SelfNode.HasCap(c) { + if _, has := self.CapMap[c]; !has { // The feature is not yet enabled. // Continue blocking until it is. gotAll = false diff --git a/cmd/tailscale/cli/serve_legacy_test.go b/cmd/tailscale/cli/serve_legacy_test.go index df68b5edd32a1..27cbb5712dd0f 100644 --- a/cmd/tailscale/cli/serve_legacy_test.go +++ b/cmd/tailscale/cli/serve_legacy_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -18,7 +18,7 @@ import ( "testing" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/client/tailscale" + "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" @@ -859,6 +859,9 @@ type fakeLocalServeClient struct { config *ipn.ServeConfig setCount int // counts calls to SetServeConfig queryFeatureResponse *mockQueryFeatureResponse // mock response to QueryFeature calls + prefs *ipn.Prefs // fake preferences, used to test GetPrefs and SetPrefs + SOMarkInUse bool // fake SO mark in use status + statusWithoutPeers *ipnstate.Status // nil for fakeStatus } // fakeStatus is a fake ipnstate.Status value for tests. @@ -875,10 +878,14 @@ var fakeStatus = &ipnstate.Status{ tailcfg.CapabilityFunnelPorts + "?ports=443,8443": nil, }, }, + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, } func (lc *fakeLocalServeClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { - return fakeStatus, nil + if lc.statusWithoutPeers == nil { + return fakeStatus, nil + } + return lc.statusWithoutPeers, nil } func (lc *fakeLocalServeClient) GetServeConfig(ctx context.Context) (*ipn.ServeConfig, error) { @@ -891,6 +898,21 @@ func (lc *fakeLocalServeClient) SetServeConfig(ctx context.Context, config *ipn. return nil } +func (lc *fakeLocalServeClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) { + if lc.prefs == nil { + lc.prefs = ipn.NewPrefs() + } + return lc.prefs, nil +} + +func (lc *fakeLocalServeClient) EditPrefs(ctx context.Context, prefs *ipn.MaskedPrefs) (*ipn.Prefs, error) { + if lc.prefs == nil { + lc.prefs = ipn.NewPrefs() + } + lc.prefs.ApplyEdits(prefs) + return lc.prefs, nil +} + type mockQueryFeatureResponse struct { resp *tailcfg.QueryFeatureResponse err error @@ -908,7 +930,7 @@ func (lc *fakeLocalServeClient) QueryFeature(ctx context.Context, feature string return &tailcfg.QueryFeatureResponse{Complete: true}, nil // fallback to already enabled } -func (lc *fakeLocalServeClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*tailscale.IPNBusWatcher, error) { +func (lc *fakeLocalServeClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*local.IPNBusWatcher, error) { return nil, nil // unused in tests } @@ -916,6 +938,10 @@ func (lc *fakeLocalServeClient) IncrementCounter(ctx context.Context, name strin return nil // unused in tests } +func (lc *fakeLocalServeClient) CheckSOMarkInUse(ctx context.Context) (bool, error) { + return lc.SOMarkInUse, nil +} + // exactError returns an error checker that wants exactly the provided want error. // If optName is non-empty, it's used in the error message. func exactErr(want error, optName ...string) func(error) string { diff --git a/cmd/tailscale/cli/serve_status.go b/cmd/tailscale/cli/serve_status.go new file mode 100644 index 0000000000000..6ba85ca59baa0 --- /dev/null +++ b/cmd/tailscale/cli/serve_status.go @@ -0,0 +1,120 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_serve + +package cli + +import ( + "context" + "maps" + "net" + "slices" + "strconv" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" +) + +// isServeConfigEmpty reports whether sc has no user-visible configuration +// to render in the non-JSON status output. +func isServeConfigEmpty(sc *ipn.ServeConfig) bool { + return sc == nil || (len(sc.TCP) == 0 && len(sc.Web) == 0 && len(sc.Services) == 0 && len(sc.AllowFunnel) == 0) +} + +// printServeStatusTrees prints the tree-style human-readable status of sc, +// including any node-level TCP and Web serve entries and any configured +// services, to [Stdout]. It does not print the funnel-status header, the +// no-config message, or the trailing funnel warning — callers are expected +// to handle those. +// +// Ordering is deterministic: node TCP forwards (existing behavior), then +// node Web entries by HostPort, then services by name. +func printServeStatusTrees(sc *ipn.ServeConfig, st *ipnstate.Status) error { + if sc == nil { + return nil + } + if sc.IsTCPForwardingAny() { + if err := printTCPStatusTree(context.Background(), sc, st); err != nil { + return err + } + printf("\n") + } + for _, hp := range slices.Sorted(maps.Keys(sc.Web)) { + _, portStr, _ := net.SplitHostPort(string(hp)) + port, err := parseServePort(portStr) + if err != nil { + return err + } + funnel := sc.AllowFunnel[hp] + https := !sc.IsServingHTTP(port, noService) + if err := printWebStatusTree(sc.Web[hp], hp, funnel, https, noService); err != nil { + return err + } + printf("\n") + } + for _, name := range slices.Sorted(maps.Keys(sc.Services)) { + if err := printServiceStatusTree(sc, st, name); err != nil { + return err + } + } + return nil +} + +// printServiceStatusTree prints the tree-style status for a single +// configured service. Each rendered URL/forward line is prefixed with the +// service name in the URL annotation (e.g. +// "https://db.example.ts.net (tailnet only) (svc:db)") so service entries +// are visually distinct from node-level serves. +func printServiceStatusTree(sc *ipn.ServeConfig, st *ipnstate.Status, name tailcfg.ServiceName) error { + svc, ok := sc.Services[name] + if !ok || svc == nil { + return nil + } + + if svc.Tun { + printf("tun (L3 forwarding) (%s)\n\n", name) + return nil + } + + suffix := "" + if st != nil && st.CurrentTailnet != nil { + suffix = st.CurrentTailnet.MagicDNSSuffix + } + host := name.WithoutPrefix() + if suffix != "" { + host = host + "." + suffix + } + + // TCP forwards configured directly on the service. + for _, p := range slices.Sorted(maps.Keys(svc.TCP)) { + h := svc.TCP[p] + if h == nil || h.TCPForward == "" { + continue + } + hp := ipn.HostPort(net.JoinHostPort(host, strconv.Itoa(int(p)))) + if h.TerminateTLS != "" { + printf("tcp://%s (TLS-terminated TCP, tailnet only) (%s)\n", hp, name) + } else { + printf("tcp://%s (tailnet only) (%s)\n", hp, name) + } + printf("|--> tcp://%s\n\n", h.TCPForward) + } + + // Web entries (HTTP/HTTPS). Services have no Funnel concept. + for _, hp := range slices.Sorted(maps.Keys(svc.Web)) { + _, portStr, _ := net.SplitHostPort(string(hp)) + port, err := parseServePort(portStr) + if err != nil { + return err + } + https := !sc.IsServingHTTP(port, name) + if err := printWebStatusTree(svc.Web[hp], hp, false, https, name); err != nil { + return err + } + printf("\n") + } + + return nil +} diff --git a/cmd/tailscale/cli/serve_status_test.go b/cmd/tailscale/cli/serve_status_test.go new file mode 100644 index 0000000000000..821a1d667c6de --- /dev/null +++ b/cmd/tailscale/cli/serve_status_test.go @@ -0,0 +1,308 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_serve + +package cli + +import ( + "bytes" + "encoding/json" + "io" + "net" + "strings" + "testing" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/tstest" +) + +// statusTestStatus is a minimal ipnstate.Status used by serve-status tests. +var statusTestStatus = &ipnstate.Status{ + BackendState: ipn.Running.String(), + Self: &ipnstate.PeerStatus{ + DNSName: "foo.test.ts.net.", + }, + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, +} + +func TestPrintServeStatusTrees(t *testing.T) { + tests := []struct { + name string + sc *ipn.ServeConfig + want string + }{ + { + name: "node_web_tailnet_only", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:3000"}, + }}, + }, + }, + want: strings.Join([]string{ + "https://foo.test.ts.net (tailnet only)", + "|-- / proxy http://127.0.0.1:3000", + "", + "", + }, "\n"), + }, + { + name: "node_tcp_funnel_on", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{2222: {TCPForward: "127.0.0.1:22"}}, + AllowFunnel: map[ipn.HostPort]bool{ + "foo.test.ts.net:2222": true, + }, + }, + want: strings.Join([]string{ + "|-- tcp://foo.test.ts.net:2222 (Funnel on)", + "|--> tcp://127.0.0.1:22", + "", + "", + }, "\n"), + }, + { + name: "node_tls_terminated_tcp_tailnet", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "127.0.0.1:8080", TerminateTLS: "foo.test.ts.net"}, + }, + }, + want: strings.Join([]string{ + "|-- tcp://foo.test.ts.net:443 (TLS-terminated TCP, tailnet only)", + "|--> tcp://127.0.0.1:8080", + "", + "", + }, "\n"), + }, + { + name: "service_web_only", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:db": { + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "db.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:5432"}, + }}, + }, + }, + }, + }, + want: strings.Join([]string{ + "https://db.test.ts.net (tailnet only) (svc:db)", + "|-- / proxy http://127.0.0.1:5432", + "", + "", + }, "\n"), + }, + { + name: "service_tcp_forward", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:ssh": { + TCP: map[uint16]*ipn.TCPPortHandler{2222: {TCPForward: "127.0.0.1:22"}}, + }, + }, + }, + want: strings.Join([]string{ + "tcp://ssh.test.ts.net:2222 (tailnet only) (svc:ssh)", + "|--> tcp://127.0.0.1:22", + "", + "", + }, "\n"), + }, + { + name: "service_tls_terminated_tcp", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "127.0.0.1:8080", TerminateTLS: "foo.test.ts.net"}, + }, + }, + }, + }, + want: strings.Join([]string{ + "tcp://foo.test.ts.net:443 (TLS-terminated TCP, tailnet only) (svc:foo)", + "|--> tcp://127.0.0.1:8080", + "", + "", + }, "\n"), + }, + { + name: "service_tun", + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:vpn": {Tun: true}, + }, + }, + want: strings.Join([]string{ + "tun (L3 forwarding) (svc:vpn)", + "", + "", + }, "\n"), + }, + { + name: "node_and_services_mixed", + sc: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:3000"}, + }}, + }, + AllowFunnel: map[ipn.HostPort]bool{ + "foo.test.ts.net:443": true, + }, + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:db": { + TCP: map[uint16]*ipn.TCPPortHandler{5432: {TCPForward: "127.0.0.1:5432"}}, + }, + }, + }, + want: strings.Join([]string{ + "https://foo.test.ts.net (Funnel on)", + "|-- / proxy http://127.0.0.1:3000", + "", + "tcp://db.test.ts.net:5432 (tailnet only) (svc:db)", + "|--> tcp://127.0.0.1:5432", + "", + "", + }, "\n"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + tstest.Replace(t, &Stdout, io.Writer(&stdout)) + tstest.Replace(t, &Stderr, io.Writer(&stderr)) + + if err := printServeStatusTrees(tt.sc, statusTestStatus); err != nil { + t.Fatalf("printServeStatusTrees: %v", err) + } + if got := stdout.String(); got != tt.want { + t.Errorf("\nGot:\n%q\nExpected:\n%q", got, tt.want) + } + if got := stderr.String(); got != "" { + t.Errorf("unexpected Stderr output: %q", got) + } + }) + } +} + +// TestPrintServeStatusTreesParity asserts that the host-identifying keys +// visible in the JSON serialization of a ServeConfig also appear in the +// human-readable output, so the two views stay in lockstep. This is the +// parity contract from issue #34163. +// +// It checks: +// - every Services key (service name) +// - every node-level Web HostPort host +// - every service-level Web HostPort host +// - every node-level TCP forward as a host:port string +// - every tun-mode service rendering the "tun" marker after its name +func TestPrintServeStatusTreesParity(t *testing.T) { + sc := &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + 2222: {TCPForward: "127.0.0.1:22"}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://127.0.0.1:3000"}, + }}, + }, + AllowFunnel: map[ipn.HostPort]bool{ + "foo.test.ts.net:2222": true, + }, + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:db": { + TCP: map[uint16]*ipn.TCPPortHandler{5432: {TCPForward: "127.0.0.1:5432"}}, + }, + "svc:web": { + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "web.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/api": {Proxy: "http://127.0.0.1:9000"}, + }}, + }, + }, + "svc:vpn": {Tun: true}, + }, + } + + // Marshal to JSON and reparse as a generic map so the parity check walks + // the same wire shape clients see, not the typed Go values. + jsonBytes, err := json.Marshal(sc) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + var raw map[string]any + if err := json.Unmarshal(jsonBytes, &raw); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + + var stdout, stderr bytes.Buffer + tstest.Replace(t, &Stdout, io.Writer(&stdout)) + tstest.Replace(t, &Stderr, io.Writer(&stderr)) + + if err := printServeStatusTrees(sc, statusTestStatus); err != nil { + t.Fatalf("printServeStatusTrees: %v", err) + } + if got := stderr.String(); got != "" { + t.Errorf("unexpected Stderr output: %q", got) + } + human := stdout.String() + + services, _ := raw["Services"].(map[string]any) + for name, sval := range services { + if !strings.Contains(human, name) { + t.Errorf("human output missing service name %q\n--- human ---\n%s", name, human) + } + svc, _ := sval.(map[string]any) + if tun, _ := svc["Tun"].(bool); tun { + tunLine := "tun (L3 forwarding) (" + name + ")" + if !strings.Contains(human, tunLine) { + t.Errorf("human output missing tun marker for %q\n--- human ---\n%s", tunLine, human) + } + } + web, _ := svc["Web"].(map[string]any) + for hp := range web { + host := strings.SplitN(hp, ":", 2)[0] + if !strings.Contains(human, host) { + t.Errorf("human output missing service %s Web host %q\n--- human ---\n%s", name, host, human) + } + } + } + + if web, ok := raw["Web"].(map[string]any); ok { + for hp := range web { + host := strings.SplitN(hp, ":", 2)[0] + if !strings.Contains(human, host) { + t.Errorf("human output missing node Web host %q\n--- human ---\n%s", host, human) + } + } + } + + nodeHost := strings.TrimSuffix(statusTestStatus.Self.DNSName, ".") + if tcp, ok := raw["TCP"].(map[string]any); ok { + for portStr, hVal := range tcp { + h, _ := hVal.(map[string]any) + fwd, _ := h["TCPForward"].(string) + if fwd == "" { + continue + } + hostport := net.JoinHostPort(nodeHost, portStr) + if !strings.Contains(human, hostport) { + t.Errorf("human output missing node TCP forward %q\n--- human ---\n%s", hostport, human) + } + } + } +} diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 3e173ce28d8c1..13f5c09b8eac5 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_serve + package cli import ( @@ -18,16 +20,24 @@ import ( "os/signal" "path" "path/filepath" + "regexp" + "runtime" + "slices" "sort" "strconv" "strings" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/client/tailscale" + "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/ipn/conffile" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" + "tailscale.com/types/ipproto" + "tailscale.com/util/dnsname" "tailscale.com/util/mak" + "tailscale.com/util/prompt" + "tailscale.com/util/set" "tailscale.com/util/slicesx" "tailscale.com/version" ) @@ -40,10 +50,95 @@ type commandInfo struct { LongHelp string } +type serviceNameFlag struct { + Value *tailcfg.ServiceName +} + +func (s *serviceNameFlag) Set(sv string) error { + if sv == "" { + s.Value = new(tailcfg.ServiceName) + return nil + } + v := tailcfg.ServiceName(sv) + if err := v.Validate(); err != nil { + return fmt.Errorf("invalid service name: %q", sv) + } + *s.Value = v + return nil +} + +// String returns the string representation of service name. +func (s *serviceNameFlag) String() string { + return s.Value.String() +} + +type bgBoolFlag struct { + Value bool + IsSet bool // tracks if the flag was set by the user +} + +// Set sets the boolean flag and whether it's explicitly set by user based on the string value. +func (b *bgBoolFlag) Set(s string) error { + v, err := strconv.ParseBool(s) + if err != nil { + return err + } + b.Value = v + b.IsSet = true + return nil +} + +// This is a hack to make the flag package recognize that this is a boolean flag. +func (b *bgBoolFlag) IsBoolFlag() bool { return true } + +// String returns the string representation of the boolean flag. +func (b *bgBoolFlag) String() string { + if !b.IsSet { + return "default" + } + return strconv.FormatBool(b.Value) +} + +type acceptAppCapsFlag struct { + Value *[]tailcfg.PeerCapability +} + +// An application capability name has the form {domain}/{name}. +// Both parts must use the (simplified) FQDN label character set. +// The "name" can contain forward slashes. +// \pL = Unicode Letter, \pN = Unicode Number, - = Hyphen +var validAppCap = regexp.MustCompile(`^([\pL\pN-]+\.)+[\pL\pN-]+\/[\pL\pN-/]+$`) + +// Set appends s to the list of appCaps to accept. +func (u *acceptAppCapsFlag) Set(s string) error { + if s == "" { + return nil + } + appCaps := strings.SplitSeq(s, ",") + for appCap := range appCaps { + appCap = strings.TrimSpace(appCap) + if !validAppCap.MatchString(appCap) { + return fmt.Errorf("%q does not match the form {domain}/{name}, where domain must be a fully qualified domain name", appCap) + } + *u.Value = append(*u.Value, tailcfg.PeerCapability(appCap)) + } + return nil +} + +// String returns the string representation of the slice of appCaps to accept. +func (u *acceptAppCapsFlag) String() string { + s := make([]string, len(*u.Value)) + for i, v := range *u.Value { + s[i] = string(v) + } + return strings.Join(s, ",") +} + var serveHelpCommon = strings.TrimSpace(` can be a file, directory, text, or most commonly the location to a service running on the local machine. The location to the location service can be expressed as a port number (e.g., 3000), a partial URL (e.g., localhost:3000), or a full URL including a path (e.g., http://localhost:3000/foo). +On Unix-like systems, you can also specify a Unix domain socket (e.g., unix:/tmp/myservice.sock). EXAMPLES - Expose an HTTP server running at 127.0.0.1:3000 in the foreground: @@ -55,6 +150,9 @@ EXAMPLES - Expose an HTTPS server with invalid or self-signed certificates at https://localhost:8443 $ tailscale %[1]s https+insecure://localhost:8443 + - Expose a service listening on a Unix socket (Linux/macOS/BSD only): + $ tailscale %[1]s unix:/var/run/myservice.sock + For more examples and use cases visit our docs site https://tailscale.com/kb/1247/funnel-serve-use-cases `) @@ -72,8 +170,27 @@ const ( serveTypeHTTP serveTypeTCP serveTypeTLSTerminatedTCP + serveTypeTUN ) +func serveTypeFromConfString(sp conffile.ServiceProtocol) (st serveType, ok bool) { + switch sp { + case conffile.ProtoHTTP: + return serveTypeHTTP, true + case conffile.ProtoHTTPS, conffile.ProtoHTTPSInsecure, conffile.ProtoFile: + return serveTypeHTTPS, true + case conffile.ProtoTCP: + return serveTypeTCP, true + case conffile.ProtoTLSTerminatedTCP: + return serveTypeTLSTerminatedTCP, true + case conffile.ProtoTUN: + return serveTypeTUN, true + } + return -1, false +} + +const noService tailcfg.ServiceName = "" + var infoMap = map[serveMode]commandInfo{ serve: { Name: "serve", @@ -102,7 +219,7 @@ var errHelpFunc = func(m serveMode) error { // newServeV2Command returns a new "serve" subcommand using e as its environment. func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { if subcmd != serve && subcmd != funnel { - log.Fatalf("newServeDevCommand called with unknown subcmd %q", subcmd) + log.Fatalf("newServeDevCommand called with unknown subcmd %v", subcmd) } info := infoMap[subcmd] @@ -119,35 +236,101 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { Exec: e.runServeCombined(subcmd), FlagSet: e.newFlags("serve-set", func(fs *flag.FlagSet) { - fs.BoolVar(&e.bg, "bg", false, "Run the command as a background process (default false)") + fs.Var(&e.bg, "bg", "Run the command as a background process (default false, when --service is set defaults to true).") fs.StringVar(&e.setPath, "set-path", "", "Appends the specified path to the base URL for accessing the underlying service") fs.UintVar(&e.https, "https", 0, "Expose an HTTPS server at the specified port (default mode)") if subcmd == serve { fs.UintVar(&e.http, "http", 0, "Expose an HTTP server at the specified port") + fs.Var(&acceptAppCapsFlag{Value: &e.acceptAppCaps}, "accept-app-caps", "App capabilities to forward to the server (specify multiple capabilities with a comma-separated list)") + fs.Var(&serviceNameFlag{Value: &e.service}, "service", "Serve for a service with distinct virtual IP instead on node itself.") + fs.BoolVar(&e.tun, "tun", false, "Forward all traffic to the local machine (default false), only supported for services. Refer to docs for more information.") } fs.UintVar(&e.tcp, "tcp", 0, "Expose a TCP forwarder to forward raw TCP packets at the specified port") fs.UintVar(&e.tlsTerminatedTCP, "tls-terminated-tcp", 0, "Expose a TCP forwarder to forward TLS-terminated TCP packets at the specified port") + fs.UintVar(&e.proxyProtocol, "proxy-protocol", 0, "PROXY protocol version (1 or 2) for TCP forwarding") fs.BoolVar(&e.yes, "yes", false, "Update without interactive prompts (default false)") }), UsageFunc: usageFuncNoDefaultValues, - Subcommands: []*ffcli.Command{ - { - Name: "status", - ShortUsage: "tailscale " + info.Name + " status [--json]", - Exec: e.runServeStatus, - ShortHelp: "View current " + info.Name + " configuration", - FlagSet: e.newFlags("serve-status", func(fs *flag.FlagSet) { - fs.BoolVar(&e.json, "json", false, "output JSON") - }), - }, - { - Name: "reset", - ShortUsage: "tailscale " + info.Name + " reset", - ShortHelp: "Reset current " + info.Name + " config", - Exec: e.runServeReset, - FlagSet: e.newFlags("serve-reset", nil), - }, - }, + Subcommands: func() []*ffcli.Command { + subcmds := []*ffcli.Command{ + { + Name: "status", + ShortUsage: "tailscale " + info.Name + " status [--json]", + Exec: e.runServeStatus, + ShortHelp: "View current " + info.Name + " configuration", + FlagSet: e.newFlags("serve-status", func(fs *flag.FlagSet) { + fs.BoolVar(&e.json, "json", false, "output JSON") + }), + }, + { + Name: "reset", + ShortUsage: "tailscale " + info.Name + " reset", + ShortHelp: "Reset current " + info.Name + " config", + Exec: e.runServeReset, + FlagSet: e.newFlags("serve-reset", nil), + }, + } + if subcmd == serve { + subcmds = append(subcmds, []*ffcli.Command{ + { + Name: "drain", + ShortUsage: fmt.Sprintf("tailscale %s drain ", info.Name), + ShortHelp: "Drain a service from the current node", + LongHelp: "Make the current node no longer accept new connections for the specified service.\n" + + "Existing connections will continue to work until they are closed, but no new connections will be accepted.\n" + + "Use this command to gracefully remove a service from the current node without disrupting existing connections.\n" + + " should be a service name (e.g., svc:my-service).", + Exec: e.runServeDrain, + }, + { + Name: "clear", + ShortUsage: fmt.Sprintf("tailscale %s clear ", info.Name), + ShortHelp: "Remove all config for a service", + LongHelp: "Remove all handlers configured for the specified service.", + Exec: e.runServeClear, + }, + { + Name: "advertise", + ShortUsage: fmt.Sprintf("tailscale %s advertise ", info.Name), + ShortHelp: "Advertise this node as a service proxy to the tailnet", + LongHelp: "Advertise this node as a service proxy to the tailnet. This command is used\n" + + "to make the current node be considered as a service host for a service. This is\n" + + "useful to bring a service back after it has been drained. (i.e. after running \n" + + "`tailscale serve drain `). This is not needed if you are using `tailscale serve` to initialize a service.", + Exec: e.runServeAdvertise, + }, + { + Name: "get-config", + ShortUsage: fmt.Sprintf("tailscale %s get-config [--service=] [--all]", info.Name), + ShortHelp: "Get service configuration to save to a file", + LongHelp: "Get the configuration for services that this node is currently hosting in a\n" + + "format that can later be provided to set-config. This can be used to declaratively set\n" + + "configuration for a service host.", + Exec: e.runServeGetConfig, + FlagSet: e.newFlags("serve-get-config", func(fs *flag.FlagSet) { + fs.BoolVar(&e.allServices, "all", false, "read config from all services") + fs.Var(&serviceNameFlag{Value: &e.service}, "service", "read config from a particular service") + }), + }, + { + Name: "set-config", + ShortUsage: fmt.Sprintf("tailscale %s set-config [--service=] [--all]", info.Name), + ShortHelp: "Define service configuration from a file", + LongHelp: "Read the provided configuration file and use it to declaratively set the configuration\n" + + "for either a single service, or for all services that this node is hosting. If --service is specified,\n" + + "all endpoint handlers for that service are overwritten. If --all is specified, all endpoint handlers for\n" + + "all services are overwritten.\n\n" + + "For information on the file format, see tailscale.com/kb/1589/tailscale-services-configuration-file", + Exec: e.runServeSetConfig, + FlagSet: e.newFlags("serve-set-config", func(fs *flag.FlagSet) { + fs.BoolVar(&e.allServices, "all", false, "apply config to all services") + fs.Var(&serviceNameFlag{Value: &e.service}, "service", "apply config to a particular service") + }), + }, + }...) + } + return subcmds + }(), } } @@ -161,9 +344,16 @@ func (e *serveEnv) validateArgs(subcmd serveMode, args []string) error { fmt.Fprint(e.stderr(), "\nPlease see https://tailscale.com/kb/1242/tailscale-serve for more information.\n") return errHelpFunc(subcmd) } + if len(args) == 0 && e.tun { + return nil + } if len(args) == 0 { return flag.ErrHelp } + if e.tun && len(args) > 1 { + fmt.Fprintln(e.stderr(), "Error: invalid argument format") + return errHelpFunc(subcmd) + } if len(args) > 2 { fmt.Fprintf(e.stderr(), "Error: invalid number of arguments (%d)\n", len(args)) return errHelpFunc(subcmd) @@ -205,7 +395,16 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { ctx, cancel := signal.NotifyContext(ctx, os.Interrupt) defer cancel() + forService := e.service != "" + if !e.bg.IsSet { + e.bg.Value = forService + } + funnel := subcmd == funnel + if forService && funnel { + return errors.New("Error: --service flag is not supported with funnel") + } + if funnel { // verify node has funnel capabilities if err := e.verifyFunnelEnabled(ctx, 443); err != nil { @@ -213,6 +412,10 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } + if forService && !e.bg.Value { + return errors.New("Error: --service flag is only compatible with background mode") + } + mount, err := cleanURLPath(e.setPath) if err != nil { return fmt.Errorf("failed to clean the mount point: %w", err) @@ -224,6 +427,14 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { return errHelpFunc(subcmd) } + if (srvType == serveTypeHTTP || srvType == serveTypeHTTPS) && e.proxyProtocol != 0 { + return fmt.Errorf("PROXY protocol is only supported for TCP forwarding, not HTTP/HTTPS") + } + // Validate PROXY protocol version + if e.proxyProtocol != 0 && e.proxyProtocol != 1 && e.proxyProtocol != 2 { + return fmt.Errorf("invalid PROXY protocol version %d; must be 1 or 2", e.proxyProtocol) + } + sc, err := e.lc.GetServeConfig(ctx) if err != nil { return fmt.Errorf("error getting serve config: %w", err) @@ -238,6 +449,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { return fmt.Errorf("getting client status: %w", err) } dnsName := strings.TrimSuffix(st.Self.DNSName, ".") + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix // set parent serve config to always be persisted // at the top level, but a nested config might be @@ -245,7 +457,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { // foreground or background. parentSC := sc - turnOff := "off" == args[len(args)-1] + turnOff := len(args) > 0 && "off" == args[len(args)-1] if !turnOff && srvType == serveTypeHTTPS { // Running serve with https requires that the tailnet has enabled // https cert provisioning. Send users through an interactive flow @@ -261,18 +473,26 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } - var watcher *tailscale.IPNBusWatcher - wantFg := !e.bg && !turnOff - if wantFg { - // validate the config before creating a WatchIPNBus session - if err := e.validateConfig(parentSC, srvPort, srvType); err != nil { - return err - } + var watcher *local.IPNBusWatcher + svcName := noService + if forService { + svcName = e.service + dnsName = e.service.String() + } + tagged := st.Self.Tags != nil && st.Self.Tags.Len() > 0 + if forService && !tagged && !turnOff { + return errors.New("service hosts must be tagged nodes") + } + if !forService && srvType == serveTypeTUN { + return errors.New("tun mode is only supported for services") + } + wantFg := !e.bg.Value && !turnOff + if wantFg { // if foreground mode, create a WatchIPNBus session // and use the nested config for all following operations // TODO(marwan-at-work): nested-config validations should happen here or previous to this point. - watcher, err = e.lc.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys) + watcher, err = e.lc.WatchIPNBus(ctx, ipn.NotifyInitialState) if err != nil { return err } @@ -291,12 +511,20 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { var msg string if turnOff { - err = e.unsetServe(sc, dnsName, srvType, srvPort, mount) + // only unset serve when trying to unset with type and port flags. + err = e.unsetServe(sc, dnsName, srvType, srvPort, mount, magicDNSSuffix) } else { - if err := e.validateConfig(parentSC, srvPort, srvType); err != nil { + if forService { + e.addServiceToPrefs(ctx, svcName) + } + target := "" + if len(args) > 0 { + target = args[0] + } + if err := e.shouldWarnRemoteDestCompatibility(ctx, target); err != nil { return err } - err = e.setServe(sc, st, dnsName, srvType, srvPort, mount, args[0], funnel) + err = e.setServe(sc, dnsName, srvType, srvPort, mount, target, funnel, magicDNSSuffix, e.acceptAppCaps, int(e.proxyProtocol)) msg = e.messageForPort(sc, st, dnsName, srvType, srvPort) } if err != nil { @@ -305,7 +533,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } if err := e.lc.SetServeConfig(ctx, parentSC); err != nil { - if tailscale.IsPreconditionsFailedError(err) { + if local.IsPreconditionsFailedError(err) { fmt.Fprintln(e.stderr(), "Another client is changing the serve config; please try again.") } return err @@ -331,47 +559,363 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } -const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" +func (e *serveEnv) addServiceToPrefs(ctx context.Context, serviceName tailcfg.ServiceName) error { + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting prefs: %w", err) + } + advertisedServices := prefs.AdvertiseServices + if slices.Contains(advertisedServices, serviceName.String()) { + return nil // already advertised + } + advertisedServices = append(advertisedServices, serviceName.String()) + _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: advertisedServices, + }, + }) + return err +} -func (e *serveEnv) validateConfig(sc *ipn.ServeConfig, port uint16, wantServe serveType) error { - sc, isFg := sc.FindConfig(port) - if sc == nil { +func (e *serveEnv) removeServiceFromPrefs(ctx context.Context, serviceName tailcfg.ServiceName) error { + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting prefs: %w", err) + } + if len(prefs.AdvertiseServices) == 0 { + return nil // nothing to remove + } + initialLen := len(prefs.AdvertiseServices) + prefs.AdvertiseServices = slices.DeleteFunc(prefs.AdvertiseServices, func(s string) bool { return s == serviceName.String() }) + if initialLen == len(prefs.AdvertiseServices) { + return nil // serviceName not advertised + } + _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: prefs.AdvertiseServices, + }, + }) + return err +} + +func (e *serveEnv) runServeDrain(ctx context.Context, args []string) error { + if len(args) == 0 { + return errHelp + } + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := args[0] + svcName := tailcfg.ServiceName(svc) + if err := svcName.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + return e.removeServiceFromPrefs(ctx, svcName) +} + +func (e *serveEnv) runServeClear(ctx context.Context, args []string) error { + if len(args) == 0 { + return errHelp + } + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := tailcfg.ServiceName(args[0]) + if err := svc.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + sc, err := e.lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("error getting serve config: %w", err) + } + if _, ok := sc.Services[svc]; !ok { + log.Printf("service %s not found in serve config, nothing to clear", svc) return nil } - if isFg { - return errors.New("foreground already exists under this port") + delete(sc.Services, svc) + if err := e.removeServiceFromPrefs(ctx, svc); err != nil { + return fmt.Errorf("error removing service %s from prefs: %w", svc, err) } - if !e.bg { - return fmt.Errorf(backgroundExistsMsg, infoMap[e.subcmd].Name, wantServe.String(), port) + return e.lc.SetServeConfig(ctx, sc) +} + +func (e *serveEnv) runServeAdvertise(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("error: missing service name argument") } - existingServe := serveFromPortHandler(sc.TCP[port]) - if wantServe != existingServe { - return fmt.Errorf("want %q but port is already serving %q", wantServe, existingServe) + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp } - return nil + svc := tailcfg.ServiceName(args[0]) + if err := svc.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + return e.addServiceToPrefs(ctx, svc) } -func serveFromPortHandler(tcp *ipn.TCPPortHandler) serveType { - switch { - case tcp.HTTP: - return serveTypeHTTP - case tcp.HTTPS: - return serveTypeHTTPS - case tcp.TerminateTLS != "": - return serveTypeTLSTerminatedTCP - case tcp.TCPForward != "": - return serveTypeTCP - default: - return -1 +func (e *serveEnv) runServeGetConfig(ctx context.Context, args []string) (err error) { + forSingleService := e.service.Validate() == nil + sc, err := e.lc.GetServeConfig(ctx) + if err != nil { + return err + } + + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return err + } + advertised := set.SetOf(prefs.AdvertiseServices) + + st, err := e.getLocalClientStatusWithoutPeers(ctx) + if err != nil { + return err + } + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix + + handleService := func(svcName tailcfg.ServiceName, serviceConfig *ipn.ServiceConfig) (*conffile.ServiceDetailsFile, error) { + var sdf conffile.ServiceDetailsFile + // Leave unset for true case since that's the default. + if !advertised.Contains(svcName.String()) { + sdf.Advertised.Set(false) + } + + if serviceConfig.Tun { + mak.Set(&sdf.Endpoints, &tailcfg.ProtoPortRange{Ports: tailcfg.PortRangeAny}, &conffile.Target{ + Protocol: conffile.ProtoTUN, + Destination: "", + DestinationPorts: tailcfg.PortRange{}, + }) + } + + for port, config := range serviceConfig.TCP { + sniName := fmt.Sprintf("%s.%s", svcName.WithoutPrefix(), magicDNSSuffix) + ppr := tailcfg.ProtoPortRange{Proto: int(ipproto.TCP), Ports: tailcfg.PortRange{First: port, Last: port}} + if config.TCPForward != "" { + var proto conffile.ServiceProtocol + if config.TerminateTLS != "" { + proto = conffile.ProtoTLSTerminatedTCP + } else { + proto = conffile.ProtoTCP + } + destHost, destPortStr, err := net.SplitHostPort(config.TCPForward) + if err != nil { + return nil, fmt.Errorf("parse TCPForward=%q: %w", config.TCPForward, err) + } + destPort, err := strconv.ParseUint(destPortStr, 10, 16) + if err != nil { + return nil, fmt.Errorf("parse port %q: %w", destPortStr, err) + } + mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ + Protocol: proto, + Destination: destHost, + DestinationPorts: tailcfg.PortRange{First: uint16(destPort), Last: uint16(destPort)}, + }) + } else if config.HTTP || config.HTTPS { + webKey := ipn.HostPort(net.JoinHostPort(sniName, strconv.FormatUint(uint64(port), 10))) + handlers, ok := serviceConfig.Web[webKey] + if !ok { + return nil, fmt.Errorf("service %q: HTTP/HTTPS is set but no handlers in config", svcName) + } + defaultHandler, ok := handlers.Handlers["/"] + if !ok { + return nil, fmt.Errorf("service %q: root handler not set", svcName) + } + if defaultHandler.Path != "" { + mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ + Protocol: conffile.ProtoFile, + Destination: defaultHandler.Path, + DestinationPorts: tailcfg.PortRange{}, + }) + } else if defaultHandler.Proxy != "" { + proto, rest, ok := strings.Cut(defaultHandler.Proxy, "://") + if !ok { + return nil, fmt.Errorf("service %q: invalid proxy handler %q", svcName, defaultHandler.Proxy) + } + host, portStr, err := net.SplitHostPort(rest) + if err != nil { + return nil, fmt.Errorf("service %q: invalid proxy handler %q: %w", svcName, defaultHandler.Proxy, err) + } + + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return nil, fmt.Errorf("service %q: parse port %q: %w", svcName, portStr, err) + } + + mak.Set(&sdf.Endpoints, &ppr, &conffile.Target{ + Protocol: conffile.ServiceProtocol(proto), + Destination: host, + DestinationPorts: tailcfg.PortRange{First: uint16(port), Last: uint16(port)}, + }) + } + } + } + + return &sdf, nil + } + + var j []byte + + if e.allServices && forSingleService { + return errors.New("cannot specify both --all and --service") + } else if e.allServices { + var scf conffile.ServicesConfigFile + scf.Version = "0.0.1" + for svcName, serviceConfig := range sc.Services { + sdf, err := handleService(svcName, serviceConfig) + if err != nil { + return err + } + mak.Set(&scf.Services, svcName, sdf) + } + j, err = json.MarshalIndent(scf, "", " ") + if err != nil { + return err + } + } else if forSingleService { + serviceConfig, ok := sc.Services[e.service] + if !ok { + j = []byte("{}") + } else { + sdf, err := handleService(e.service, serviceConfig) + if err != nil { + return err + } + sdf.Version = "0.0.1" + j, err = json.MarshalIndent(sdf, "", " ") + if err != nil { + return err + } + } + } else { + return errors.New("must specify either --service=svc: or --all") } + + j = append(j, '\n') + _, err = e.stdout().Write(j) + return err } -func (e *serveEnv) setServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool) error { +func (e *serveEnv) runServeSetConfig(ctx context.Context, args []string) (err error) { + if len(args) != 1 { + return errors.New("must specify filename") + } + forSingleService := e.service.Validate() == nil + + var scf *conffile.ServicesConfigFile + if e.allServices && forSingleService { + return errors.New("cannot specify both --all and --service") + } else if e.allServices { + scf, err = conffile.LoadServicesConfig(args[0], "") + } else if forSingleService { + scf, err = conffile.LoadServicesConfig(args[0], e.service.String()) + } else { + return errors.New("must specify either --service=svc: or --all") + } + if err != nil { + return fmt.Errorf("could not read config from file %q: %w", args[0], err) + } + + st, err := e.getLocalClientStatusWithoutPeers(ctx) + if err != nil { + return fmt.Errorf("getting client status: %w", err) + } + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix + sc, err := e.lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("getting current serve config: %w", err) + } + + // Clear all existing config. + if forSingleService { + if sc.Services != nil { + if sc.Services[e.service] != nil { + delete(sc.Services, e.service) + } + } + } else { + sc.Services = map[tailcfg.ServiceName]*ipn.ServiceConfig{} + } + advertisedServices := set.Set[string]{} + + for name, details := range scf.Services { + for ppr, ep := range details.Endpoints { + if ep.Protocol == conffile.ProtoTUN { + err := e.setServe(sc, name.String(), serveTypeTUN, 0, "", "", false, magicDNSSuffix, nil, 0 /* proxy protocol */) + if err != nil { + return err + } + // TUN mode is exclusive. + break + } + + if ppr.Proto != int(ipproto.TCP) { + return fmt.Errorf("service %q: source ports must be TCP", name) + } + serveType, _ := serveTypeFromConfString(ep.Protocol) + for port := ppr.Ports.First; port <= ppr.Ports.Last; port++ { + var target string + if ep.Protocol == conffile.ProtoFile { + target = ep.Destination + } else { + // map source port range 1-1 to destination port range + destPort := ep.DestinationPorts.First + (port - ppr.Ports.First) + portStr := fmt.Sprint(destPort) + target = fmt.Sprintf("%s://%s", ep.Protocol, net.JoinHostPort(ep.Destination, portStr)) + } + err := e.setServe(sc, name.String(), serveType, port, "/", target, false, magicDNSSuffix, nil, 0 /* proxy protocol */) + if err != nil { + return fmt.Errorf("service %q: %w", name, err) + } + } + } + if v, set := details.Advertised.Get(); !set || v { + advertisedServices.Add(name.String()) + } + } + + var changed bool + var servicesList []string + if e.allServices { + servicesList = advertisedServices.Slice() + changed = true + } else if advertisedServices.Contains(e.service.String()) { + // If allServices wasn't set, the only service that could have been + // advertised is the one that was provided as a flag. + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return err + } + if !slices.Contains(prefs.AdvertiseServices, e.service.String()) { + servicesList = append(prefs.AdvertiseServices, e.service.String()) + changed = true + } + } + if changed { + _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: servicesList, + }, + }) + if err != nil { + return err + } + } + + return e.lc.SetServeConfig(ctx, sc) +} + +func (e *serveEnv) setServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool, mds string, caps []tailcfg.PeerCapability, proxyProtocol int) error { // update serve config based on the type switch srvType { case serveTypeHTTPS, serveTypeHTTP: useTLS := srvType == serveTypeHTTPS - err := e.applyWebServe(sc, dnsName, srvPort, useTLS, mount, target) + err := e.applyWebServe(sc, dnsName, srvPort, useTLS, mount, target, mds, caps) if err != nil { return fmt.Errorf("failed apply web serve: %w", err) } @@ -379,45 +923,61 @@ func (e *serveEnv) setServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName st if e.setPath != "" { return fmt.Errorf("cannot mount a path for TCP serve") } - - err := e.applyTCPServe(sc, dnsName, srvType, srvPort, target) + err := e.applyTCPServe(sc, dnsName, srvType, srvPort, target, mds, proxyProtocol) if err != nil { return fmt.Errorf("failed to apply TCP serve: %w", err) } + case serveTypeTUN: + // Caller checks that TUN mode is only supported for services. + svcName := tailcfg.ServiceName(dnsName) + if _, ok := sc.Services[svcName]; !ok { + mak.Set(&sc.Services, svcName, new(ipn.ServiceConfig)) + } + sc.Services[svcName].Tun = true default: return fmt.Errorf("invalid type %q", srvType) } // update the serve config based on if funnel is enabled - e.applyFunnel(sc, dnsName, srvPort, allowFunnel) - + // Since funnel is not supported for services, we only apply it for node's serve. + if svcName := tailcfg.AsServiceName(dnsName); svcName == noService { + e.applyFunnel(sc, dnsName, srvPort, allowFunnel) + } return nil } var ( - msgFunnelAvailable = "Available on the internet:" - msgServeAvailable = "Available within your tailnet:" - msgRunningInBackground = "%s started and running in the background." - msgDisableProxy = "To disable the proxy, run: tailscale %s --%s=%d off" - msgToExit = "Press Ctrl+C to exit." + msgFunnelAvailable = "Available on the internet:" + msgServeAvailable = "Available within your tailnet:" + msgServiceWaitingApproval = "This machine is configured as a service proxy for %s, but approval from an admin is required. Once approved, it will be available in your Tailnet as:" + msgRunningInBackground = "%s started and running in the background." + msgRunningTunService = "IPv4 and IPv6 traffic to %s is being routed to your operating system." + msgDisableProxy = "To disable the proxy, run: tailscale %s --%s=%d off" + msgDisableServiceProxy = "To disable the proxy, run: tailscale serve --service=%s --%s=%d off" + msgDisableServiceTun = "To disable the service in TUN mode, run: tailscale serve --service=%s --tun off" + msgDisableService = "To remove config for the service, run: tailscale serve clear %s" + msgWarnRemoteDestCompatibility = "Warning: %s doesn't support connecting to remote destinations from non-default route, see tailscale.com/kb/1552/tailscale-services for detail." + msgToExit = "Press Ctrl+C to exit." ) // messageForPort returns a message for the given port based on the // serve config and status. func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16) string { var output strings.Builder - - hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort)))) - - if sc.AllowFunnel[hp] == true { - output.WriteString(msgFunnelAvailable) - } else { - output.WriteString(msgServeAvailable) + svcName := tailcfg.AsServiceName(dnsName) + forService := svcName != noService + var webConfig *ipn.WebServerConfig + var tcpHandler *ipn.TCPPortHandler + ips := st.TailscaleIPs + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix + host := dnsName + if forService { + host = strings.Join([]string{svcName.WithoutPrefix(), magicDNSSuffix}, ".") } - output.WriteString("\n\n") + hp := ipn.HostPort(net.JoinHostPort(host, strconv.Itoa(int(srvPort)))) scheme := "https" - if sc.IsServingHTTP(srvPort) { + if sc.IsServingHTTP(srvPort, svcName) { scheme = "http" } @@ -438,37 +998,71 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN } return "", "" } + if forService { + serviceIPMaps, err := tailcfg.UnmarshalNodeCapJSON[tailcfg.ServiceIPMappings](st.Self.CapMap, tailcfg.NodeAttrServiceHost) + if err != nil || len(serviceIPMaps) == 0 || serviceIPMaps[0][svcName] == nil { + // The capmap does not contain IPs for this service yet. Usually this means + // the service hasn't been added to prefs and sent to control yet. + output.WriteString(fmt.Sprintf(msgServiceWaitingApproval, svcName.String())) + ips = nil + } else { + output.WriteString(msgServeAvailable) + ips = serviceIPMaps[0][svcName] + } + output.WriteString("\n\n") + svc := sc.Services[svcName] + if srvType == serveTypeTUN && svc.Tun { + output.WriteString(fmt.Sprintf(msgRunningTunService, host)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableServiceTun, dnsName)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableService, dnsName)) + return output.String() + } + if svc != nil { + webConfig = svc.Web[hp] + tcpHandler = svc.TCP[srvPort] + } + } else { + if sc.AllowFunnel[hp] == true { + output.WriteString(msgFunnelAvailable) + } else { + output.WriteString(msgServeAvailable) + } + output.WriteString("\n\n") + webConfig = sc.Web[hp] + tcpHandler = sc.TCP[srvPort] + } - if sc.Web[hp] != nil { - mounts := slicesx.MapKeys(sc.Web[hp].Handlers) + if webConfig != nil { + mounts := slicesx.MapKeys(webConfig.Handlers) sort.Slice(mounts, func(i, j int) bool { return len(mounts[i]) < len(mounts[j]) }) - for _, m := range mounts { - h := sc.Web[hp].Handlers[m] - t, d := srvTypeAndDesc(h) - output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, dnsName, portPart, m)) + t, d := srvTypeAndDesc(webConfig.Handlers[m]) + output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, host, portPart, m)) output.WriteString(fmt.Sprintf("%s %-5s %s\n\n", "|--", t, d)) } - } else if sc.TCP[srvPort] != nil { - h := sc.TCP[srvPort] + } else if tcpHandler != nil { tlsStatus := "TLS over TCP" - if h.TerminateTLS != "" { + if tcpHandler.TerminateTLS != "" { tlsStatus = "TLS terminated" } + if ver := tcpHandler.ProxyProtocol; ver != 0 { + tlsStatus = fmt.Sprintf("%s, PROXY protocol v%d", tlsStatus, ver) + } - output.WriteString(fmt.Sprintf("%s://%s%s\n", scheme, dnsName, portPart)) - output.WriteString(fmt.Sprintf("|-- tcp://%s (%s)\n", hp, tlsStatus)) - for _, a := range st.TailscaleIPs { + output.WriteString(fmt.Sprintf("|-- tcp://%s:%d (%s)\n", host, srvPort, tlsStatus)) + for _, a := range ips { ipp := net.JoinHostPort(a.String(), strconv.Itoa(int(srvPort))) output.WriteString(fmt.Sprintf("|-- tcp://%s\n", ipp)) } - output.WriteString(fmt.Sprintf("|--> tcp://%s\n", h.TCPForward)) + output.WriteString(fmt.Sprintf("|--> tcp://%s\n\n", tcpHandler.TCPForward)) } - if !e.bg { + if !forService && !e.bg.Value { output.WriteString(msgToExit) return output.String() } @@ -478,14 +1072,90 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN output.WriteString(fmt.Sprintf(msgRunningInBackground, subCmdUpper)) output.WriteString("\n") - output.WriteString(fmt.Sprintf(msgDisableProxy, subCmd, srvType.String(), srvPort)) + if forService { + output.WriteString(fmt.Sprintf(msgDisableServiceProxy, dnsName, srvType.String(), srvPort)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableService, dnsName)) + } else { + output.WriteString(fmt.Sprintf(msgDisableProxy, subCmd, srvType.String(), srvPort)) + } return output.String() } -func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, useTLS bool, mount, target string) error { - h := new(ipn.HTTPHandler) +// isRemote reports whether the given destination from serve config +// is a remote destination. +func isRemote(target string) bool { + // target being a port number means it's localhost + if _, err := strconv.ParseUint(target, 10, 16); err == nil { + return false + } + // prepend tmp:// if no scheme is present just to help parsing + if !strings.Contains(target, "://") { + target = "tmp://" + target + } + + // make sure we can parse the target, whether it's a full URL or just a host:port + u, err := url.ParseRequestURI(target) + if err != nil { + // If we can't parse the target, it doesn't matter if it's remote or not + return false + } + validHN := dnsname.ValidHostname(u.Hostname()) == nil + validIP := net.ParseIP(u.Hostname()) != nil + if !validHN && !validIP { + return false + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" || u.Hostname() == "::1" { + return false + } + return true +} + +// shouldWarnRemoteDestCompatibility reports whether we should warn the user +// that their current OS/environment may not be compatible with +// service's proxy destination. +func (e *serveEnv) shouldWarnRemoteDestCompatibility(ctx context.Context, target string) error { + // no target means nothing to check + if target == "" { + return nil + } + + if filepath.IsAbs(target) || strings.HasPrefix(target, "text:") { + // local path or text target, nothing to check + return nil + } + + // only check for remote destinations + if !isRemote(target) { + return nil + } + + // Check if running as Mac extension and warn + if version.IsMacAppStore() || version.IsMacSysExt() { + return fmt.Errorf(msgWarnRemoteDestCompatibility, "the MacOS extension") + } + + // Check for linux, if it's running with TS_FORCE_LINUX_BIND_TO_DEVICE=true + // and tailscale bypass mark is not working. If any of these conditions are true, and the dest is + // a remote destination, return true. + if runtime.GOOS == "linux" { + SOMarkInUse, err := e.lc.CheckSOMarkInUse(ctx) + if err != nil { + log.Printf("error checking SO mark in use: %v", err) + return nil + } + if !SOMarkInUse { + return fmt.Errorf(msgWarnRemoteDestCompatibility, "the Linux tailscaled without SO_MARK") + } + } + + return nil +} + +func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, useTLS bool, mount, target, mds string, caps []tailcfg.PeerCapability) error { + h := new(ipn.HTTPHandler) switch { case strings.HasPrefix(target, "text:"): text := strings.TrimPrefix(target, "text:") @@ -513,24 +1183,27 @@ func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort ui } h.Path = target default: - t, err := ipn.ExpandProxyTargetValue(target, []string{"http", "https", "https+insecure"}, "http") + // Include unix in supported schemes for HTTP(S) serve + t, err := ipn.ExpandProxyTargetValue(target, []string{"http", "https", "https+insecure", "unix"}, "http") if err != nil { return err } h.Proxy = t + h.AcceptAppCaps = caps } // TODO: validation needs to check nested foreground configs - if sc.IsTCPForwardingOnPort(srvPort) { + svcName := tailcfg.AsServiceName(dnsName) + if sc.IsTCPForwardingOnPort(srvPort, svcName) { return errors.New("cannot serve web; already serving TCP") } - sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS) + sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS, mds) return nil } -func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType serveType, srcPort uint16, target string) error { +func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType serveType, srcPort uint16, target string, mds string, proxyProtocol int) error { var terminateTLS bool switch srcType { case serveTypeTCP: @@ -541,6 +1214,8 @@ func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType se return fmt.Errorf("invalid TCP target %q", target) } + svcName := tailcfg.AsServiceName(dnsName) + targetURL, err := ipn.ExpandProxyTargetValue(target, []string{"tcp"}, "tcp") if err != nil { return fmt.Errorf("unable to expand target: %v", err) @@ -551,13 +1226,23 @@ func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType se return fmt.Errorf("invalid TCP target %q: %v", target, err) } + if sc.IsServingWeb(srcPort, svcName) { + return fmt.Errorf("cannot serve TCP; already serving web on %d for %s", srcPort, dnsName) + } + // TODO: needs to account for multiple configs from foreground mode - if sc.IsServingWeb(srcPort) { - return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) + if svcName := tailcfg.AsServiceName(dnsName); svcName != "" { + sc.SetTCPForwardingForService(srcPort, dstURL.Host, terminateTLS, svcName, proxyProtocol, mds) + return nil } - sc.SetTCPForwarding(srcPort, dstURL.Host, terminateTLS, dnsName) + // TODO: needs to account for multiple configs from foreground mode + if svcName != "" { + sc.SetTCPForwardingForService(srcPort, dstURL.Host, terminateTLS, svcName, proxyProtocol, mds) + return nil + } + sc.SetTCPForwarding(srcPort, dstURL.Host, terminateTLS, proxyProtocol, dnsName) return nil } @@ -577,18 +1262,25 @@ func (e *serveEnv) applyFunnel(sc *ipn.ServeConfig, dnsName string, srvPort uint } // unsetServe removes the serve config for the given serve port. -func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string) error { +// dnsName is a FQDN or a serviceName (with `svc:` prefix). mds +// is the Magic DNS suffix, which is used to recreate serve's host. +func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, mds string) error { switch srvType { case serveTypeHTTPS, serveTypeHTTP: - err := e.removeWebServe(sc, dnsName, srvPort, mount) + err := e.removeWebServe(sc, dnsName, srvPort, mount, mds) if err != nil { return fmt.Errorf("failed to remove web serve: %w", err) } case serveTypeTCP, serveTypeTLSTerminatedTCP: - err := e.removeTCPServe(sc, srvPort) + err := e.removeTCPServe(sc, dnsName, srvPort) if err != nil { return fmt.Errorf("failed to remove TCP serve: %w", err) } + case serveTypeTUN: + err := e.removeTunServe(sc, dnsName) + if err != nil { + return fmt.Errorf("failed to remove TUN serve: %w", err) + } default: return fmt.Errorf("invalid type %q", srvType) } @@ -619,11 +1311,16 @@ func srvTypeAndPortFromFlags(e *serveEnv) (srvType serveType, srvPort uint16, er } } + if e.tun { + srcTypeCount++ + srvType = serveTypeTUN + } + if srcTypeCount > 1 { return 0, 0, fmt.Errorf("cannot serve multiple types for a single mount point") - } else if srcTypeCount == 0 { - srvType = serveTypeHTTPS - srvPort = 443 + } + if srcTypeCount == 0 { + return serveTypeHTTPS, 443, nil } return srvType, srvPort, nil @@ -726,59 +1423,100 @@ func isLegacyInvocation(subcmd serveMode, args []string) (string, bool) { // removeWebServe removes a web handler from the serve config // and removes funnel if no remaining mounts exist for the serve port. // The srvPort argument is the serving port and the mount argument is -// the mount point or registered path to remove. -func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, mount string) error { - if sc.IsTCPForwardingOnPort(srvPort) { - return errors.New("cannot remove web handler; currently serving TCP") +// the mount point or registered path to remove. mds is the Magic DNS suffix, +// which is used to recreate serve's host. +func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, mount string, mds string) error { + if sc == nil { + return nil } portStr := strconv.Itoa(int(srvPort)) - hp := ipn.HostPort(net.JoinHostPort(dnsName, portStr)) + hostName := dnsName + webServeMap := sc.Web + svcName := tailcfg.AsServiceName(dnsName) + forService := svcName != noService + if forService { + svc := sc.Services[svcName] + if svc == nil { + return errors.New("service does not exist") + } + hostName = strings.Join([]string{svcName.WithoutPrefix(), mds}, ".") + webServeMap = svc.Web + } + hp := ipn.HostPort(net.JoinHostPort(hostName, portStr)) + + if sc.IsTCPForwardingOnPort(srvPort, svcName) { + return errors.New("cannot remove web handler; currently serving TCP") + } var targetExists bool var mounts []string // mount is deduced from e.setPath but it is ambiguous as // to whether the user explicitly passed "/" or it was defaulted to. if e.setPath == "" { - targetExists = sc.Web[hp] != nil && len(sc.Web[hp].Handlers) > 0 + targetExists = webServeMap[hp] != nil && len(webServeMap[hp].Handlers) > 0 if targetExists { - for mount := range sc.Web[hp].Handlers { + for mount := range webServeMap[hp].Handlers { mounts = append(mounts, mount) } } } else { - targetExists = sc.WebHandlerExists(hp, mount) + targetExists = sc.WebHandlerExists(svcName, hp, mount) mounts = []string{mount} } if !targetExists { - return errors.New("error: handler does not exist") + return errors.New("handler does not exist") } if len(mounts) > 1 { msg := fmt.Sprintf("Are you sure you want to delete %d handlers under port %s?", len(mounts), portStr) - if !e.yes && !promptYesNo(msg) { + if !e.yes && !prompt.YesNo(msg, true) { return nil } } - sc.RemoveWebHandler(dnsName, srvPort, mounts, true) + if forService { + sc.RemoveServiceWebHandler(svcName, hostName, srvPort, mounts) + } else { + sc.RemoveWebHandler(dnsName, srvPort, mounts, true) + } return nil } // removeTCPServe removes the TCP forwarding configuration for the -// given srvPort, or serving port. -func (e *serveEnv) removeTCPServe(sc *ipn.ServeConfig, src uint16) error { +// given srvPort, or serving port for the given dnsName. +func (e *serveEnv) removeTCPServe(sc *ipn.ServeConfig, dnsName string, src uint16) error { if sc == nil { return nil } - if sc.GetTCPPortHandler(src) == nil { - return errors.New("error: serve config does not exist") + svcName := tailcfg.AsServiceName(dnsName) + if sc.GetTCPPortHandler(src, svcName) == nil { + return errors.New("serve config does not exist") } - if sc.IsServingWeb(src) { + if sc.IsServingWeb(src, svcName) { return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", src) } - sc.RemoveTCPForwarding(src) + sc.RemoveTCPForwarding(svcName, src) + return nil +} + +func (e *serveEnv) removeTunServe(sc *ipn.ServeConfig, dnsName string) error { + if sc == nil { + return nil + } + svcName := tailcfg.ServiceName(dnsName) + svc, ok := sc.Services[svcName] + if !ok || svc == nil { + return errors.New("service does not exist") + } + if !svc.Tun { + return errors.New("service is not being served in TUN mode") + } + delete(sc.Services, svcName) + if len(sc.Services) == 0 { + sc.Services = nil // clean up empty map + } return nil } diff --git a/cmd/tailscale/cli/serve_v2_test.go b/cmd/tailscale/cli/serve_v2_test.go index 5768127ad0421..1d2a8ef86a2e4 100644 --- a/cmd/tailscale/cli/serve_v2_test.go +++ b/cmd/tailscale/cli/serve_v2_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -8,9 +8,12 @@ import ( "context" "encoding/json" "fmt" + "net/netip" "os" "path/filepath" "reflect" + "regexp" + "slices" "strconv" "strings" "testing" @@ -19,6 +22,8 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/views" ) func TestServeDevConfigMutations(t *testing.T) { @@ -30,10 +35,11 @@ func TestServeDevConfigMutations(t *testing.T) { } // group is a group of steps that share the same - // config mutation, but always starts from an empty config + // config mutation type group struct { - name string - steps []step + name string + steps []step + initialState fakeLocalServeClient // use the zero value for empty config } // creaet a temporary directory for path-based destinations @@ -214,10 +220,20 @@ func TestServeDevConfigMutations(t *testing.T) { }}, }, { - name: "invalid_host", + name: "ip_host", + initialState: fakeLocalServeClient{ + SOMarkInUse: true, + }, steps: []step{{ - command: cmd("serve --https=443 --bg http://somehost:3000"), // invalid host - wantErr: anyErr(), + command: cmd("serve --https=443 --bg http://192.168.1.1:3000"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://192.168.1.1:3000"}, + }}, + }, + }, }}, }, { @@ -227,6 +243,16 @@ func TestServeDevConfigMutations(t *testing.T) { wantErr: anyErr(), }}, }, + { + name: "no_scheme_remote_host_tcp", + initialState: fakeLocalServeClient{ + SOMarkInUse: true, + }, + steps: []step{{ + command: cmd("serve --https=443 --bg 192.168.1.1:3000"), + wantErr: exactErrMsg(errHelp), + }}, + }, { name: "turn_off_https", steps: []step{ @@ -396,15 +422,11 @@ func TestServeDevConfigMutations(t *testing.T) { }, }}, }, - { - name: "unknown_host_tcp", - steps: []step{{ - command: cmd("serve --tls-terminated-tcp=443 --bg tcp://somehost:5432"), - wantErr: exactErrMsg(errHelp), - }}, - }, { name: "tcp_port_too_low", + initialState: fakeLocalServeClient{ + SOMarkInUse: true, + }, steps: []step{{ command: cmd("serve --tls-terminated-tcp=443 --bg tcp://somehost:0"), wantErr: exactErrMsg(errHelp), @@ -412,6 +434,9 @@ func TestServeDevConfigMutations(t *testing.T) { }, { name: "tcp_port_too_high", + initialState: fakeLocalServeClient{ + SOMarkInUse: true, + }, steps: []step{{ command: cmd("serve --tls-terminated-tcp=443 --bg tcp://somehost:65536"), wantErr: exactErrMsg(errHelp), @@ -526,6 +551,9 @@ func TestServeDevConfigMutations(t *testing.T) { }, { name: "bad_path", + initialState: fakeLocalServeClient{ + SOMarkInUse: true, + }, steps: []step{{ command: cmd("serve --bg --https=443 bad/path"), wantErr: exactErrMsg(errHelp), @@ -792,36 +820,186 @@ func TestServeDevConfigMutations(t *testing.T) { }, }, { - name: "forground_with_bg_conflict", + name: "advertise_service", + initialState: fakeLocalServeClient{ + statusWithoutPeers: &ipnstate.Status{ + BackendState: ipn.Running.String(), + Self: &ipnstate.PeerStatus{ + DNSName: "foo.test.ts.net", + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrFunnel: nil, + tailcfg.CapabilityFunnelPorts + "?ports=443,8443": nil, + }, + Tags: ptrToReadOnlySlice([]string{"some-tag"}), + }, + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + SOMarkInUse: true, + }, + steps: []step{{ + command: cmd("serve --service=svc:foo --http=80 text:foo"), + want: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Text: "foo"}, + }}, + }, + }, + }, + }, + }}, + }, + { + name: "advertise_service_from_untagged_node", + steps: []step{{ + command: cmd("serve --service=svc:foo --http=80 text:foo"), + wantErr: anyErr(), + }}, + }, + { + name: "forward_grant_header", steps: []step{ { - command: cmd("serve --bg --http=3000 localhost:3000"), + command: cmd("serve --bg --accept-app-caps=example.com/cap/foo 3000"), want: &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{3000: {HTTP: true}}, + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "foo.test.ts.net:3000": {Handlers: map[string]*ipn.HTTPHandler{ - "/": {Proxy: "http://localhost:3000"}, + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": { + Proxy: "http://127.0.0.1:3000", + AcceptAppCaps: []tailcfg.PeerCapability{"example.com/cap/foo"}, + }, + }}, + }, + }, + }, + { + command: cmd("serve --bg --accept-app-caps=example.com/cap/foo,example.com/cap/bar 3000"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": { + Proxy: "http://127.0.0.1:3000", + AcceptAppCaps: []tailcfg.PeerCapability{"example.com/cap/foo", "example.com/cap/bar"}, + }, + }}, + }, + }, + }, + { + command: cmd("serve --bg --accept-app-caps=example.com/cap/bar 3000"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": { + Proxy: "http://127.0.0.1:3000", + AcceptAppCaps: []tailcfg.PeerCapability{"example.com/cap/bar"}, + }, }}, }, }, }, + }, + }, + { + name: "invalid_accept_caps_invalid_app_cap", + steps: []step{ + { + command: cmd("serve --bg --accept-app-caps=example.com/cap/fine,NOTFINE 3000"), // should be {domain.tld}/{name} + wantErr: func(err error) (badErrMsg string) { + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("%q does not match", "NOTFINE")) { + return fmt.Sprintf("wanted validation error that quotes the non-matching capability (and nothing more) but got %q", err.Error()) + } + return "" + }, + }, + }, + }, + { + name: "tcp_with_proxy_protocol_v1", + steps: []step{{ + command: cmd("serve --tcp=8000 --proxy-protocol=1 --bg tcp://localhost:5432"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 8000: { + TCPForward: "localhost:5432", + ProxyProtocol: 1, + }, + }, + }, + }}, + }, + { + name: "tls_terminated_tcp_with_proxy_protocol_v2", + steps: []step{{ + command: cmd("serve --tls-terminated-tcp=443 --proxy-protocol=2 --bg tcp://localhost:5432"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + TCPForward: "localhost:5432", + TerminateTLS: "foo.test.ts.net", + ProxyProtocol: 2, + }, + }, + }, + }}, + }, + { + name: "tcp_update_to_add_proxy_protocol", + steps: []step{ + { + command: cmd("serve --tcp=8000 --bg tcp://localhost:5432"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 8000: {TCPForward: "localhost:5432"}, + }, + }, + }, { - command: cmd("serve --http=3000 localhost:3000"), - wantErr: exactErrMsg(fmt.Errorf(backgroundExistsMsg, "serve", "http", 3000)), + command: cmd("serve --tcp=8000 --proxy-protocol=1 --bg tcp://localhost:5432"), + want: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 8000: { + TCPForward: "localhost:5432", + ProxyProtocol: 1, + }, + }, + }, }, }, }, + { + name: "tcp_proxy_protocol_invalid_version", + steps: []step{{ + command: cmd("serve --tcp=8000 --proxy-protocol=3 --bg tcp://localhost:5432"), + wantErr: anyErr(), + }}, + }, + { + name: "proxy_protocol_without_tcp", + steps: []step{{ + command: cmd("serve --https=443 --proxy-protocol=1 --bg http://localhost:3000"), + wantErr: anyErr(), + }}, + }, } for _, group := range groups { t.Run(group.name, func(t *testing.T) { - lc := &fakeLocalServeClient{} + lc := group.initialState for i, st := range group.steps { var stderr bytes.Buffer var stdout bytes.Buffer var flagOut bytes.Buffer e := &serveEnv{ - lc: lc, + lc: &lc, testFlagOut: &flagOut, testStdout: &stdout, testStderr: &stderr, @@ -869,111 +1047,6 @@ func TestServeDevConfigMutations(t *testing.T) { } } -func TestValidateConfig(t *testing.T) { - tests := [...]struct { - name string - desc string - cfg *ipn.ServeConfig - servePort uint16 - serveType serveType - bg bool - wantErr bool - }{ - { - name: "nil_config", - desc: "when config is nil, all requests valid", - cfg: nil, - servePort: 3000, - serveType: serveTypeHTTPS, - }, - { - name: "new_bg_tcp", - desc: "no error when config exists but we're adding a new bg tcp port", - cfg: &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{ - 443: {HTTPS: true}, - }, - }, - bg: true, - servePort: 10000, - serveType: serveTypeHTTPS, - }, - { - name: "override_bg_tcp", - desc: "no error when overwriting previous port under the same serve type", - cfg: &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{ - 443: {TCPForward: "http://localhost:4545"}, - }, - }, - bg: true, - servePort: 443, - serveType: serveTypeTCP, - }, - { - name: "override_bg_tcp", - desc: "error when overwriting previous port under a different serve type", - cfg: &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{ - 443: {HTTPS: true}, - }, - }, - bg: true, - servePort: 443, - serveType: serveTypeHTTP, - wantErr: true, - }, - { - name: "new_fg_port", - desc: "no error when serving a new foreground port", - cfg: &ipn.ServeConfig{ - TCP: map[uint16]*ipn.TCPPortHandler{ - 443: {HTTPS: true}, - }, - Foreground: map[string]*ipn.ServeConfig{ - "abc123": { - TCP: map[uint16]*ipn.TCPPortHandler{ - 3000: {HTTPS: true}, - }, - }, - }, - }, - servePort: 4040, - serveType: serveTypeTCP, - }, - { - name: "same_fg_port", - desc: "error when overwriting a previous fg port", - cfg: &ipn.ServeConfig{ - Foreground: map[string]*ipn.ServeConfig{ - "abc123": { - TCP: map[uint16]*ipn.TCPPortHandler{ - 3000: {HTTPS: true}, - }, - }, - }, - }, - servePort: 3000, - serveType: serveTypeTCP, - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - se := serveEnv{bg: tc.bg} - err := se.validateConfig(tc.cfg, tc.servePort, tc.serveType) - if err == nil && tc.wantErr { - t.Fatal("expected an error but got nil") - } - if err != nil && !tc.wantErr { - t.Fatalf("expected no error but got: %v", err) - } - }) - } - -} - func TestSrcTypeFromFlags(t *testing.T) { tests := []struct { name string @@ -983,42 +1056,49 @@ func TestSrcTypeFromFlags(t *testing.T) { expectedErr bool }{ { - name: "only http set", + name: "only-http-set", env: &serveEnv{http: 80}, expectedType: serveTypeHTTP, expectedPort: 80, expectedErr: false, }, { - name: "only https set", + name: "only-https-set", env: &serveEnv{https: 10000}, expectedType: serveTypeHTTPS, expectedPort: 10000, expectedErr: false, }, { - name: "only tcp set", + name: "only-tcp-set", env: &serveEnv{tcp: 8000}, expectedType: serveTypeTCP, expectedPort: 8000, expectedErr: false, }, { - name: "only tls-terminated-tcp set", + name: "only-tls-terminated-tcp-set", env: &serveEnv{tlsTerminatedTCP: 8080}, expectedType: serveTypeTLSTerminatedTCP, expectedPort: 8080, expectedErr: false, }, { - name: "defaults to https, port 443", + name: "defaults-to-https-443", env: &serveEnv{}, expectedType: serveTypeHTTPS, expectedPort: 443, expectedErr: false, }, { - name: "multiple types set", + name: "defaults-to-https-443-for-service", + env: &serveEnv{service: "svc:foo"}, + expectedType: serveTypeHTTPS, + expectedPort: 443, + expectedErr: false, + }, + { + name: "multiple-types-set", env: &serveEnv{http: 80, https: 443}, expectedPort: 0, expectedErr: true, @@ -1041,21 +1121,134 @@ func TestSrcTypeFromFlags(t *testing.T) { } } +func TestAcceptSetAppCapsFlag(t *testing.T) { + testCases := []struct { + name string + inputs []string + expectErr bool + expectErrToMatch *regexp.Regexp + expectedValue []tailcfg.PeerCapability + }{ + { + name: "valid_simple", + inputs: []string{"example.com/name"}, + expectErr: false, + expectedValue: []tailcfg.PeerCapability{"example.com/name"}, + }, + { + name: "valid_unicode", + inputs: []string{"bÃŧcher.de/something"}, + expectErr: false, + expectedValue: []tailcfg.PeerCapability{"bÃŧcher.de/something"}, + }, + { + name: "more_valid_unicode", + inputs: []string{"example.tw/某某某"}, + expectErr: false, + expectedValue: []tailcfg.PeerCapability{"example.tw/某某某"}, + }, + { + name: "valid_path_slashes", + inputs: []string{"domain.com/path/to/name"}, + expectErr: false, + expectedValue: []tailcfg.PeerCapability{"domain.com/path/to/name"}, + }, + { + name: "valid_multiple_sets", + inputs: []string{"one.com/foo,two.com/bar"}, + expectErr: false, + expectedValue: []tailcfg.PeerCapability{"one.com/foo", "two.com/bar"}, + }, + { + name: "valid_empty_string", + inputs: []string{""}, + expectErr: false, + expectedValue: nil, // Empty string should be a no-op and not append anything. + }, + { + name: "invalid_path_chars", + inputs: []string{"domain.com/path_with_underscore"}, + expectErr: true, + expectErrToMatch: regexp.MustCompile(`"domain.com/path_with_underscore"`), + expectedValue: nil, // Slice should remain empty. + }, + { + name: "valid_subdomain", + inputs: []string{"sub.domain.com/name"}, + expectErr: false, + expectedValue: []tailcfg.PeerCapability{"sub.domain.com/name"}, + }, + { + name: "invalid_no_path", + inputs: []string{"domain.com/"}, + expectErr: true, + expectErrToMatch: regexp.MustCompile(`"domain.com/"`), + expectedValue: nil, + }, + { + name: "invalid_no_domain", + inputs: []string{"/path/only"}, + expectErr: true, + expectErrToMatch: regexp.MustCompile(`"/path/only"`), + expectedValue: nil, + }, + { + name: "some_invalid_some_valid", + inputs: []string{"one.com/foo,bad/bar,two.com/baz"}, + expectErr: true, + expectErrToMatch: regexp.MustCompile(`"bad/bar"`), + expectedValue: []tailcfg.PeerCapability{"one.com/foo"}, // Parsing will stop after first error + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var v []tailcfg.PeerCapability + flag := &acceptAppCapsFlag{Value: &v} + + var err error + for _, s := range tc.inputs { + err = flag.Set(s) + if err != nil { + break + } + } + + if tc.expectErr && err == nil { + t.Errorf("expected an error, but got none") + } + if tc.expectErrToMatch != nil { + if !tc.expectErrToMatch.MatchString(err.Error()) { + t.Errorf("expected error to match %q, but was %q", tc.expectErrToMatch, err) + } + } + if !tc.expectErr && err != nil { + t.Errorf("did not expect an error, but got: %v", err) + } + + if !reflect.DeepEqual(tc.expectedValue, v) { + t.Errorf("unexpected value, got: %q, want: %q", v, tc.expectedValue) + } + }) + } +} + func TestCleanURLPath(t *testing.T) { tests := []struct { + name string input string expected string wantErr bool }{ - {input: "", expected: "/"}, - {input: "/", expected: "/"}, - {input: "/foo", expected: "/foo"}, - {input: "/foo/", expected: "/foo/"}, - {input: "/../bar", wantErr: true}, + {name: "empty", input: "", expected: "/"}, + {name: "slash", input: "/", expected: "/"}, + {name: "foo", input: "/foo", expected: "/foo"}, + {name: "foo-trailing-slash", input: "/foo/", expected: "/foo/"}, + {name: "dotdot-bar", input: "/../bar", wantErr: true}, } for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { actual, err := cleanURLPath(tt.input) if tt.wantErr == true && err == nil { @@ -1075,12 +1268,118 @@ func TestCleanURLPath(t *testing.T) { } } +func TestAddServiceToPrefs(t *testing.T) { + tests := []struct { + name string + svcName tailcfg.ServiceName + startServices []string + expected []string + }{ + { + name: "add-service-to-empty-prefs", + svcName: "svc:foo", + expected: []string{"svc:foo"}, + }, + { + name: "add-service-to-existing-prefs", + svcName: "svc:bar", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo", "svc:bar"}, + }, + { + name: "add-existing-service-to-prefs", + svcName: "svc:foo", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lc := &fakeLocalServeClient{} + ctx := t.Context() + lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: tt.startServices, + }, + }) + e := &serveEnv{lc: lc, bg: bgBoolFlag{true, false}} + err := e.addServiceToPrefs(ctx, tt.svcName) + if err != nil { + t.Fatalf("addServiceToPrefs(%q) returned unexpected error: %v", tt.svcName, err) + } + if !slices.Equal(lc.prefs.AdvertiseServices, tt.expected) { + t.Errorf("addServiceToPrefs(%q) = %v, want %v", tt.svcName, lc.prefs.AdvertiseServices, tt.expected) + } + }) + } + +} + +func TestRemoveServiceFromPrefs(t *testing.T) { + tests := []struct { + name string + svcName tailcfg.ServiceName + startServices []string + expected []string + }{ + { + name: "remove-service-from-empty-prefs", + svcName: "svc:foo", + expected: []string{}, + }, + { + name: "remove-existing-service-from-prefs", + svcName: "svc:foo", + startServices: []string{"svc:foo"}, + expected: []string{}, + }, + { + name: "remove-service-not-in-prefs", + svcName: "svc:bar", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lc := &fakeLocalServeClient{} + ctx := t.Context() + lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: tt.startServices, + }, + }) + e := &serveEnv{lc: lc, bg: bgBoolFlag{true, false}} + err := e.removeServiceFromPrefs(ctx, tt.svcName) + if err != nil { + t.Fatalf("removeServiceFromPrefs(%q) returned unexpected error: %v", tt.svcName, err) + } + if !slices.Equal(lc.prefs.AdvertiseServices, tt.expected) { + t.Errorf("removeServiceFromPrefs(%q) = %v, want %v", tt.svcName, lc.prefs.AdvertiseServices, tt.expected) + } + }) + } +} + func TestMessageForPort(t *testing.T) { + svcIPMap := tailcfg.ServiceIPMappings{ + "svc:foo": []netip.Addr{ + netip.MustParseAddr("100.101.101.101"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:6565:6565"), + }, + } + svcIPMapJSON, _ := json.Marshal(svcIPMap) + svcIPMapJSONRawMSG := tailcfg.RawMessage(svcIPMapJSON) + tests := []struct { name string subcmd serveMode serveConfig *ipn.ServeConfig status *ipnstate.Status + prefs *ipn.Prefs dnsName string srvType serveType srvPort uint16 @@ -1104,7 +1403,7 @@ func TestMessageForPort(t *testing.T) { "foo.test.ts.net:443": true, }, }, - status: &ipnstate.Status{}, + status: &ipnstate.Status{CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}}, dnsName: "foo.test.ts.net", srvType: serveTypeHTTPS, srvPort: 443, @@ -1133,7 +1432,7 @@ func TestMessageForPort(t *testing.T) { }, }, }, - status: &ipnstate.Status{}, + status: &ipnstate.Status{CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}}, dnsName: "foo.test.ts.net", srvType: serveTypeHTTP, srvPort: 80, @@ -1147,10 +1446,206 @@ func TestMessageForPort(t *testing.T) { fmt.Sprintf(msgDisableProxy, "serve", "http", 80), }, "\n"), }, + { + name: "serve-service-http", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{ + AdvertiseServices: []string{"svc:foo"}, + }, + dnsName: "svc:foo", + srvType: serveTypeHTTP, + srvPort: 80, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "http://foo.test.ts.net/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "http", 80), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve-service-no-capmap", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{ + AdvertiseServices: []string{"svc:bar"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + expected: strings.Join([]string{ + fmt.Sprintf(msgServiceWaitingApproval, "svc:bar"), + "", + "http://bar.test.ts.net/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:bar", "http", 80), + fmt.Sprintf(msgDisableService, "svc:bar"), + }, "\n"), + }, + { + name: "serve-service-https-non-default-port", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 2200: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:2200": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeHTTPS, + srvPort: 2200, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "https://foo.test.ts.net:2200/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "https", 2200), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve-service-TCPForward", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 2200: {TCPForward: "localhost:3000"}, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeTCP, + srvPort: 2200, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "|-- tcp://foo.test.ts.net:2200 (TLS over TCP)", + "|-- tcp://100.101.101.101:2200", + "|-- tcp://[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:2200", + "|--> tcp://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "tcp", 2200), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve-service-Tun", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + Tun: true, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeTUN, + expected: strings.Join([]string{ + msgServeAvailable, + "", + fmt.Sprintf(msgRunningTunService, "foo.test.ts.net"), + fmt.Sprintf(msgDisableServiceTun, "svc:foo"), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, } for _, tt := range tests { - e := &serveEnv{bg: true, subcmd: tt.subcmd} + e := &serveEnv{bg: bgBoolFlag{true, false}, subcmd: tt.subcmd} t.Run(tt.name, func(t *testing.T) { actual := e.messageForPort(tt.serveConfig, tt.status, tt.dnsName, tt.srvType, tt.srvPort) @@ -1271,7 +1766,581 @@ func TestIsLegacyInvocation(t *testing.T) { } if gotTranslation != tt.translation { - t.Fatalf("expected translaction to be %q but got %q", tt.translation, gotTranslation) + t.Fatalf("expected translation to be %q but got %q", tt.translation, gotTranslation) + } + }) + } +} + +func TestSetServe(t *testing.T) { + e := &serveEnv{} + magicDNSSuffix := "test.ts.net" + tests := []struct { + name string + desc string + cfg *ipn.ServeConfig + st *ipnstate.Status + dnsName string + srvType serveType + srvPort uint16 + mountPath string + target string + allowFunnel bool + proxyProtocol int + expected *ipn.ServeConfig + expectErr bool + }{ + { + name: "add-new-handler", + desc: "add a new http handler to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3000", + expected: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + { + name: "update-http-handler", + desc: "update an existing http handler on the same port to same type", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + { + name: "update-TCP-handler", + desc: "update an existing TCP handler on the same port to a http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "http://localhost:3000"}}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expectErr: true, + }, + { + name: "add-new-service-handler", + desc: "add a new service TCP handler to empty config", + cfg: &ipn.ServeConfig{}, + + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + target: "3000", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + }, + { + name: "update-service-handler", + desc: "update an existing service TCP handler on the same port to same type", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + target: "3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3001"}}, + }, + }, + }, + }, + { + name: "update-service-handler", + desc: "update an existing service TCP handler on the same port to a http handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expectErr: true, + }, + { + name: "add-new-service-handler", + desc: "add a new service HTTP handler to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3000", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "update-existing-service-handler", + desc: "update an existing service HTTP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add-new-service-handler", + desc: "add a new service HTTP handler to existing service config", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 88, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 88: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + "bar.test.ts.net:88": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add-new-service-mount", + desc: "add a new service mount to existing service config", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/added", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + "/added": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add-new-service-handler", + desc: "add a new service handler in tun mode to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "svc:bar", + srvType: serveTypeTUN, + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + Tun: true, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := e.setServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mountPath, tt.target, tt.allowFunnel, magicDNSSuffix, nil, tt.proxyProtocol) + if err != nil && !tt.expectErr { + t.Fatalf("got error: %v; did not expect error.", err) + } + if err == nil && tt.expectErr { + t.Fatalf("got no error; expected error.") + } + if !tt.expectErr { + if diff := cmp.Diff(tt.expected, tt.cfg); diff != "" { + // svcName := tailcfg.ServiceName(tt.dnsName) + t.Fatalf("got diff:\n%s", diff) + } + } + }) + } +} + +func TestUnsetServe(t *testing.T) { + tests := []struct { + name string + desc string + cfg *ipn.ServeConfig + st *ipnstate.Status + dnsName string + srvType serveType + srvPort uint16 + mount string + setServeEnv bool + serveEnv *serveEnv // if set, use this instead of the default serveEnv + expected *ipn.ServeConfig + expectErr bool + }{ + { + name: "unset-http-handler", + desc: "remove an existing http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/", + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset-service-handler", + desc: "remove an existing service TCP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/", + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset-service-handler-tun", + desc: "remove an existing service handler in tun mode", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + Tun: true, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTUN, + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset-service-handler-tcp", + desc: "remove an existing service TCP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {TCPForward: "11.11.11.11:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset-http-handler-not-found", + desc: "try to remove a non-existing http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "bar.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/abc", + expected: &ipn.ServeConfig{}, + expectErr: true, + }, + { + name: "unset-service-handler-not-found", + desc: "try to remove a non-existing service TCP handler", + + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/abc", + setServeEnv: true, + serveEnv: &serveEnv{setPath: "/abc"}, + expected: &ipn.ServeConfig{}, + expectErr: true, + }, + { + name: "unset-service-doesnt-exist", + desc: "try to remove a non-existing service's handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {TCPForward: "11.11.11.11:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:foo", + srvType: serveTypeTCP, + srvPort: 80, + expectErr: true, + }, + { + name: "unset-tcp-while-port-in-use", + desc: "try to remove a TCP handler while the port is used for web", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeTCP, + srvPort: 80, + mount: "/", + expectErr: true, + }, + { + name: "unset-service-tcp-while-port-in-use", + desc: "try to remove a service TCP handler while the port is used for web", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + mount: "/", + expectErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &serveEnv{} + if tt.setServeEnv { + e = tt.serveEnv + } + err := e.unsetServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mount, tt.st.CurrentTailnet.MagicDNSSuffix) + if err != nil && !tt.expectErr { + t.Fatalf("got error: %v; did not expect error.", err) + } + if err == nil && tt.expectErr { + t.Fatalf("got no error; expected error.") + } + if !tt.expectErr && !reflect.DeepEqual(tt.cfg, tt.expected) { + t.Fatalf("got: %v; expected: %v", tt.cfg, tt.expected) } }) } @@ -1287,3 +2356,8 @@ func exactErrMsg(want error) func(error) string { return fmt.Sprintf("\ngot: %v\nwant: %v\n", got, want) } } + +func ptrToReadOnlySlice[T any](s []T) *views.Slice[T] { + vs := views.SliceOf(s) + return &vs +} diff --git a/cmd/tailscale/cli/serve_v2_unix_test.go b/cmd/tailscale/cli/serve_v2_unix_test.go new file mode 100644 index 0000000000000..671cdfbfbd1bc --- /dev/null +++ b/cmd/tailscale/cli/serve_v2_unix_test.go @@ -0,0 +1,86 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package cli + +import ( + "path/filepath" + "testing" + + "tailscale.com/ipn" +) + +func TestServeUnixSocketCLI(t *testing.T) { + // Create a temporary directory for our socket path + tmpDir := t.TempDir() + socketPath := filepath.Join(tmpDir, "test.sock") + + // Test that Unix socket targets are accepted by ExpandProxyTargetValue + target := "unix:" + socketPath + result, err := ipn.ExpandProxyTargetValue(target, []string{"http", "https", "https+insecure", "unix"}, "http") + if err != nil { + t.Fatalf("ExpandProxyTargetValue failed: %v", err) + } + + if result != target { + t.Errorf("ExpandProxyTargetValue(%q) = %q, want %q", target, result, target) + } +} + +func TestServeUnixSocketConfigPreserved(t *testing.T) { + // Test that Unix socket URLs are preserved in ServeConfig + sc := &ipn.ServeConfig{ + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:443": {Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "unix:/tmp/test.sock"}, + }}, + }, + } + + // Verify the proxy value is preserved + handler := sc.Web["foo.test.ts.net:443"].Handlers["/"] + if handler.Proxy != "unix:/tmp/test.sock" { + t.Errorf("proxy = %q, want %q", handler.Proxy, "unix:/tmp/test.sock") + } +} + +func TestServeUnixSocketVariousPaths(t *testing.T) { + tests := []struct { + name string + target string + wantErr bool + }{ + { + name: "absolute-path", + target: "unix:/var/run/docker.sock", + }, + { + name: "tmp-path", + target: "unix:/tmp/myservice.sock", + }, + { + name: "relative-path", + target: "unix:./local.sock", + }, + { + name: "home-path", + target: "unix:/home/user/.local/service.sock", + }, + { + name: "empty-path", + target: "unix:", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ipn.ExpandProxyTargetValue(tt.target, []string{"http", "https", "unix"}, "http") + if (err != nil) != tt.wantErr { + t.Errorf("ExpandProxyTargetValue(%q) error = %v, wantErr %v", tt.target, err, tt.wantErr) + } + }) + } +} diff --git a/cmd/tailscale/cli/set.go b/cmd/tailscale/cli/set.go index 66e74d77ff5a5..6fd4b09ad6790 100644 --- a/cmd/tailscale/cli/set.go +++ b/cmd/tailscale/cli/set.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -11,20 +11,21 @@ import ( "net/netip" "os/exec" "runtime" + "slices" "strconv" "strings" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/client/web" - "tailscale.com/clientupdate" "tailscale.com/cmd/tailscale/cli/ffcomplete" + "tailscale.com/feature/buildfeatures" "tailscale.com/ipn" "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" "tailscale.com/safesocket" + "tailscale.com/tsconst" "tailscale.com/types/opt" - "tailscale.com/types/ptr" "tailscale.com/types/views" + "tailscale.com/util/set" "tailscale.com/version" ) @@ -43,28 +44,30 @@ Only settings explicitly mentioned will be set. There are no default values.`, } type setArgsT struct { - acceptRoutes bool - acceptDNS bool - exitNodeIP string - exitNodeAllowLANAccess bool - shieldsUp bool - runSSH bool - runWebClient bool - hostname string - advertiseRoutes string - advertiseDefaultRoute bool - advertiseConnector bool - opUser string - acceptedRisks string - profileName string - forceDaemon bool - updateCheck bool - updateApply bool - reportPosture bool - snat bool - statefulFiltering bool - netfilterMode string - relayServerPort string + acceptRoutes bool + acceptDNS bool + exitNodeIP string + exitNodeAllowLANAccess bool + shieldsUp bool + runSSH bool + runWebClient bool + hostname string + advertiseRoutes string + advertiseDefaultRoute bool + advertiseConnector bool + opUser string + acceptedRisks string + profileName string + forceDaemon bool + updateCheck bool + updateApply bool + reportPosture bool + snat bool + statefulFiltering bool + sync bool + netfilterMode string + relayServerPort string + relayServerStaticEndpoints string } func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { @@ -73,7 +76,7 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { setf.StringVar(&setArgs.profileName, "nickname", "", "nickname for the current account") setf.BoolVar(&setArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes") setf.BoolVar(&setArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") - setf.StringVar(&setArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node") + setf.StringVar(&setArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") setf.BoolVar(&setArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") setf.BoolVar(&setArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") setf.BoolVar(&setArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") @@ -85,7 +88,9 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { setf.BoolVar(&setArgs.updateApply, "auto-update", false, "automatically update to the latest available version") setf.BoolVar(&setArgs.reportPosture, "report-posture", false, "allow management plane to gather device posture information") setf.BoolVar(&setArgs.runWebClient, "webclient", false, "expose the web interface for managing this node over Tailscale at port 5252") - setf.StringVar(&setArgs.relayServerPort, "relay-server-port", "", hidden+"UDP port number (0 will pick a random unused port) for the relay server to bind to, on all interfaces, or empty string to disable relay server functionality") + setf.BoolVar(&setArgs.sync, "sync", false, hidden+"actively sync configuration from the control plane (set to false only for network failure testing)") + setf.StringVar(&setArgs.relayServerPort, "relay-server-port", "", "UDP port number (0 will pick a random unused port) for the relay server to bind to, on all interfaces, or empty string to disable relay server functionality") + setf.StringVar(&setArgs.relayServerStaticEndpoints, "relay-server-static-endpoints", "", "static IP:port endpoints to advertise as candidates for relay connections (comma-separated, e.g. \"[2001:db8::1]:40000,192.0.2.1:40000\") or empty string to not advertise any static endpoints") ffcomplete.Flag(setf, "exit-node", func(args []string) ([]string, ffcomplete.ShellCompDirective, error) { st, err := localClient.Status(context.Background()) @@ -108,7 +113,7 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { switch goos { case "linux": setf.BoolVar(&setArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes") - setf.BoolVar(&setArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)") + setf.BoolVar(&setArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, and so on)") setf.StringVar(&setArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)") case "windows": setf.BoolVar(&setArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)") @@ -149,6 +154,7 @@ func runSet(ctx context.Context, args []string) (retErr error) { OperatorUser: setArgs.opUser, NoSNAT: !setArgs.snat, ForceDaemon: setArgs.forceDaemon, + Sync: opt.NewBool(setArgs.sync), AutoUpdate: ipn.AutoUpdatePrefs{ Check: setArgs.updateCheck, Apply: opt.NewBool(setArgs.updateApply), @@ -173,19 +179,19 @@ func runSet(ctx context.Context, args []string) (retErr error) { } if setArgs.exitNodeIP != "" { - if err := maskedPrefs.Prefs.SetExitNodeIP(setArgs.exitNodeIP, st); err != nil { - var e ipn.ExitNodeLocalIPError - if errors.As(err, &e) { + if expr, useAutoExitNode := ipn.ParseAutoExitNodeString(setArgs.exitNodeIP); useAutoExitNode { + maskedPrefs.AutoExitNode = expr + maskedPrefs.AutoExitNodeSet = true + } else if err := maskedPrefs.Prefs.SetExitNodeIP(setArgs.exitNodeIP, st); err != nil { + if _, ok := errors.AsType[ipn.ExitNodeLocalIPError](err); ok { return fmt.Errorf("%w; did you mean --advertise-exit-node?", err) } return err } } - warnOnAdvertiseRouts(ctx, &maskedPrefs.Prefs) - if err := checkExitNodeRisk(ctx, &maskedPrefs.Prefs, setArgs.acceptedRisks); err != nil { - return err - } + warnOnAdvertiseRoutes(ctx, &maskedPrefs.Prefs) + var advertiseExitNodeSet, advertiseRoutesSet bool setFlagSet.Visit(func(f *flag.Flag) { updateMaskedPrefsFromUpOrSetFlag(maskedPrefs, f.Name) @@ -223,21 +229,14 @@ func runSet(ctx context.Context, args []string) (retErr error) { return err } } - if maskedPrefs.AutoUpdateSet.ApplySet { - if !clientupdate.CanAutoUpdate() { - return errors.New("automatic updates are not supported on this platform") + if maskedPrefs.AutoUpdateSet.ApplySet && buildfeatures.HasClientUpdate && version.IsMacSysExt() { + apply := "0" + if maskedPrefs.AutoUpdate.Apply.EqualBool(true) { + apply = "1" } - // On macsys, tailscaled will set the Sparkle auto-update setting. It - // does not use clientupdate. - if version.IsMacSysExt() { - apply := "0" - if maskedPrefs.AutoUpdate.Apply.EqualBool(true) { - apply = "1" - } - out, err := exec.Command("defaults", "write", "io.tailscale.ipn.macsys", "SUAutomaticallyUpdate", apply).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to enable automatic updates: %v, %q", err, out) - } + out, err := exec.Command("defaults", "write", "io.tailscale.ipn.macsys", "SUAutomaticallyUpdate", apply).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to enable automatic updates: %v, %q", err, out) } } @@ -246,11 +245,31 @@ func runSet(ctx context.Context, args []string) (retErr error) { if err != nil { return fmt.Errorf("failed to set relay server port: %v", err) } - maskedPrefs.Prefs.RelayServerPort = ptr.To(int(uport)) + maskedPrefs.Prefs.RelayServerPort = new(uint16(uport)) + } + + if setArgs.relayServerStaticEndpoints != "" { + endpointsSet := make(set.Set[netip.AddrPort]) + endpointsSplit := strings.SplitSeq(setArgs.relayServerStaticEndpoints, ",") + for s := range endpointsSplit { + ap, err := netip.ParseAddrPort(s) + if err != nil { + return fmt.Errorf("failed to set relay server static endpoints: %q is not a valid IP:port", s) + } + endpointsSet.Add(ap) + } + endpoints := endpointsSet.Slice() + slices.SortFunc(endpoints, netip.AddrPort.Compare) + maskedPrefs.Prefs.RelayServerStaticEndpoints = endpoints } checkPrefs := curPrefs.Clone() checkPrefs.ApplyEdits(maskedPrefs) + // We want to make sure user is aware setting --snat-subnet-routes=false with --advertise-exit-node would break exitnode, + // but we won't prevent them from doing it since there are current dependencies on that combination. (as of 2026-03-25) + if checkPrefs.NoSNAT && checkPrefs.AdvertisesExitNode() { + warnf("--snat-subnet-routes=false is set with --advertise-exit-node; internet traffic through this exit node may not work as expected") + } if err := localClient.CheckPrefs(ctx, checkPrefs); err != nil { return err } @@ -261,7 +280,7 @@ func runSet(ctx context.Context, args []string) (retErr error) { } if setArgs.runWebClient && len(st.TailscaleIPs) > 0 { - printf("\nWeb interface now running at %s:%d", st.TailscaleIPs[0], web.ListenPort) + printf("\nWeb interface now running at %s:%d\n", st.TailscaleIPs[0], tsconst.WebListenPort) } return nil diff --git a/cmd/tailscale/cli/set_test.go b/cmd/tailscale/cli/set_test.go index a2f211f8cdc36..e2c3ae5f64116 100644 --- a/cmd/tailscale/cli/set_test.go +++ b/cmd/tailscale/cli/set_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -11,7 +11,6 @@ import ( "tailscale.com/ipn" "tailscale.com/net/tsaddr" - "tailscale.com/types/ptr" ) func TestCalcAdvertiseRoutesForSet(t *testing.T) { @@ -28,80 +27,80 @@ func TestCalcAdvertiseRoutesForSet(t *testing.T) { }, { name: "advertise-exit", - setExit: ptr.To(true), + setExit: new(true), want: tsaddr.ExitRoutes(), }, { name: "advertise-exit/already-routes", was: []netip.Prefix{pfx("34.0.0.0/16")}, - setExit: ptr.To(true), + setExit: new(true), want: []netip.Prefix{pfx("34.0.0.0/16"), tsaddr.AllIPv4(), tsaddr.AllIPv6()}, }, { name: "advertise-exit/already-exit", was: tsaddr.ExitRoutes(), - setExit: ptr.To(true), + setExit: new(true), want: tsaddr.ExitRoutes(), }, { name: "stop-advertise-exit", was: tsaddr.ExitRoutes(), - setExit: ptr.To(false), + setExit: new(false), want: nil, }, { name: "stop-advertise-exit/with-routes", was: []netip.Prefix{pfx("34.0.0.0/16"), tsaddr.AllIPv4(), tsaddr.AllIPv6()}, - setExit: ptr.To(false), + setExit: new(false), want: []netip.Prefix{pfx("34.0.0.0/16")}, }, { name: "advertise-routes", - setRoutes: ptr.To("10.0.0.0/24,192.168.0.0/16"), + setRoutes: new("10.0.0.0/24,192.168.0.0/16"), want: []netip.Prefix{pfx("10.0.0.0/24"), pfx("192.168.0.0/16")}, }, { name: "advertise-routes/already-exit", was: tsaddr.ExitRoutes(), - setRoutes: ptr.To("10.0.0.0/24,192.168.0.0/16"), + setRoutes: new("10.0.0.0/24,192.168.0.0/16"), want: []netip.Prefix{pfx("10.0.0.0/24"), pfx("192.168.0.0/16"), tsaddr.AllIPv4(), tsaddr.AllIPv6()}, }, { name: "advertise-routes/already-diff-routes", was: []netip.Prefix{pfx("34.0.0.0/16")}, - setRoutes: ptr.To("10.0.0.0/24,192.168.0.0/16"), + setRoutes: new("10.0.0.0/24,192.168.0.0/16"), want: []netip.Prefix{pfx("10.0.0.0/24"), pfx("192.168.0.0/16")}, }, { name: "stop-advertise-routes", was: []netip.Prefix{pfx("34.0.0.0/16")}, - setRoutes: ptr.To(""), + setRoutes: new(""), want: nil, }, { name: "stop-advertise-routes/already-exit", was: []netip.Prefix{pfx("34.0.0.0/16"), tsaddr.AllIPv4(), tsaddr.AllIPv6()}, - setRoutes: ptr.To(""), + setRoutes: new(""), want: tsaddr.ExitRoutes(), }, { name: "advertise-routes-and-exit", - setExit: ptr.To(true), - setRoutes: ptr.To("10.0.0.0/24,192.168.0.0/16"), + setExit: new(true), + setRoutes: new("10.0.0.0/24,192.168.0.0/16"), want: []netip.Prefix{pfx("10.0.0.0/24"), pfx("192.168.0.0/16"), tsaddr.AllIPv4(), tsaddr.AllIPv6()}, }, { name: "advertise-routes-and-exit/already-exit", was: tsaddr.ExitRoutes(), - setExit: ptr.To(true), - setRoutes: ptr.To("10.0.0.0/24,192.168.0.0/16"), + setExit: new(true), + setRoutes: new("10.0.0.0/24,192.168.0.0/16"), want: []netip.Prefix{pfx("10.0.0.0/24"), pfx("192.168.0.0/16"), tsaddr.AllIPv4(), tsaddr.AllIPv6()}, }, { name: "advertise-routes-and-exit/already-routes", was: []netip.Prefix{pfx("10.0.0.0/24"), pfx("192.168.0.0/16")}, - setExit: ptr.To(true), - setRoutes: ptr.To("10.0.0.0/24,192.168.0.0/16"), + setExit: new(true), + setRoutes: new("10.0.0.0/24,192.168.0.0/16"), want: []netip.Prefix{pfx("10.0.0.0/24"), pfx("192.168.0.0/16"), tsaddr.AllIPv4(), tsaddr.AllIPv6()}, }, } diff --git a/cmd/tailscale/cli/ssh.go b/cmd/tailscale/cli/ssh.go index ba70e97e9f925..eb1ce37f8b006 100644 --- a/cmd/tailscale/cli/ssh.go +++ b/cmd/tailscale/cli/ssh.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -11,9 +11,9 @@ import ( "log" "net/netip" "os" - "os/user" "path/filepath" "runtime" + "slices" "strings" "github.com/peterbourgon/ff/v3/ffcli" @@ -58,11 +58,7 @@ func runSSH(ctx context.Context, args []string) error { username, host, ok := strings.Cut(arg, "@") if !ok { host = arg - lu, err := user.Current() - if err != nil { - return nil - } - username = lu.Username + username = "" } st, err := localClient.Status(ctx) @@ -70,12 +66,28 @@ func runSSH(ctx context.Context, args []string) error { return err } + prefs, err := localClient.GetPrefs(ctx) + if err != nil { + return err + } + // hostForSSH is the hostname we'll tell OpenSSH we're // connecting to, so we have to maintain fewer entries in the // known_hosts files. hostForSSH := host - if v, ok := nodeDNSNameFromArg(st, host); ok { - hostForSSH = v + ps, ok := peerStatusFromArg(st, host) + if ok { + hostForSSH = ps.DNSName + + // If MagicDNS isn't enabled on the client, + // we will use the first IPv4 we know about + // or fallback to the first IPv6 address + if !prefs.CorpDNS { + ipHost, found := ipFromPeerStatus(ps) + if found { + hostForSSH = ipHost + } + } } ssh, err := findSSH() @@ -129,7 +141,11 @@ func runSSH(ctx context.Context, args []string) error { // to use a different one, we'll later be making stock ssh // work well by default too. (doing things like automatically // setting known_hosts, etc) - argv = append(argv, username+"@"+hostForSSH) + if username == "" { + argv = append(argv, hostForSSH) + } else { + argv = append(argv, username+"@"+hostForSSH) + } argv = append(argv, argRest...) @@ -169,11 +185,38 @@ func genKnownHosts(st *ipnstate.Status) []byte { continue } fmt.Fprintf(&buf, "%s %s\n", ps.DNSName, hostKey) + for _, ip := range ps.TailscaleIPs { + fmt.Fprintf(&buf, "%s %s\n", ip.String(), hostKey) + } } } return buf.Bytes() } +// peerStatusFromArg returns the PeerStatus that matches +// the input arg which can be a base name, full DNS name, or an IP. +func peerStatusFromArg(st *ipnstate.Status, arg string) (*ipnstate.PeerStatus, bool) { + if arg == "" { + return nil, false + } + argIP, _ := netip.ParseAddr(arg) + for _, ps := range st.Peer { + if argIP.IsValid() { + if slices.Contains(ps.TailscaleIPs, argIP) { + return ps, true + } + continue + } + if strings.EqualFold(strings.TrimSuffix(arg, "."), strings.TrimSuffix(ps.DNSName, ".")) { + return ps, true + } + if base, _, ok := strings.Cut(ps.DNSName, "."); ok && strings.EqualFold(base, arg) { + return ps, true + } + } + return nil, false +} + // nodeDNSNameFromArg returns the PeerStatus.DNSName value from a peer // in st that matches the input arg which can be a base name, full // DNS name, or an IP. @@ -185,10 +228,8 @@ func nodeDNSNameFromArg(st *ipnstate.Status, arg string) (dnsName string, ok boo for _, ps := range st.Peer { dnsName = ps.DNSName if argIP.IsValid() { - for _, ip := range ps.TailscaleIPs { - if ip == argIP { - return dnsName, true - } + if slices.Contains(ps.TailscaleIPs, argIP) { + return dnsName, true } continue } @@ -202,6 +243,20 @@ func nodeDNSNameFromArg(st *ipnstate.Status, arg string) (dnsName string, ok boo return "", false } +func ipFromPeerStatus(ps *ipnstate.PeerStatus) (string, bool) { + if len(ps.TailscaleIPs) < 1 { + return "", false + } + + // Look for a IPv4 address or default to the first IP of the list + for _, ip := range ps.TailscaleIPs { + if ip.Is4() { + return ip.String(), true + } + } + return ps.TailscaleIPs[0].String(), true +} + // getSSHClientEnvVar returns the "SSH_CLIENT" environment variable // for the current process group, if any. var getSSHClientEnvVar = func() string { diff --git a/cmd/tailscale/cli/ssh_exec.go b/cmd/tailscale/cli/ssh_exec.go index 10e52903dea64..ecfd3c4e6052a 100644 --- a/cmd/tailscale/cli/ssh_exec.go +++ b/cmd/tailscale/cli/ssh_exec.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !js && !windows diff --git a/cmd/tailscale/cli/ssh_exec_js.go b/cmd/tailscale/cli/ssh_exec_js.go index 40effc7cafc7e..bf631c3b82d24 100644 --- a/cmd/tailscale/cli/ssh_exec_js.go +++ b/cmd/tailscale/cli/ssh_exec_js.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/ssh_exec_windows.go b/cmd/tailscale/cli/ssh_exec_windows.go index e249afe667401..f9d306463c635 100644 --- a/cmd/tailscale/cli/ssh_exec_windows.go +++ b/cmd/tailscale/cli/ssh_exec_windows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -28,9 +28,8 @@ func execSSH(ssh string, argv []string) error { cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - var ee *exec.ExitError err := cmd.Run() - if errors.As(err, &ee) { + if ee, ok := errors.AsType[*exec.ExitError](err); ok { os.Exit(ee.ExitCode()) } return err diff --git a/cmd/tailscale/cli/ssh_unix.go b/cmd/tailscale/cli/ssh_unix.go index 71c0caaa69ad5..1cc3ccbe8c66f 100644 --- a/cmd/tailscale/cli/ssh_unix.go +++ b/cmd/tailscale/cli/ssh_unix.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !wasm && !windows && !plan9 @@ -39,7 +39,7 @@ func init() { return "" } prefix := []byte("SSH_CLIENT=") - for _, env := range bytes.Split(b, []byte{0}) { + for env := range bytes.SplitSeq(b, []byte{0}) { if bytes.HasPrefix(env, prefix) { return string(env[len(prefix):]) } diff --git a/cmd/tailscale/cli/status.go b/cmd/tailscale/cli/status.go index e4dccc247fd54..9ce4debda8dea 100644 --- a/cmd/tailscale/cli/status.go +++ b/cmd/tailscale/cli/status.go @@ -1,10 +1,9 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli import ( - "bytes" "cmp" "context" "encoding/json" @@ -15,12 +14,12 @@ import ( "net/http" "net/netip" "os" - "strconv" "strings" + "text/tabwriter" "github.com/peterbourgon/ff/v3/ffcli" - "github.com/toqueteos/webbrowser" "golang.org/x/net/idna" + "tailscale.com/feature" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/net/netmon" @@ -56,6 +55,7 @@ https://github.com/tailscale/tailscale/blob/main/ipn/ipnstate/ipnstate.go fs.BoolVar(&statusArgs.peers, "peers", true, "show status of peers") fs.StringVar(&statusArgs.listen, "listen", "127.0.0.1:8384", "listen address for web mode; use port 0 for automatic") fs.BoolVar(&statusArgs.browser, "browser", true, "Open a browser in web mode") + fs.BoolVar(&statusArgs.header, "header", false, "show column headers in table format") return fs })(), } @@ -68,8 +68,11 @@ var statusArgs struct { active bool // in CLI mode, filter output to only peers with active sessions self bool // in CLI mode, show status of local machine peers bool // in CLI mode, show status of peer machines + header bool // in CLI mode, show column headers in table format } +const mullvadTCD = "mullvad.ts.net." + func runStatus(ctx context.Context, args []string) error { if len(args) > 0 { return errors.New("unexpected non-flag arguments to 'tailscale status'") @@ -109,7 +112,9 @@ func runStatus(ctx context.Context, args []string) error { ln.Close() }() if statusArgs.browser { - go webbrowser.Open(statusURL) + if f, ok := hookOpenURL.GetOk(); ok { + go f(statusURL) + } } err = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.RequestURI != "/" { @@ -149,10 +154,15 @@ func runStatus(ctx context.Context, args []string) error { os.Exit(1) } - var buf bytes.Buffer - f := func(format string, a ...any) { fmt.Fprintf(&buf, format, a...) } + w := tabwriter.NewWriter(Stdout, 0, 0, 2, ' ', 0) + f := func(format string, a ...any) { fmt.Fprintf(w, format, a...) } + if statusArgs.header { + fmt.Fprintln(w, "IP\tHostname\tOwner\tOS\tStatus\t") + fmt.Fprintln(w, "--\t--------\t-----\t--\t------\t") + } + printPS := func(ps *ipnstate.PeerStatus) { - f("%-15s %-20s %-12s %-7s ", + f("%s\t%s\t%s\t%s\t", firstIPString(ps.TailscaleIPs), dnsOrQuoteHostname(st, ps), ownerLogin(st, ps), @@ -162,17 +172,17 @@ func runStatus(ctx context.Context, args []string) error { anyTraffic := ps.TxBytes != 0 || ps.RxBytes != 0 var offline string if !ps.Online { - offline = "; offline" + offline = "; offline" + lastSeenFmt(ps.LastSeen) } if !ps.Active { if ps.ExitNode { - f("idle; exit node" + offline) + f("idle; exit node%s", offline) } else if ps.ExitNodeOption { - f("idle; offers exit node" + offline) + f("idle; offers exit node%s", offline) } else if anyTraffic { - f("idle" + offline) + f("idle%s", offline) } else if !ps.Online { - f("offline") + f("offline%s", lastSeenFmt(ps.LastSeen)) } else { f("-") } @@ -183,19 +193,21 @@ func runStatus(ctx context.Context, args []string) error { } else if ps.ExitNodeOption { f("offers exit node; ") } - if relay != "" && ps.CurAddr == "" { + if relay != "" && ps.CurAddr == "" && ps.PeerRelay == "" { f("relay %q", relay) } else if ps.CurAddr != "" { f("direct %s", ps.CurAddr) + } else if ps.PeerRelay != "" { + f("peer-relay %s", ps.PeerRelay) } if !ps.Online { - f("; offline") + f("%s", offline) } } if anyTraffic { f(", tx %d rx %d", ps.TxBytes, ps.RxBytes) } - f("\n") + f("\t\n") } if statusArgs.self && st.Self != nil { @@ -210,9 +222,8 @@ func runStatus(ctx context.Context, args []string) error { if ps.ShareeNode { continue } - if ps.Location != nil && ps.ExitNodeOption && !ps.ExitNode { - // Location based exit nodes are only shown with the - // `exit-node list` command. + if ps.ExitNodeOption && !ps.ExitNode && strings.HasSuffix(ps.DNSName, mullvadTCD) { + // Mullvad exit nodes are only shown with the `exit-node list` command. locBasedExitNode = true continue } @@ -226,7 +237,8 @@ func runStatus(ctx context.Context, args []string) error { printPS(ps) } } - Stdout.Write(buf.Bytes()) + w.Flush() + if locBasedExitNode { outln() printf("# To see the full list of exit nodes, including location-based exit nodes, run `tailscale exit-node list` \n") @@ -235,44 +247,15 @@ func runStatus(ctx context.Context, args []string) error { outln() printHealth() } - printFunnelStatus(ctx) + if f, ok := hookPrintFunnelStatus.GetOk(); ok { + f(ctx) + } return nil } -// printFunnelStatus prints the status of the funnel, if it's running. -// It prints nothing if the funnel is not running. -func printFunnelStatus(ctx context.Context) { - sc, err := localClient.GetServeConfig(ctx) - if err != nil { - outln() - printf("# Funnel:\n") - printf("# - Unable to get Funnel status: %v\n", err) - return - } - if !sc.IsFunnelOn() { - return - } - outln() - printf("# Funnel on:\n") - for hp, on := range sc.AllowFunnel { - if !on { // if present, should be on - continue - } - sni, portStr, _ := net.SplitHostPort(string(hp)) - p, _ := strconv.ParseUint(portStr, 10, 16) - isTCP := sc.IsTCPForwardingOnPort(uint16(p)) - url := "https://" - if isTCP { - url = "tcp://" - } - url += sni - if isTCP || p != 443 { - url += ":" + portStr - } - printf("# - %s\n", url) - } - outln() -} +var hookOpenURL feature.Hook[func(string) error] + +var hookPrintFunnelStatus feature.Hook[func(context.Context)] // isRunningOrStarting reports whether st is in state Running or Starting. // It also returns a description of the status suitable to display to a user. diff --git a/cmd/tailscale/cli/switch.go b/cmd/tailscale/cli/switch.go index af8b513263d37..bd90c522e3393 100644 --- a/cmd/tailscale/cli/switch.go +++ b/cmd/tailscale/cli/switch.go @@ -1,10 +1,11 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli import ( "context" + "encoding/json" "flag" "fmt" "os" @@ -18,22 +19,42 @@ import ( ) var switchCmd = &ffcli.Command{ - Name: "switch", - ShortUsage: "tailscale switch ", - ShortHelp: "Switch to a different Tailscale account", + Name: "switch", + ShortUsage: strings.Join([]string{ + "tailscale switch ", + "tailscale switch --list [--json]", + }, "\n"), + ShortHelp: "Switch to a different Tailscale account", LongHelp: `"tailscale switch" switches between logged in accounts. You can use the ID that's returned from 'tailnet switch -list' to pick which profile you want to switch to. Alternatively, you -can use the Tailnet or the account names to switch as well. +can use the Tailnet, account names, or display names to switch as well. This command is currently in alpha and may change in the future.`, FlagSet: func() *flag.FlagSet { fs := flag.NewFlagSet("switch", flag.ExitOnError) fs.BoolVar(&switchArgs.list, "list", false, "list available accounts") + fs.BoolVar(&switchArgs.json, "json", false, "list available accounts in JSON format") return fs }(), Exec: switchProfile, + + // Add remove subcommand + Subcommands: []*ffcli.Command{ + { + Name: "remove", + ShortUsage: "tailscale switch remove ", + ShortHelp: "Remove a Tailscale account", + LongHelp: `"tailscale switch remove" removes a Tailscale account from the +local machine. This does not delete the account itself, but +it will no longer be available for switching to. You can +add it back by logging in again. + +This command is currently in alpha and may change in the future.`, + Exec: removeProfile, + }, + }, } func init() { @@ -46,7 +67,7 @@ func init() { seen := make(map[string]bool, 3*len(all)) wordfns := []func(prof ipn.LoginProfile) string{ func(prof ipn.LoginProfile) string { return string(prof.ID) }, - func(prof ipn.LoginProfile) string { return prof.NetworkProfile.DomainName }, + func(prof ipn.LoginProfile) string { return prof.NetworkProfile.DisplayNameOrDefault() }, func(prof ipn.LoginProfile) string { return prof.Name }, } @@ -57,7 +78,7 @@ func init() { continue } seen[word] = true - words = append(words, fmt.Sprintf("%s\tid: %s, tailnet: %s, account: %s", word, prof.ID, prof.NetworkProfile.DomainName, prof.Name)) + words = append(words, fmt.Sprintf("%s\tid: %s, tailnet: %s, account: %s", word, prof.ID, prof.NetworkProfile.DisplayNameOrDefault(), prof.Name)) } } return words, ffcomplete.ShellCompDirectiveNoFileComp, nil @@ -66,6 +87,7 @@ func init() { var switchArgs struct { list bool + json bool } func listProfiles(ctx context.Context) error { @@ -86,17 +108,55 @@ func listProfiles(ctx context.Context) error { } printRow( string(prof.ID), - prof.NetworkProfile.DomainName, + prof.NetworkProfile.DisplayNameOrDefault(), name, ) } return nil } +type switchProfileJSON struct { + ID string `json:"id"` + Nickname string `json:"nickname"` + Tailnet string `json:"tailnet"` + Account string `json:"account"` + Selected bool `json:"selected"` +} + +func listProfilesJSON(ctx context.Context) error { + curP, all, err := localClient.ProfileStatus(ctx) + if err != nil { + return err + } + profiles := make([]switchProfileJSON, 0, len(all)) + for _, prof := range all { + profiles = append(profiles, switchProfileJSON{ + ID: string(prof.ID), + Tailnet: prof.NetworkProfile.DisplayNameOrDefault(), + Account: prof.UserProfile.LoginName, + Nickname: prof.Name, + Selected: prof.ID == curP.ID, + }) + } + profilesJSON, err := json.MarshalIndent(profiles, "", " ") + if err != nil { + return err + } + printf("%s\n", profilesJSON) + return nil +} + func switchProfile(ctx context.Context, args []string) error { if switchArgs.list { + if switchArgs.json { + return listProfilesJSON(ctx) + } return listProfiles(ctx) } + if switchArgs.json { + outln("--json argument cannot be used with tailscale switch NAME") + os.Exit(1) + } if len(args) != 1 { outln("usage: tailscale switch NAME") os.Exit(1) @@ -106,32 +166,8 @@ func switchProfile(ctx context.Context, args []string) error { errf("Failed to switch to account: %v\n", err) os.Exit(1) } - var profID ipn.ProfileID - // Allow matching by ID, Tailnet, or Account - // in that order. - for _, p := range all { - if p.ID == ipn.ProfileID(args[0]) { - profID = p.ID - break - } - } - if profID == "" { - for _, p := range all { - if p.NetworkProfile.DomainName == args[0] { - profID = p.ID - break - } - } - } - if profID == "" { - for _, p := range all { - if p.Name == args[0] { - profID = p.ID - break - } - } - } - if profID == "" { + profID, ok := matchProfile(args[0], all) + if !ok { errf("No profile named %q\n", args[0]) os.Exit(1) } @@ -178,3 +214,54 @@ func switchProfile(ctx context.Context, args []string) error { } } } + +func removeProfile(ctx context.Context, args []string) error { + if len(args) != 1 { + outln("usage: tailscale switch remove NAME") + os.Exit(1) + } + cp, all, err := localClient.ProfileStatus(ctx) + if err != nil { + errf("Failed to remove account: %v\n", err) + os.Exit(1) + } + + profID, ok := matchProfile(args[0], all) + if !ok { + errf("No profile named %q\n", args[0]) + os.Exit(1) + } + + if profID == cp.ID { + printf("Already on account %q\n", args[0]) + os.Exit(0) + } + + return localClient.DeleteProfile(ctx, profID) +} + +func matchProfile(arg string, all []ipn.LoginProfile) (ipn.ProfileID, bool) { + // Allow matching by ID, Tailnet, Account, or Display Name + // in that order. + for _, p := range all { + if p.ID == ipn.ProfileID(arg) { + return p.ID, true + } + } + for _, p := range all { + if p.NetworkProfile.DomainName == arg { + return p.ID, true + } + } + for _, p := range all { + if p.Name == arg { + return p.ID, true + } + } + for _, p := range all { + if p.NetworkProfile.DisplayName == arg { + return p.ID, true + } + } + return "", false +} diff --git a/cmd/tailscale/cli/syspolicy.go b/cmd/tailscale/cli/syspolicy.go index a71952a9f7f62..e44b01d5ffa15 100644 --- a/cmd/tailscale/cli/syspolicy.go +++ b/cmd/tailscale/cli/syspolicy.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_syspolicy + package cli import ( @@ -20,38 +22,42 @@ var syspolicyArgs struct { json bool // JSON output mode } -var syspolicyCmd = &ffcli.Command{ - Name: "syspolicy", - ShortHelp: "Diagnose the MDM and system policy configuration", - LongHelp: "The 'tailscale syspolicy' command provides tools for diagnosing the MDM and system policy configuration.", - ShortUsage: "tailscale syspolicy ", - UsageFunc: usageFuncNoDefaultValues, - Subcommands: []*ffcli.Command{ - { - Name: "list", - ShortUsage: "tailscale syspolicy list", - Exec: runSysPolicyList, - ShortHelp: "Print effective policy settings", - LongHelp: "The 'tailscale syspolicy list' subcommand displays the effective policy settings and their sources (e.g., MDM or environment variables).", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("syspolicy list") - fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format") - return fs - })(), - }, - { - Name: "reload", - ShortUsage: "tailscale syspolicy reload", - Exec: runSysPolicyReload, - ShortHelp: "Force a reload of policy settings, even if no changes are detected, and prints the result", - LongHelp: "The 'tailscale syspolicy reload' subcommand forces a reload of policy settings, even if no changes are detected, and prints the result.", - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("syspolicy reload") - fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format") - return fs - })(), - }, - }, +func init() { + sysPolicyCmd = func() *ffcli.Command { + return &ffcli.Command{ + Name: "syspolicy", + ShortHelp: "Diagnose the MDM and system policy configuration", + LongHelp: "The 'tailscale syspolicy' command provides tools for diagnosing the MDM and system policy configuration.", + ShortUsage: "tailscale syspolicy ", + UsageFunc: usageFuncNoDefaultValues, + Subcommands: []*ffcli.Command{ + { + Name: "list", + ShortUsage: "tailscale syspolicy list", + Exec: runSysPolicyList, + ShortHelp: "Print effective policy settings", + LongHelp: "The 'tailscale syspolicy list' subcommand displays the effective policy settings and their sources (e.g., MDM or environment variables).", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("syspolicy list") + fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format") + return fs + })(), + }, + { + Name: "reload", + ShortUsage: "tailscale syspolicy reload", + Exec: runSysPolicyReload, + ShortHelp: "Force a reload of policy settings, even if no changes are detected, and prints the result", + LongHelp: "The 'tailscale syspolicy reload' subcommand forces a reload of policy settings, even if no changes are detected, and prints the result.", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("syspolicy reload") + fs.BoolVar(&syspolicyArgs.json, "json", false, "output in JSON format") + return fs + })(), + }, + }, + } + } } func runSysPolicyList(ctx context.Context, args []string) error { @@ -61,7 +67,6 @@ func runSysPolicyList(ctx context.Context, args []string) error { } printPolicySettings(policy) return nil - } func runSysPolicyReload(ctx context.Context, args []string) error { diff --git a/cmd/tailscale/cli/systray.go b/cmd/tailscale/cli/systray.go new file mode 100644 index 0000000000000..07de5c7868fcf --- /dev/null +++ b/cmd/tailscale/cli/systray.go @@ -0,0 +1,37 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux && !ts_omit_systray + +package cli + +import ( + "context" + "flag" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/systray" +) + +var systrayCmd = &ffcli.Command{ + Name: "systray", + ShortUsage: "tailscale systray", + ShortHelp: "Run a systray application to manage Tailscale", + LongHelp: "Run a systray application to manage Tailscale.", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("systray") + fs.StringVar(&systrayArgs.theme, "theme", "dark", "color theme for Tailscale icon: dark, dark:nobg, light, light:nobg") + return fs + })(), + Exec: runSystray, +} + +var systrayArgs struct { + theme string +} + +func runSystray(ctx context.Context, _ []string) error { + systray.SetTheme(systrayArgs.theme) + new(systray.Menu).Run(&localClient) + return nil +} diff --git a/cmd/tailscale/cli/systray_omit.go b/cmd/tailscale/cli/systray_omit.go new file mode 100644 index 0000000000000..83ec199a7d895 --- /dev/null +++ b/cmd/tailscale/cli/systray_omit.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !linux || ts_omit_systray + +package cli + +import ( + "context" + "fmt" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" +) + +// TODO(will): update URL to KB article when available +var systrayHelp = strings.TrimSpace(` +The Tailscale systray app is not included in this client build. +To run it manually, see https://github.com/tailscale/tailscale/tree/main/cmd/systray +`) + +var systrayCmd = &ffcli.Command{ + Name: "systray", + ShortUsage: "tailscale systray", + ShortHelp: "Not available in this client build", + LongHelp: hidden + systrayHelp, + Exec: func(_ context.Context, _ []string) error { + fmt.Println(systrayHelp) + return nil + }, +} diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/tailnet-lock.go similarity index 76% rename from cmd/tailscale/cli/network-lock.go rename to cmd/tailscale/cli/tailnet-lock.go index ae1e90bbfaea9..6e3f7028ce86d 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/tailnet-lock.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_tailnetlock + package cli import ( @@ -8,44 +10,52 @@ import ( "context" "crypto/rand" "encoding/hex" - "encoding/json" + jsonv1 "encoding/json" "errors" "flag" "fmt" + "io" "os" "strconv" "strings" "time" + "github.com/mattn/go-isatty" "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/cmd/tailscale/cli/jsonoutput" "tailscale.com/ipn/ipnstate" "tailscale.com/tka" "tailscale.com/tsconst" "tailscale.com/types/key" "tailscale.com/types/tkatype" + "tailscale.com/util/prompt" ) -var netlockCmd = &ffcli.Command{ +func init() { + maybeTailnetLockCmd = func() *ffcli.Command { return tailnetLockCmd } +} + +var tailnetLockCmd = &ffcli.Command{ Name: "lock", ShortUsage: "tailscale lock [arguments...]", ShortHelp: "Manage tailnet lock", LongHelp: "Manage tailnet lock", Subcommands: []*ffcli.Command{ - nlInitCmd, - nlStatusCmd, - nlAddCmd, - nlRemoveCmd, - nlSignCmd, - nlDisableCmd, - nlDisablementKDFCmd, - nlLogCmd, - nlLocalDisableCmd, - nlRevokeKeysCmd, + tlInitCmd, + tlStatusCmd, + tlAddCmd, + tlRemoveCmd, + tlSignCmd, + tlDisableCmd, + tlDisablementKDFCmd, + tlLogCmd, + tlLocalDisableCmd, + tlRevokeKeysCmd, }, - Exec: runNetworkLockNoSubcommand, + Exec: runTailnetLockNoSubcommand, } -func runNetworkLockNoSubcommand(ctx context.Context, args []string) error { +func runTailnetLockNoSubcommand(ctx context.Context, args []string) error { // Detect & handle the deprecated command 'lock tskey-wrap'. if len(args) >= 2 && args[0] == "tskey-wrap" { return runTskeyWrapCmd(ctx, args[1:]) @@ -54,7 +64,7 @@ func runNetworkLockNoSubcommand(ctx context.Context, args []string) error { return fmt.Errorf("tailscale lock: unknown subcommand: %s", args[0]) } - return runNetworkLockStatus(ctx, args) + return runTailnetLockStatus(ctx, args) } var nlInitArgs struct { @@ -63,7 +73,7 @@ var nlInitArgs struct { confirm bool } -var nlInitCmd = &ffcli.Command{ +var tlInitCmd = &ffcli.Command{ Name: "init", ShortUsage: "tailscale lock init [--gen-disablement-for-support] --gen-disablements N ...", ShortHelp: "Initialize tailnet lock", @@ -88,7 +98,7 @@ will be generated and transmitted to Tailscale, which support can use to disable tailnet lock. We recommend setting this flag. `), - Exec: runNetworkLockInit, + Exec: runTailnetLockInit, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock init") fs.IntVar(&nlInitArgs.numDisablements, "gen-disablements", 1, "number of disablement secrets to generate") @@ -98,8 +108,8 @@ tailnet lock. We recommend setting this flag. })(), } -func runNetworkLockInit(ctx context.Context, args []string) error { - st, err := localClient.NetworkLockStatus(ctx) +func runTailnetLockInit(ctx context.Context, args []string) error { + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } @@ -108,7 +118,7 @@ func runNetworkLockInit(ctx context.Context, args []string) error { } // Parse initially-trusted keys & disablement values. - keys, disablementValues, err := parseNLArgs(args, true, true) + keys, disablementValues, err := parseTLArgs(args, true, true) if err != nil { return err } @@ -173,9 +183,9 @@ func runNetworkLockInit(ctx context.Context, args []string) error { fmt.Fprintln(&successMsg, "A disablement secret for Tailscale support has been generated and transmitted to Tailscale.") } - // The state returned by NetworkLockInit likely doesn't contain the initialized state, + // The state returned by TailnetLockInit likely doesn't contain the initialized state, // because that has to tick through from netmaps. - if _, err := localClient.NetworkLockInit(ctx, keys, disablementValues, supportDisablement); err != nil { + if _, err := localClient.TailnetLockInit(ctx, keys, disablementValues, supportDisablement); err != nil { return err } @@ -185,50 +195,52 @@ func runNetworkLockInit(ctx context.Context, args []string) error { } var nlStatusArgs struct { - json bool + json jsonoutput.SchemaVersion } -var nlStatusCmd = &ffcli.Command{ +var tlStatusCmd = &ffcli.Command{ Name: "status", ShortUsage: "tailscale lock status", ShortHelp: "Output the state of tailnet lock", - Exec: runNetworkLockStatus, + Exec: runTailnetLockStatus, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock status") - fs.BoolVar(&nlStatusArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)") + fs.Var(&nlStatusArgs.json, "json", "output in JSON format") return fs })(), } -func runNetworkLockStatus(ctx context.Context, args []string) error { +func runTailnetLockStatus(ctx context.Context, args []string) error { if len(args) > 0 { return fmt.Errorf("tailscale lock status: unexpected argument") } - st, err := localClient.NetworkLockStatus(ctx) + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } - if nlStatusArgs.json { - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - return enc.Encode(st) + if nlStatusArgs.json.IsSet { + if nlStatusArgs.json.Version == 1 { + return jsonoutput.PrintTailnetLockStatusJSONV1(os.Stdout, st) + } else { + return fmt.Errorf("unrecognised version: %d", nlStatusArgs.json.Version) + } } if st.Enabled { - fmt.Println("Tailnet lock is ENABLED.") + fmt.Println("Tailnet Lock is ENABLED.") } else { - fmt.Println("Tailnet lock is NOT enabled.") + fmt.Println("Tailnet Lock is NOT enabled.") } fmt.Println() if st.Enabled && st.NodeKey != nil && !st.PublicKey.IsZero() { if st.NodeKeySigned { - fmt.Println("This node is accessible under tailnet lock. Node signature:") + fmt.Println("This node is accessible under Tailnet Lock. Node signature:") fmt.Println(st.NodeKeySignature.String()) } else { - fmt.Println("This node is LOCKED OUT by tailnet-lock, and action is required to establish connectivity.") + fmt.Println("This node is LOCKED OUT by Tailnet Lock, and action is required to establish connectivity.") fmt.Printf("Run the following command on a node with a trusted key:\n\ttailscale lock sign %v %s\n", st.NodeKey, st.PublicKey.CLIString()) } fmt.Println() @@ -289,24 +301,22 @@ func runNetworkLockStatus(ctx context.Context, args []string) error { return nil } -var nlAddCmd = &ffcli.Command{ +var tlAddCmd = &ffcli.Command{ Name: "add", ShortUsage: "tailscale lock add ...", ShortHelp: "Add one or more trusted signing keys to tailnet lock", - Exec: func(ctx context.Context, args []string) error { - return runNetworkLockModify(ctx, args, nil) - }, + Exec: runTailnetLockAdd, } var nlRemoveArgs struct { resign bool } -var nlRemoveCmd = &ffcli.Command{ +var tlRemoveCmd = &ffcli.Command{ Name: "remove", ShortUsage: "tailscale lock remove [--re-sign=false] ...", ShortHelp: "Remove one or more trusted signing keys from tailnet lock", - Exec: runNetworkLockRemove, + Exec: runTailnetLockRemove, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock remove") fs.BoolVar(&nlRemoveArgs.resign, "re-sign", true, "resign signatures which would be invalidated by removal of trusted signing keys") @@ -314,12 +324,15 @@ var nlRemoveCmd = &ffcli.Command{ })(), } -func runNetworkLockRemove(ctx context.Context, args []string) error { - removeKeys, _, err := parseNLArgs(args, true, false) +func runTailnetLockRemove(ctx context.Context, args []string) error { + removeKeys, _, err := parseTLArgs(args, true, false) if err != nil { return err } - st, err := localClient.NetworkLockStatus(ctx) + if len(removeKeys) == 0 { + return fmt.Errorf("missing argument, expected one or more tailnet lock keys") + } + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } @@ -346,7 +359,7 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { // Resign affected signatures for each of the keys we are removing. for _, k := range removeKeys { kID, _ := k.ID() // err already checked above - sigs, err := localClient.NetworkLockAffectedSigs(ctx, kID) + sigs, err := localClient.TailnetLockAffectedSigs(ctx, kID) if err != nil { return fmt.Errorf("affected sigs for key %X: %w", kID, err) } @@ -361,20 +374,32 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { return fmt.Errorf("failed decoding pubkey for signature: %w", err) } - // Safety: NetworkLockAffectedSigs() verifies all signatures before + // Safety: TailnetLockAffectedSigs() verifies all signatures before // successfully returning. rotationKey, _ := sig.UnverifiedWrappingPublic() - if err := localClient.NetworkLockSign(ctx, nodeKey, []byte(rotationKey)); err != nil { + if err := localClient.TailnetLockSign(ctx, nodeKey, []byte(rotationKey)); err != nil { return fmt.Errorf("failed to sign %v: %w", nodeKey, err) } } } + } else { + if isatty.IsTerminal(os.Stdout.Fd()) { + fmt.Printf(`Warning +Removal of a signing key(s) without resigning nodes (--re-sign=false) +will cause any nodes signed by the the given key(s) to be locked out +of the Tailscale network. Proceed with caution. +`) + if !prompt.YesNo("Are you sure you want to remove the signing key(s)?", true) { + fmt.Printf("aborting removal of signing key(s)\n") + os.Exit(0) + } + } } - return localClient.NetworkLockModify(ctx, nil, removeKeys) + return localClient.TailnetLockModify(ctx, nil, removeKeys) } -// parseNLArgs parses a slice of strings into slices of tka.Key & disablement +// parseTLArgs parses a slice of strings into slices of tka.Key & disablement // values/secrets. // The keys encoded in args should be specified using their key.NLPublic.MarshalText // representation with an optional '?' suffix. @@ -383,7 +408,7 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { // // If any element could not be parsed, // a nil slice is returned along with an appropriate error. -func parseNLArgs(args []string, parseKeys, parseDisablements bool) (keys []tka.Key, disablements [][]byte, err error) { +func parseTLArgs(args []string, parseKeys, parseDisablements bool) (keys []tka.Key, disablements [][]byte, err error) { for i, a := range args { if parseDisablements && (strings.HasPrefix(a, "disablement:") || strings.HasPrefix(a, "disablement-secret:")) { b, err := hex.DecodeString(a[strings.Index(a, ":")+1:]) @@ -421,31 +446,30 @@ func parseNLArgs(args []string, parseKeys, parseDisablements bool) (keys []tka.K return keys, disablements, nil } -func runNetworkLockModify(ctx context.Context, addArgs, removeArgs []string) error { - st, err := localClient.NetworkLockStatus(ctx) +func runTailnetLockAdd(ctx context.Context, addArgs []string) error { + addKeys, _, err := parseTLArgs(addArgs, true, false) if err != nil { - return fixTailscaledConnectError(err) + return err } - if !st.Enabled { - return errors.New("tailnet lock is not enabled") + if len(addKeys) == 0 { + return fmt.Errorf("missing argument, expected one or more tailnet lock keys") } - addKeys, _, err := parseNLArgs(addArgs, true, false) + st, err := localClient.TailnetLockStatus(ctx) if err != nil { - return err + return fixTailscaledConnectError(err) } - removeKeys, _, err := parseNLArgs(removeArgs, true, false) - if err != nil { - return err + if !st.Enabled { + return errors.New("tailnet lock is not enabled") } - if err := localClient.NetworkLockModify(ctx, addKeys, removeKeys); err != nil { + if err := localClient.TailnetLockModify(ctx, addKeys, nil); err != nil { return err } return nil } -var nlSignCmd = &ffcli.Command{ +var tlSignCmd = &ffcli.Command{ Name: "sign", ShortUsage: "tailscale lock sign []\ntailscale lock sign ", ShortHelp: "Sign a node or pre-approved auth key", @@ -457,10 +481,10 @@ var nlSignCmd = &ffcli.Command{ If any of the key arguments begin with "file:", the key is retrieved from the file at the path specified in the argument suffix.`, - Exec: runNetworkLockSign, + Exec: runTailnetLockSign, } -func runNetworkLockSign(ctx context.Context, args []string) error { +func runTailnetLockSign(ctx context.Context, args []string) error { // If any of the arguments start with "file:", replace that argument // with the contents of the file. We do this early, before the check // to see if the first argument is an auth key. @@ -495,7 +519,7 @@ func runNetworkLockSign(ctx context.Context, args []string) error { } } - err := localClient.NetworkLockSign(ctx, nodeKey, []byte(rotationKey.Verifier())) + err := localClient.TailnetLockSign(ctx, nodeKey, []byte(rotationKey.Verifier())) // Provide a better help message for when someone clicks through the signing flow // on the wrong device. if err != nil && strings.Contains(err.Error(), tsconst.TailnetLockNotTrustedMsg) { @@ -507,7 +531,7 @@ func runNetworkLockSign(ctx context.Context, args []string) error { return err } -var nlDisableCmd = &ffcli.Command{ +var tlDisableCmd = &ffcli.Command{ Name: "disable", ShortUsage: "tailscale lock disable ", ShortHelp: "Consume a disablement secret to shut down tailnet lock for the tailnet", @@ -522,21 +546,21 @@ Once this secret is used, it has been distributed to all nodes in the tailnet and should be considered public. `), - Exec: runNetworkLockDisable, + Exec: runTailnetLockDisable, } -func runNetworkLockDisable(ctx context.Context, args []string) error { - _, secrets, err := parseNLArgs(args, false, true) +func runTailnetLockDisable(ctx context.Context, args []string) error { + _, secrets, err := parseTLArgs(args, false, true) if err != nil { return err } if len(secrets) != 1 { return errors.New("usage: tailscale lock disable ") } - return localClient.NetworkLockDisable(ctx, secrets[0]) + return localClient.TailnetLockDisable(ctx, secrets[0]) } -var nlLocalDisableCmd = &ffcli.Command{ +var tlLocalDisableCmd = &ffcli.Command{ Name: "local-disable", ShortUsage: "tailscale lock local-disable", ShortHelp: "Disable tailnet lock for this node only", @@ -551,22 +575,22 @@ that the current node will accept traffic from other nodes in the tailnet that are locked out. `), - Exec: runNetworkLockLocalDisable, + Exec: runTailnetLockLocalDisable, } -func runNetworkLockLocalDisable(ctx context.Context, args []string) error { - return localClient.NetworkLockForceLocalDisable(ctx) +func runTailnetLockLocalDisable(ctx context.Context, args []string) error { + return localClient.TailnetLockForceLocalDisable(ctx) } -var nlDisablementKDFCmd = &ffcli.Command{ +var tlDisablementKDFCmd = &ffcli.Command{ Name: "disablement-kdf", ShortUsage: "tailscale lock disablement-kdf ", ShortHelp: "Compute a disablement value from a disablement secret (advanced users only)", LongHelp: "Compute a disablement value from a disablement secret (advanced users only)", - Exec: runNetworkLockDisablementKDF, + Exec: runTailnetLockDisablementKDF, } -func runNetworkLockDisablementKDF(ctx context.Context, args []string) error { +func runTailnetLockDisablementKDF(ctx context.Context, args []string) error { if len(args) != 1 { return errors.New("usage: tailscale lock disablement-kdf ") } @@ -580,24 +604,24 @@ func runNetworkLockDisablementKDF(ctx context.Context, args []string) error { var nlLogArgs struct { limit int - json bool + json jsonoutput.SchemaVersion } -var nlLogCmd = &ffcli.Command{ +var tlLogCmd = &ffcli.Command{ Name: "log", ShortUsage: "tailscale lock log [--limit N]", ShortHelp: "List changes applied to tailnet lock", LongHelp: "List changes applied to tailnet lock", - Exec: runNetworkLockLog, + Exec: runTailnetLockLog, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock log") fs.IntVar(&nlLogArgs.limit, "limit", 50, "max number of updates to list") - fs.BoolVar(&nlLogArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)") + fs.Var(&nlLogArgs.json, "json", "output in JSON format") return fs })(), } -func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, error) { +func nlDescribeUpdate(update ipnstate.TailnetLockUpdate, color bool) (string, error) { terminalYellow := "" terminalClear := "" if color { @@ -609,7 +633,7 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er printKey := func(key *tka.Key, prefix string) { fmt.Fprintf(&stanza, "%sType: %s\n", prefix, key.Kind.String()) if keyID, err := key.ID(); err == nil { - fmt.Fprintf(&stanza, "%sKeyID: %x\n", prefix, keyID) + fmt.Fprintf(&stanza, "%sKeyID: tlpub:%x\n", prefix, keyID) } else { // Older versions of the client shouldn't explode when they encounter an // unknown key type. @@ -625,16 +649,20 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er return "", fmt.Errorf("decoding: %w", err) } - fmt.Fprintf(&stanza, "%supdate %x (%s)%s\n", terminalYellow, update.Hash, update.Change, terminalClear) + tkaHead, err := aum.Hash().MarshalText() + if err != nil { + return "", fmt.Errorf("decoding AUM hash: %w", err) + } + fmt.Fprintf(&stanza, "%supdate %s (%s)%s\n", terminalYellow, string(tkaHead), update.Change, terminalClear) switch update.Change { case tka.AUMAddKey.String(): printKey(aum.Key, "") case tka.AUMRemoveKey.String(): - fmt.Fprintf(&stanza, "KeyID: %x\n", aum.KeyID) + fmt.Fprintf(&stanza, "KeyID: tlpub:%x\n", aum.KeyID) case tka.AUMUpdateKey.String(): - fmt.Fprintf(&stanza, "KeyID: %x\n", aum.KeyID) + fmt.Fprintf(&stanza, "KeyID: tlpub:%x\n", aum.KeyID) if aum.Votes != nil { fmt.Fprintf(&stanza, "Votes: %d\n", aum.Votes) } @@ -644,7 +672,7 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er case tka.AUMCheckpoint.String(): fmt.Fprintln(&stanza, "Disablement values:") - for _, v := range aum.State.DisablementSecrets { + for _, v := range aum.State.DisablementValues { fmt.Fprintf(&stanza, " - %x\n", v) } fmt.Fprintln(&stanza, "Keys:") @@ -654,7 +682,7 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er default: // Print a JSON encoding of the AUM as a fallback. - e := json.NewEncoder(&stanza) + e := jsonv1.NewEncoder(&stanza) e.SetIndent("", "\t") if err := e.Encode(aum); err != nil { return "", err @@ -665,19 +693,34 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er return stanza.String(), nil } -func runNetworkLockLog(ctx context.Context, args []string) error { - updates, err := localClient.NetworkLockLog(ctx, nlLogArgs.limit) +func runTailnetLockLog(ctx context.Context, args []string) error { + st, err := localClient.TailnetLockStatus(ctx) if err != nil { return fixTailscaledConnectError(err) } - if nlLogArgs.json { - enc := json.NewEncoder(Stdout) - enc.SetIndent("", " ") - return enc.Encode(updates) + if !st.Enabled { + return errors.New("Tailnet Lock is not enabled") + } + + updates, err := localClient.TailnetLockLog(ctx, nlLogArgs.limit) + if err != nil { + return fixTailscaledConnectError(err) } out, useColor := colorableOutput() + return printTailnetLockLog(updates, out, nlLogArgs.json, useColor) +} + +func printTailnetLockLog(updates []ipnstate.NetworkLockUpdate, out io.Writer, jsonSchema jsonoutput.SchemaVersion, useColor bool) error { + if jsonSchema.IsSet { + if jsonSchema.Version == 1 { + return jsonoutput.PrintTailnetLockLogJSONV1(out, updates) + } else { + return fmt.Errorf("unrecognised version: %d", jsonSchema.Version) + } + } + for _, update := range updates { stanza, err := nlDescribeUpdate(update, useColor) if err != nil { @@ -729,11 +772,11 @@ func wrapAuthKey(ctx context.Context, keyStr string, status *ipnstate.Status) er Meta: m, } - wrapped, err := localClient.NetworkLockWrapPreauthKey(ctx, keyStr, priv) + wrapped, err := localClient.TailnetLockWrapPreauthKey(ctx, keyStr, priv) if err != nil { return fmt.Errorf("wrapping failed: %w", err) } - if err := localClient.NetworkLockModify(ctx, []tka.Key{k}, nil); err != nil { + if err := localClient.TailnetLockModify(ctx, []tka.Key{k}, nil); err != nil { return fmt.Errorf("add key failed: %w", err) } @@ -741,13 +784,13 @@ func wrapAuthKey(ctx context.Context, keyStr string, status *ipnstate.Status) er return nil } -var nlRevokeKeysArgs struct { +var tlRevokeKeysArgs struct { cosign bool finish bool forkFrom string } -var nlRevokeKeysCmd = &ffcli.Command{ +var tlRevokeKeysCmd = &ffcli.Command{ Name: "revoke-keys", ShortUsage: "tailscale lock revoke-keys ...\n revoke-keys [--cosign] [--finish] ", ShortHelp: "Revoke compromised tailnet-lock keys", @@ -763,26 +806,30 @@ Revocation is a multi-step process that requires several signing nodes to ` + "` most recent command output on the next signing node in sequence. 3. Once the number of ` + "`--cosign`" + `s is greater than the number of keys being revoked, run the command one final time with ` + "`--finish`" + ` instead of ` + "`--cosign`" + `.`, - Exec: runNetworkLockRevokeKeys, + Exec: runTailnetLockRevokeKeys, FlagSet: (func() *flag.FlagSet { fs := newFlagSet("lock revoke-keys") - fs.BoolVar(&nlRevokeKeysArgs.cosign, "cosign", false, "continue generating the recovery using the tailnet lock key on this device and the provided recovery blob") - fs.BoolVar(&nlRevokeKeysArgs.finish, "finish", false, "finish the recovery process by transmitting the revocation") - fs.StringVar(&nlRevokeKeysArgs.forkFrom, "fork-from", "", "parent AUM hash to rewrite from (advanced users only)") + fs.BoolVar(&tlRevokeKeysArgs.cosign, "cosign", false, "continue generating the recovery using the tailnet lock key on this device and the provided recovery blob") + fs.BoolVar(&tlRevokeKeysArgs.finish, "finish", false, "finish the recovery process by transmitting the revocation") + fs.StringVar(&tlRevokeKeysArgs.forkFrom, "fork-from", "", "parent AUM hash to rewrite from (advanced users only)") return fs })(), } -func runNetworkLockRevokeKeys(ctx context.Context, args []string) error { +func runTailnetLockRevokeKeys(ctx context.Context, args []string) error { // First step in the process - if !nlRevokeKeysArgs.cosign && !nlRevokeKeysArgs.finish { - removeKeys, _, err := parseNLArgs(args, true, false) + if !tlRevokeKeysArgs.cosign && !tlRevokeKeysArgs.finish { + revokeKeys, _, err := parseTLArgs(args, true, false) if err != nil { return err } - keyIDs := make([]tkatype.KeyID, len(removeKeys)) - for i, k := range removeKeys { + if len(revokeKeys) == 0 { + return fmt.Errorf("missing argument, expected one or more tailnet lock keys") + } + + keyIDs := make([]tkatype.KeyID, len(revokeKeys)) + for i, k := range revokeKeys { keyIDs[i], err = k.ID() if err != nil { return fmt.Errorf("generating keyID: %v", err) @@ -790,22 +837,22 @@ func runNetworkLockRevokeKeys(ctx context.Context, args []string) error { } var forkFrom tka.AUMHash - if nlRevokeKeysArgs.forkFrom != "" { - if len(nlRevokeKeysArgs.forkFrom) == (len(forkFrom) * 2) { + if tlRevokeKeysArgs.forkFrom != "" { + if len(tlRevokeKeysArgs.forkFrom) == (len(forkFrom) * 2) { // Hex-encoded: like the output of the lock log command. - b, err := hex.DecodeString(nlRevokeKeysArgs.forkFrom) + b, err := hex.DecodeString(tlRevokeKeysArgs.forkFrom) if err != nil { return fmt.Errorf("invalid fork-from hash: %v", err) } copy(forkFrom[:], b) } else { - if err := forkFrom.UnmarshalText([]byte(nlRevokeKeysArgs.forkFrom)); err != nil { + if err := forkFrom.UnmarshalText([]byte(tlRevokeKeysArgs.forkFrom)); err != nil { return fmt.Errorf("invalid fork-from hash: %v", err) } } } - aumBytes, err := localClient.NetworkLockGenRecoveryAUM(ctx, keyIDs, forkFrom) + aumBytes, err := localClient.TailnetLockGenRecoveryAUM(ctx, keyIDs, forkFrom) if err != nil { return fmt.Errorf("generation of recovery AUM failed: %w", err) } @@ -826,8 +873,8 @@ func runNetworkLockRevokeKeys(ctx context.Context, args []string) error { return fmt.Errorf("decoding recovery AUM: %v", err) } - if nlRevokeKeysArgs.cosign { - aumBytes, err := localClient.NetworkLockCosignRecoveryAUM(ctx, recoveryAUM) + if tlRevokeKeysArgs.cosign { + aumBytes, err := localClient.TailnetLockCosignRecoveryAUM(ctx, recoveryAUM) if err != nil { return fmt.Errorf("co-signing recovery AUM failed: %w", err) } @@ -842,8 +889,8 @@ Alternatively if you are done with co-signing, complete recovery by running the `, os.Args[0], aumBytes, os.Args[0], aumBytes) } - if nlRevokeKeysArgs.finish { - if err := localClient.NetworkLockSubmitRecoveryAUM(ctx, recoveryAUM); err != nil { + if tlRevokeKeysArgs.finish { + if err := localClient.TailnetLockSubmitRecoveryAUM(ctx, recoveryAUM); err != nil { return fmt.Errorf("submitting recovery AUM failed: %w", err) } fmt.Println("Recovery completed.") diff --git a/cmd/tailscale/cli/tailnet-lock_test.go b/cmd/tailscale/cli/tailnet-lock_test.go new file mode 100644 index 0000000000000..f5822226cdc44 --- /dev/null +++ b/cmd/tailscale/cli/tailnet-lock_test.go @@ -0,0 +1,369 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "bytes" + "net/netip" + "testing" + + "github.com/google/go-cmp/cmp" + "go4.org/mem" + "tailscale.com/cmd/tailscale/cli/jsonoutput" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/tka" + "tailscale.com/types/key" + "tailscale.com/types/tkatype" +) + +func TestTailnetLockLogOutput(t *testing.T) { + votes := uint(1) + aum1 := tka.AUM{ + MessageKind: tka.AUMAddKey, + Key: &tka.Key{ + Kind: tka.Key25519, + Votes: 1, + Public: []byte{2, 2}, + }, + } + h1 := aum1.Hash() + aum2 := tka.AUM{ + MessageKind: tka.AUMRemoveKey, + KeyID: []byte{3, 3}, + PrevAUMHash: h1[:], + Signatures: []tkatype.Signature{ + { + KeyID: []byte{3, 4}, + Signature: []byte{4, 5}, + }, + }, + Meta: map[string]string{"en": "three", "de": "drei", "es": "tres"}, + } + h2 := aum2.Hash() + aum3 := tka.AUM{ + MessageKind: tka.AUMCheckpoint, + PrevAUMHash: h2[:], + State: &tka.State{ + Keys: []tka.Key{ + { + Kind: tka.Key25519, + Votes: 1, + Public: []byte{1, 1}, + Meta: map[string]string{"en": "one", "de": "eins", "es": "uno"}, + }, + }, + DisablementValues: [][]byte{ + {1, 2, 3}, + {4, 5, 6}, + {7, 8, 9}, + }, + }, + Votes: &votes, + } + + updates := []ipnstate.TailnetLockUpdate{ + { + Hash: aum3.Hash(), + Change: aum3.MessageKind.String(), + Raw: aum3.Serialize(), + }, + { + Hash: aum2.Hash(), + Change: aum2.MessageKind.String(), + Raw: aum2.Serialize(), + }, + { + Hash: aum1.Hash(), + Change: aum1.MessageKind.String(), + Raw: aum1.Serialize(), + }, + } + + t.Run("human-readable", func(t *testing.T) { + t.Parallel() + + var outBuf bytes.Buffer + json := jsonoutput.SchemaVersion{} + useColor := false + + printTailnetLockLog(updates, &outBuf, json, useColor) + + t.Logf("%s", outBuf.String()) + + want := `update 4M4Q3IXBARPQMFVXHJBDCYQMWU5H5FBKD7MFF75HE4O5JMIWR2UA (checkpoint) +Disablement values: + - 010203 + - 040506 + - 070809 +Keys: + Type: 25519 + KeyID: tlpub:0101 + Metadata: map[de:eins en:one es:uno] + +update BKVVXHOVBW7Y7YXYTLVVLMNSYG6DS5GVRVSYZLASNU3AQKA732XQ (remove-key) +KeyID: tlpub:0303 + +update UKJIKFHILQ62AEN7MQIFHXJ6SFVDGQCQA3OHVI3LWVPM736EMSAA (add-key) +Type: 25519 +KeyID: tlpub:0202 + +` + + if diff := cmp.Diff(outBuf.String(), want); diff != "" { + t.Fatalf("wrong output (-got, +want):\n%s", diff) + } + }) + + jsonV1 := `{ + "SchemaVersion": "1", + "Messages": [ + { + "Hash": "4M4Q3IXBARPQMFVXHJBDCYQMWU5H5FBKD7MFF75HE4O5JMIWR2UA", + "AUM": { + "MessageKind": "checkpoint", + "PrevAUMHash": "BKVVXHOVBW7Y7YXYTLVVLMNSYG6DS5GVRVSYZLASNU3AQKA732XQ", + "State": { + "DisablementValues": [ + "010203", + "040506", + "070809" + ], + "Keys": [ + { + "Kind": "25519", + "Votes": 1, + "Public": "tlpub:0101", + "Meta": { + "de": "eins", + "en": "one", + "es": "uno" + } + } + ], + "StateID1": 0, + "StateID2": 0 + }, + "Votes": 1 + }, + "Raw": "pAEFAlggCqtbndUNv4_i-JrrVbGywbw5dNWNZYysEm02CCgf3q8FowH2AoNDAQIDQwQFBkMHCAkDgaQBAQIBA0IBAQyjYmRlZGVpbnNiZW5jb25lYmVzY3VubwYB" + }, + { + "Hash": "BKVVXHOVBW7Y7YXYTLVVLMNSYG6DS5GVRVSYZLASNU3AQKA732XQ", + "AUM": { + "MessageKind": "remove-key", + "PrevAUMHash": "UKJIKFHILQ62AEN7MQIFHXJ6SFVDGQCQA3OHVI3LWVPM736EMSAA", + "KeyID": "tlpub:0303", + "Meta": { + "de": "drei", + "en": "three", + "es": "tres" + }, + "Signatures": [ + { + "KeyID": "tlpub:0304", + "Signature": "BAU=" + } + ] + }, + "Raw": "pQECAlggopKFFOhcPaARv2QQU90-kWozQFAG3Hqja7Vez-_EZIAEQgMDB6NiZGVkZHJlaWJlbmV0aHJlZWJlc2R0cmVzF4GiAUIDBAJCBAU=" + }, + { + "Hash": "UKJIKFHILQ62AEN7MQIFHXJ6SFVDGQCQA3OHVI3LWVPM736EMSAA", + "AUM": { + "MessageKind": "add-key", + "Key": { + "Kind": "25519", + "Votes": 1, + "Public": "tlpub:0202" + } + }, + "Raw": "owEBAvYDowEBAgEDQgIC" + } + ] +} +` + + t.Run("json-1", func(t *testing.T) { + t.Parallel() + + var outBuf bytes.Buffer + json := jsonoutput.SchemaVersion{ + IsSet: true, + Version: 1, + } + useColor := false + + printTailnetLockLog(updates, &outBuf, json, useColor) + + want := jsonV1 + + if diff := cmp.Diff(outBuf.String(), want); diff != "" { + t.Fatalf("wrong output (-got, +want):\n%s", diff) + } + }) +} + +func TestTailnetLockStatusOutput(t *testing.T) { + aum := tka.AUM{ + MessageKind: tka.AUMNoOp, + } + h := aum.Hash() + head := [32]byte(h[:]) + + nodeKey1 := key.NodePublicFromRaw32(mem.B(bytes.Repeat([]byte{1}, 32))) + nodeKey2 := key.NodePublicFromRaw32(mem.B(bytes.Repeat([]byte{2}, 32))) + nodeKey3 := key.NodePublicFromRaw32(mem.B(bytes.Repeat([]byte{3}, 32))) + + nlPub := key.NLPublicFromEd25519Unsafe(bytes.Repeat([]byte{4}, 32)) + + trustedNlPub := key.NLPublicFromEd25519Unsafe(bytes.Repeat([]byte{5}, 32)) + + tailnetIPv4_A, tailnetIPv6_A := netip.MustParseAddr("100.99.99.99"), netip.MustParseAddr("fd7a:115c:a1e0::701:b62a") + tailnetIPv4_B, tailnetIPv6_B := netip.MustParseAddr("100.88.88.88"), netip.MustParseAddr("fd7a:115c:a1e0::4101:512f") + + t.Run("json-1", func(t *testing.T) { + for _, tt := range []struct { + Name string + Status ipnstate.TailnetLockStatus + Want string + }{ + { + Name: "tailnet-lock-disabled", + Status: ipnstate.TailnetLockStatus{Enabled: false}, + Want: `{ + "SchemaVersion": "1", + "Enabled": false +} +`, + }, + { + Name: "tailnet-lock-disabled-with-keys", + Status: ipnstate.TailnetLockStatus{ + Enabled: false, + NodeKey: &nodeKey1, + PublicKey: trustedNlPub, + }, + Want: `{ + "SchemaVersion": "1", + "Enabled": false, + "PublicKey": "tlpub:0505050505050505050505050505050505050505050505050505050505050505", + "NodeKey": "nodekey:0101010101010101010101010101010101010101010101010101010101010101" +} +`, + }, + { + Name: "tailnet-lock-enabled", + Status: ipnstate.TailnetLockStatus{ + Enabled: true, + Head: &head, + PublicKey: nlPub, + NodeKey: &nodeKey1, + NodeKeySigned: false, + NodeKeySignature: nil, + TrustedKeys: []ipnstate.TKAKey{ + { + Kind: tka.Key25519.String(), + Votes: 1, + Key: trustedNlPub, + Metadata: map[string]string{"en": "one", "de": "eins", "es": "uno"}, + }, + }, + VisiblePeers: []*ipnstate.TKAPeer{ + { + Name: "authentic-associate", + ID: tailcfg.NodeID(1234), + StableID: tailcfg.StableNodeID("1234_AAAA_TEST"), + TailscaleIPs: []netip.Addr{tailnetIPv4_A, tailnetIPv6_A}, + NodeKey: nodeKey2, + NodeKeySignature: tka.NodeKeySignature{ + SigKind: tka.SigDirect, + Pubkey: []byte("22222222222222222222222222222222"), + KeyID: []byte("44444444444444444444444444444444"), + Signature: []byte("1234567890"), + WrappingPubkey: []byte("0987654321"), + }, + }, + }, + FilteredPeers: []*ipnstate.TKAPeer{ + { + Name: "bogus-bandit", + ID: tailcfg.NodeID(5678), + StableID: tailcfg.StableNodeID("5678_BBBB_TEST"), + TailscaleIPs: []netip.Addr{tailnetIPv4_B, tailnetIPv6_B}, + NodeKey: nodeKey3, + }, + }, + StateID: 98989898, + }, + Want: `{ + "SchemaVersion": "1", + "Enabled": true, + "PublicKey": "tlpub:0404040404040404040404040404040404040404040404040404040404040404", + "NodeKey": "nodekey:0101010101010101010101010101010101010101010101010101010101010101", + "Head": "WYIVHDR7JUIXBWAJT5UPSCAILEXB7OMINDFEFEPOPNTUCNXMY2KA", + "NodeKeySigned": false, + "NodeKeySignature": null, + "TrustedKeys": [ + { + "Kind": "25519", + "Votes": 1, + "Public": "tlpub:0505050505050505050505050505050505050505050505050505050505050505", + "Meta": { + "de": "eins", + "en": "one", + "es": "uno" + } + } + ], + "VisiblePeers": [ + { + "ID": "1234_AAAA_TEST", + "DNSName": "authentic-associate", + "TailscaleIPs": [ + "100.99.99.99", + "fd7a:115c:a1e0::701:b62a" + ], + "NodeKey": "nodekey:0202020202020202020202020202020202020202020202020202020202020202", + "NodeKeySignature": { + "SigKind": "direct", + "PublicKey": "tlpub:3232323232323232323232323232323232323232323232323232323232323232", + "KeyID": "tlpub:3434343434343434343434343434343434343434343434343434343434343434", + "Signature": "MTIzNDU2Nzg5MA==", + "WrappingPublicKey": "tlpub:30393837363534333231" + } + } + ], + "FilteredPeers": [ + { + "ID": "5678_BBBB_TEST", + "DNSName": "bogus-bandit", + "TailscaleIPs": [ + "100.88.88.88", + "fd7a:115c:a1e0::4101:512f" + ], + "NodeKey": "nodekey:0303030303030303030303030303030303030303030303030303030303030303" + } + ], + "State": 98989898 +} +`, + }, + } { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + + var outBuf bytes.Buffer + err := jsonoutput.PrintTailnetLockStatusJSONV1(&outBuf, &tt.Status) + if err != nil { + t.Fatalf("PrintTailnetLockStatusJSONV1: %v", err) + } + + if diff := cmp.Diff(outBuf.String(), tt.Want); diff != "" { + t.Fatalf("wrong output (-got, +want):\n%s", diff) + } + }) + } + }) +} diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index 37cdab754db18..fed7de9ae7e01 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -12,21 +12,21 @@ import ( "fmt" "log" "net/netip" - "net/url" "os" "os/signal" "reflect" "runtime" "sort" - "strconv" "strings" "syscall" "time" shellquote "github.com/kballard/go-shellquote" "github.com/peterbourgon/ff/v3/ffcli" - qrcode "github.com/skip2/go-qrcode" - "golang.org/x/oauth2/clientcredentials" + "tailscale.com/feature/buildfeatures" + _ "tailscale.com/feature/condregister/awsparamstore" + _ "tailscale.com/feature/condregister/identityfederation" + _ "tailscale.com/feature/condregister/oauthkey" "tailscale.com/health/healthmsg" "tailscale.com/internal/client/tailscale" "tailscale.com/ipn" @@ -39,6 +39,8 @@ import ( "tailscale.com/types/preftype" "tailscale.com/types/views" "tailscale.com/util/dnsname" + "tailscale.com/util/qrcodes" + "tailscale.com/util/syspolicy/policyclient" "tailscale.com/version/distro" ) @@ -93,23 +95,30 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { // When adding new flags, prefer to put them under "tailscale set" instead // of here. Setting preferences via "tailscale up" is deprecated. - upf.BoolVar(&upArgs.qr, "qr", false, "show QR code for login URLs") + if buildfeatures.HasQRCodes { + upf.BoolVar(&upArgs.qr, "qr", false, "show QR code for login URLs") + upf.StringVar(&upArgs.qrFormat, "qr-format", string(qrcodes.FormatAuto), fmt.Sprintf("QR code formatting (%s, %s, %s, %s)", qrcodes.FormatAuto, qrcodes.FormatASCII, qrcodes.FormatLarge, qrcodes.FormatSmall)) + } upf.StringVar(&upArgs.authKeyOrFile, "auth-key", "", `node authorization key; if it begins with "file:", then it's a path to a file containing the authkey`) + upf.StringVar(&upArgs.audience, "audience", "", "Audience used when requesting an ID token from an identity provider for auth keys via workload identity federation") + upf.StringVar(&upArgs.clientID, "client-id", "", "Client ID used to generate authkeys via workload identity federation") + upf.StringVar(&upArgs.clientSecretOrFile, "client-secret", "", `Client Secret used to generate authkeys via OAuth; if it begins with "file:", then it's a path to a file containing the secret`) + upf.StringVar(&upArgs.idTokenOrFile, "id-token", "", `ID token from the identity provider to exchange with the control server for workload identity federation; if it begins with "file:", then it's a path to a file containing the token`) upf.StringVar(&upArgs.server, "login-server", ipn.DefaultControlURL, "base URL of control server") upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes") upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") upf.Var(notFalseVar{}, "host-routes", hidden+"install host routes to other Tailscale nodes (must be true as of Tailscale 1.67+)") - upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node") + upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") upf.BoolVar(&upArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") - upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")") + upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request (e.g. \"tag:eng,tag:montreal,tag:ssh\"); the \"tag:\" prefix is optional and added automatically when omitted (e.g. \"eng,montreal,ssh\")") upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS") upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes") upf.BoolVar(&upArgs.advertiseConnector, "advertise-connector", false, "advertise this node as an app connector") upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet") - upf.BoolVar(&upArgs.postureChecking, "report-posture", false, hidden+"allow management plane to gather device posture information") + upf.BoolVar(&upArgs.postureChecking, "report-posture", false, "allow management plane to gather device posture information") if safesocket.GOOSUsesPeerCreds(goos) { upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo") @@ -117,7 +126,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { switch goos { case "linux": upf.BoolVar(&upArgs.snat, "snat-subnet-routes", true, "source NAT traffic to local routes advertised with --advertise-routes") - upf.BoolVar(&upArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, etc.)") + upf.BoolVar(&upArgs.statefulFiltering, "stateful-filtering", false, "apply stateful filtering to forwarded packets (subnet routers, exit nodes, and so on)") upf.StringVar(&upArgs.netfilterMode, "netfilter-mode", defaultNetfilterMode(), "netfilter mode (one of on, nodivert, off)") case "windows": upf.BoolVar(&upArgs.forceDaemon, "unattended", false, "run in \"Unattended Mode\" where Tailscale keeps running even after the current GUI user logs out (Windows-only)") @@ -132,14 +141,17 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { // Some flags are only for "up", not "login". upf.BoolVar(&upArgs.json, "json", false, "output in JSON format (WARNING: format subject to change)") upf.BoolVar(&upArgs.reset, "reset", false, "reset unspecified settings to their default values") - upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication (WARNING: this will bring down the Tailscale connection and thus should not be done remotely over SSH or RDP)") + + // There's no --force-reauth flag on "login" because all login commands + // trigger a reauth. + upf.BoolVar(&upArgs.forceReauth, "force-reauth", false, "force reauthentication (WARNING: this may bring down the Tailscale connection and thus should not be done remotely over SSH or RDP)") registerAcceptRiskFlag(upf, &upArgs.acceptedRisks) } return upf } -// notFalseVar is is a flag.Value that can only be "true", if set. +// notFalseVar is a flag.Value that can only be "true", if set. type notFalseVar struct{} func (notFalseVar) IsBoolFlag() bool { return true } @@ -163,6 +175,7 @@ func defaultNetfilterMode() string { // added to it. Add new arguments to setArgsT instead. type upArgsT struct { qr bool + qrFormat string reset bool server string acceptRoutes bool @@ -182,6 +195,10 @@ type upArgsT struct { statefulFiltering bool netfilterMode string authKeyOrFile string // "secret" or "file:/path/to/secret" + clientID string + audience string + clientSecretOrFile string // "secret" or "file:/path/to/secret" + idTokenOrFile string // "secret" or "file:/path/to/secret" hostname string opUser string json bool @@ -191,8 +208,9 @@ type upArgsT struct { postureChecking bool } -func (a upArgsT) getAuthKey() (string, error) { - v := a.authKeyOrFile +// resolveValueFromFile returns the value as-is, or if it starts with "file:", +// reads and returns the trimmed contents of the file. +func resolveValueFromFile(v string) (string, error) { if file, ok := strings.CutPrefix(v, "file:"); ok { b, err := os.ReadFile(file) if err != nil { @@ -203,6 +221,41 @@ func (a upArgsT) getAuthKey() (string, error) { return v, nil } +// resolveValueFromParameterStore resolves a value from AWS Parameter Store if +// the value looks like an SSM ARN. If the hook is not available or the value +// is not an SSM ARN, it returns the value unchanged. +func resolveValueFromParameterStore(ctx context.Context, v string) (string, error) { + if f, ok := tailscale.HookResolveValueFromParameterStore.GetOk(); ok { + return f(ctx, v) + } + return v, nil +} + +// resolveValue will take the given value (e.g. as passed to --auth-key), and +// depending on the prefix, resolve the value from either a file or AWS +// Parameter Store. Values with an unknown prefix are returned as-is. +func resolveValue(ctx context.Context, v string) (string, error) { + switch { + case strings.HasPrefix(v, "file:"): + return resolveValueFromFile(v) + case strings.HasPrefix(v, tailscale.ResolvePrefixAWSParameterStore): + return resolveValueFromParameterStore(ctx, v) + } + return v, nil +} + +func (a upArgsT) getAuthKey(ctx context.Context) (string, error) { + return resolveValue(ctx, a.authKeyOrFile) +} + +func (a upArgsT) getClientSecret(ctx context.Context) (string, error) { + return resolveValue(ctx, a.clientSecretOrFile) +} + +func (a upArgsT) getIDToken(ctx context.Context) (string, error) { + return resolveValue(ctx, a.idTokenOrFile) +} + var upArgsGlobal upArgsT // Fields output when `tailscale up --json` is used. Two JSON blocks will be output. @@ -256,9 +309,15 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo var tags []string if upArgs.advertiseTags != "" { tags = strings.Split(upArgs.advertiseTags, ",") - for _, tag := range tags { - err := tailcfg.CheckTag(tag) - if err != nil { + for i, tag := range tags { + // Allow users to omit the "tag:" prefix; if the tag has no + // colon at all, add it for them. Tags with a colon must be + // fully qualified ("tag:foo") and are validated as-is. + if !strings.Contains(tag, ":") { + tag = "tag:" + tag + tags[i] = tag + } + if err := tailcfg.CheckTag(tag); err != nil { return nil, fmt.Errorf("tag: %q: %s", tag, err) } } @@ -278,9 +337,10 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo prefs.NetfilterMode = preftype.NetfilterOff } if upArgs.exitNodeIP != "" { - if err := prefs.SetExitNodeIP(upArgs.exitNodeIP, st); err != nil { - var e ipn.ExitNodeLocalIPError - if errors.As(err, &e) { + if expr, useAutoExitNode := ipn.ParseAutoExitNodeString(upArgs.exitNodeIP); useAutoExitNode { + prefs.AutoExitNode = expr + } else if err := prefs.SetExitNodeIP(upArgs.exitNodeIP, st); err != nil { + if _, ok := errors.AsType[ipn.ExitNodeLocalIPError](err); ok { return nil, fmt.Errorf("%w; did you mean --advertise-exit-node?", err) } return nil, err @@ -303,6 +363,11 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo if goos == "linux" { prefs.NoSNAT = !upArgs.snat + // We want to make sure user is aware setting --snat-subnet-routes=false with --advertise-exit-node would break exitnode, + // but we won't prevent them from doing it since there are current dependencies on that combination. (as of 2026-03-25) + if prefs.NoSNAT && prefs.AdvertisesExitNode() { + warnf("--snat-subnet-routes=false is set with --advertise-exit-node; internet traffic through this exit node may not work as expected") + } // Backfills for NoStatefulFiltering occur when loading a profile; just set it explicitly here. prefs.NoStatefulFiltering.Set(!upArgs.statefulFiltering) @@ -353,11 +418,19 @@ func netfilterModeFromFlag(v string) (_ preftype.NetfilterMode, warning string, // It returns simpleUp if we're running a simple "tailscale up" to // transition to running from a previously-logged-in but down state, // without changing any settings. +// +// Note this can also mutate prefs to add implicit preferences for the +// user operator. +// +// TODO(alexc): the name of this function is confusing, and perhaps a +// sign that it's doing too much. Consider refactoring this so it's just +// telling the caller what to do next, but not changing anything itself. func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, justEditMP *ipn.MaskedPrefs, err error) { if !env.upArgs.reset { applyImplicitPrefs(prefs, curPrefs, env) - if err := checkForAccidentalSettingReverts(prefs, curPrefs, env); err != nil { + simpleUp, err = checkForAccidentalSettingReverts(prefs, curPrefs, env) + if err != nil { return false, nil, err } } @@ -382,18 +455,13 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus } if env.upArgs.forceReauth && isSSHOverTailscale() { - if err := presentRiskToUser(riskLoseSSH, `You are connected over Tailscale; this action will result in your SSH session disconnecting.`, env.upArgs.acceptedRisks); err != nil { + if err := presentRiskToUser(riskLoseSSH, `You are connected over Tailscale; this action may result in your SSH session disconnecting.`, env.upArgs.acceptedRisks); err != nil { return false, nil, err } } tagsChanged := !reflect.DeepEqual(curPrefs.AdvertiseTags, prefs.AdvertiseTags) - simpleUp = env.flagSet.NFlag() == 0 && - curPrefs.Persist != nil && - curPrefs.Persist.UserProfile.LoginName != "" && - env.backendState != ipn.NeedsLogin.String() - justEdit := env.backendState == ipn.Running.String() && !env.upArgs.forceReauth && env.upArgs.authKeyOrFile == "" && @@ -408,6 +476,9 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus if env.upArgs.reset { visitFlags = env.flagSet.VisitAll } + if prefs.AutoExitNode.IsSet() { + justEditMP.AutoExitNodeSet = true + } visitFlags(func(f *flag.Flag) { updateMaskedPrefsFromUpOrSetFlag(justEditMP, f.Name) }) @@ -440,6 +511,7 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE return fixTailscaledConnectError(err) } origAuthURL := st.AuthURL + origNodeKey := st.Self.PublicKey // printAuthURL reports whether we should print out the // provided auth URL from an IPN notify. @@ -480,15 +552,14 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE fatalf("%s", err) } - warnOnAdvertiseRouts(ctx, prefs) - if err := checkExitNodeRisk(ctx, prefs, upArgs.acceptedRisks); err != nil { - return err - } + warnOnAdvertiseRoutes(ctx, prefs) curPrefs, err := localClient.GetPrefs(ctx) if err != nil { return err } + effectivePrefs := curPrefs + if cmd == "up" { // "tailscale up" should not be able to change the // profile name. @@ -534,8 +605,21 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE } }() - running := make(chan bool, 1) // gets value once in state ipn.Running - watchErr := make(chan error, 1) + if !buildfeatures.HasIPNBus { + fmt.Fprintln(Stderr, "binary built with ts_omit_ipnbus; not waiting for completion") + return nil + } + + // Start watching the IPN bus before we call Start() or StartLoginInteractive(), + // or we could miss IPN notifications. + // + // In particular, if we're doing a force-reauth, we could miss the + // notification with the auth URL we should print for the user. + watcher, err := localClient.WatchIPNBus(watchCtx, 0) + if err != nil { + return err + } + defer watcher.Close() // Special case: bare "tailscale up" means to just start // running, if there's ever been a login. @@ -554,14 +638,40 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE return err } - authKey, err := upArgs.getAuthKey() + authKey, err := upArgs.getAuthKey(ctx) if err != nil { return err } - authKey, err = resolveAuthKey(ctx, authKey, upArgs.advertiseTags) - if err != nil { - return err + // Try to use an OAuth secret to generate an auth key if that functionality + // is available. + if f, ok := tailscale.HookResolveAuthKey.GetOk(); ok { + clientSecret := authKey // the authkey argument accepts client secrets, if both arguments are provided authkey has precedence + if clientSecret == "" { + clientSecret, err = upArgs.getClientSecret(ctx) + if err != nil { + return err + } + } + + authKey, err = f(ctx, clientSecret, prefs.AdvertiseTags) + if err != nil { + return err + } + } + // Try to resolve the auth key via workload identity federation if that functionality + // is available and no auth key is yet determined. + if f, ok := tailscale.HookResolveAuthKeyViaWIF.GetOk(); ok && authKey == "" { + idToken, err := upArgs.getIDToken(ctx) + if err != nil { + return err + } + + authKey, err = f(ctx, prefs.ControlURL, upArgs.clientID, idToken, upArgs.audience, prefs.AdvertiseTags) + if err != nil { + return err + } } + err = localClient.Start(ctx, ipn.Options{ AuthKey: authKey, UpdatePrefs: prefs, @@ -569,6 +679,7 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE if err != nil { return err } + effectivePrefs = prefs if upArgs.forceReauth || !st.HaveNodeKey { err := localClient.StartLoginInteractive(ctx) if err != nil { @@ -577,15 +688,32 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE } } - watcher, err := localClient.WatchIPNBus(watchCtx, ipn.NotifyInitialState) - if err != nil { - return err - } - defer watcher.Close() + upComplete := make(chan bool, 1) + watchErr := make(chan error, 1) go func() { var printed bool // whether we've yet printed anything to stdout or stderr - var lastURLPrinted string + lastURLPrinted := "" + + // If we're doing a force-reauth, we need to get two notifications: + // + // 1. IPN is running + // 2. The node key has changed + // + // These two notifications arrive separately, and trying to combine them + // has caused unexpected issues elsewhere in `tailscale up`. For now, we + // track them separately. + ipnIsRunning := false + waitingForKeyChange := upArgs.forceReauth + + // If we're doing a simple up (i.e. `tailscale up`, no flags) and + // the initial state is NeedsMachineAuth, then we never receive a + // state notification from ipn, so we print the device approval URL + // immediately. + if simpleUp && st.BackendState == ipn.NeedsMachineAuth.String() { + printed = true + printDeviceApprovalInfo(env.upArgs.json, effectivePrefs, &lastURLPrinted) + } for { n, err := watcher.Next() @@ -597,29 +725,30 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE msg := *n.ErrMessage fatalf("backend error: %v\n", msg) } + if s := n.State; s != nil && *s == ipn.NeedsMachineAuth { + printed = true + printDeviceApprovalInfo(env.upArgs.json, effectivePrefs, &lastURLPrinted) + } if s := n.State; s != nil { - switch *s { - case ipn.NeedsMachineAuth: - printed = true - if env.upArgs.json { - printUpDoneJSON(ipn.NeedsMachineAuth, "") - } else { - fmt.Fprintf(Stderr, "\nTo approve your machine, visit (as admin):\n\n\t%s\n\n", prefs.AdminPageURL()) - } - case ipn.Running: - // Done full authentication process - if env.upArgs.json { - printUpDoneJSON(ipn.Running, "") - } else if printed { - // Only need to print an update if we printed the "please click" message earlier. - fmt.Fprintf(Stderr, "Success.\n") - } - select { - case running <- true: - default: - } - cancelWatch() + ipnIsRunning = *s == ipn.Running + } + if n.SelfChange != nil && n.SelfChange.Key != origNodeKey { + waitingForKeyChange = false + } + if ipnIsRunning && !waitingForKeyChange { + // Done full authentication process + if env.upArgs.json { + printUpDoneJSON(ipn.Running, "") + } else if printed { + // Only need to print an update if we printed the "please click" message earlier. + fmt.Fprintf(Stderr, "Success.\n") + } + select { + case upComplete <- true: + default: } + cancelWatch() + return } if url := n.BrowseToURL; url != nil { authURL := *url @@ -631,9 +760,8 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE if upArgs.json { js := &upOutputJSON{AuthURL: authURL, BackendState: st.BackendState} - q, err := qrcode.New(authURL, qrcode.Medium) - if err == nil { - png, err := q.PNG(128) + if buildfeatures.HasQRCodes { + png, err := qrcodes.EncodePNG(authURL, 128) if err == nil { js.QR = "data:image/png;base64," + base64.StdEncoding.EncodeToString(png) } @@ -647,12 +775,10 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE } } else { fmt.Fprintf(Stderr, "\nTo authenticate, visit:\n\n\t%s\n\n", authURL) - if upArgs.qr { - q, err := qrcode.New(authURL, qrcode.Medium) + if upArgs.qr && buildfeatures.HasQRCodes { + _, err := qrcodes.Fprintln(Stderr, qrcodes.Format(upArgs.qrFormat), authURL) if err != nil { - log.Printf("QR code error: %v", err) - } else { - fmt.Fprintf(Stderr, "%s\n", q.ToString(false)) + log.Print(err) } } } @@ -674,18 +800,18 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE timeoutCh = timeoutTimer.C } select { - case <-running: + case <-upComplete: return nil case <-watchCtx.Done(): select { - case <-running: + case <-upComplete: return nil default: } return watchCtx.Err() case err := <-watchErr: select { - case <-running: + case <-upComplete: return nil default: } @@ -695,6 +821,21 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE } } +func printDeviceApprovalInfo(printJson bool, prefs *ipn.Prefs, lastURLPrinted *string) { + if printJson { + printUpDoneJSON(ipn.NeedsMachineAuth, "") + } else { + deviceApprovalURL := prefs.AdminPageURL(policyclient.Get()) + + if lastURLPrinted != nil && deviceApprovalURL == *lastURLPrinted { + return + } + + *lastURLPrinted = deviceApprovalURL + errf("\nTo approve your machine, visit (as admin):\n\n\t%s\n\n", deviceApprovalURL) + } +} + // upWorthWarning reports whether the health check message s is worth warning // about during "tailscale up". Many of the health checks are noisy or confusing // or very ephemeral and happen especially briefly at startup. @@ -705,7 +846,7 @@ func upWorthyWarning(s string) bool { return strings.Contains(s, healthmsg.TailscaleSSHOnBut) || strings.Contains(s, healthmsg.WarnAcceptRoutesOff) || strings.Contains(s, healthmsg.LockedOut) || - strings.Contains(s, healthmsg.WarnExitNodeUsage) || + strings.Contains(s, healthmsg.InMemoryTailnetLockState) || strings.Contains(strings.ToLower(s), "update available: ") } @@ -777,6 +918,8 @@ func init() { addPrefFlagMapping("advertise-connector", "AppConnector") addPrefFlagMapping("report-posture", "PostureChecking") addPrefFlagMapping("relay-server-port", "RelayServerPort") + addPrefFlagMapping("sync", "Sync") + addPrefFlagMapping("relay-server-static-endpoints", "RelayServerStaticEndpoints") } func addPrefFlagMapping(flagName string, prefNames ...string) { @@ -784,7 +927,7 @@ func addPrefFlagMapping(flagName string, prefNames ...string) { prefType := reflect.TypeFor[ipn.Prefs]() for _, pref := range prefNames { t := prefType - for _, name := range strings.Split(pref, ".") { + for name := range strings.SplitSeq(pref, ".") { // Crash at runtime if there's a typo in the prefName. f, ok := t.FieldByName(name) if !ok { @@ -799,7 +942,7 @@ func addPrefFlagMapping(flagName string, prefNames ...string) { // correspond to an ipn.Pref. func preflessFlag(flagName string) bool { switch flagName { - case "auth-key", "force-reauth", "reset", "qr", "json", "timeout", "accept-risk", "host-routes": + case "auth-key", "force-reauth", "reset", "qr", "qr-format", "json", "timeout", "accept-risk", "host-routes", "client-id", "audience", "client-secret", "id-token": return true } return false @@ -812,7 +955,7 @@ func updateMaskedPrefsFromUpOrSetFlag(mp *ipn.MaskedPrefs, flagName string) { if prefs, ok := prefsOfFlag[flagName]; ok { for _, pref := range prefs { f := reflect.ValueOf(mp).Elem() - for _, name := range strings.Split(pref, ".") { + for name := range strings.SplitSeq(pref, ".") { f = f.FieldByName(name + "Set") } f.SetBool(true) @@ -854,10 +997,10 @@ type upCheckEnv struct { // // mp is the mask of settings actually set, where mp.Prefs is the new // preferences to set, including any values set from implicit flags. -func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheckEnv) error { +func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, err error) { if curPrefs.ControlURL == "" { // Don't validate things on initial "up" before a control URL has been set. - return nil + return false, nil } flagIsSet := map[string]bool{} @@ -865,10 +1008,13 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck flagIsSet[f.Name] = true }) - if len(flagIsSet) == 0 { + if len(flagIsSet) == 0 && + curPrefs.Persist != nil && + curPrefs.Persist.UserProfile.LoginName != "" && + env.backendState != ipn.NeedsLogin.String() { // A bare "tailscale up" is a special case to just // mean bringing the network up without any changes. - return nil + return true, nil } // flagsCur is what flags we'd need to use to keep the exact @@ -910,7 +1056,7 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck missing = append(missing, fmtFlagValueArg(flagName, valCur)) } if len(missing) == 0 { - return nil + return false, nil } // Some previously provided flags are missing. This run of 'tailscale @@ -943,7 +1089,7 @@ func checkForAccidentalSettingReverts(newPrefs, curPrefs *ipn.Prefs, env upCheck fmt.Fprintf(&sb, " %s", a) } sb.WriteString("\n\n") - return errors.New(sb.String()) + return false, errors.New(sb.String()) } // applyImplicitPrefs mutates prefs to add implicit preferences for the user operator. @@ -1094,92 +1240,9 @@ func exitNodeIP(p *ipn.Prefs, st *ipnstate.Status) (ip netip.Addr) { return } -// resolveAuthKey either returns v unchanged (in the common case) or, if it -// starts with "tskey-client-" (as Tailscale OAuth secrets do) parses it like -// -// tskey-client-xxxx[?ephemeral=false&bar&preauthorized=BOOL&baseURL=...] -// -// and does the OAuth2 dance to get and return an authkey. The "ephemeral" -// property defaults to true if unspecified. The "preauthorized" defaults to -// false. The "baseURL" defaults to https://api.tailscale.com. -// The passed in tags are required, and must be non-empty. These will be -// set on the authkey generated by the OAuth2 dance. -func resolveAuthKey(ctx context.Context, v, tags string) (string, error) { - if !strings.HasPrefix(v, "tskey-client-") { - return v, nil - } - if tags == "" { - return "", errors.New("oauth authkeys require --advertise-tags") - } - - clientSecret, named, _ := strings.Cut(v, "?") - attrs, err := url.ParseQuery(named) - if err != nil { - return "", err - } - for k := range attrs { - switch k { - case "ephemeral", "preauthorized", "baseURL": - default: - return "", fmt.Errorf("unknown attribute %q", k) - } - } - getBool := func(name string, def bool) (bool, error) { - v := attrs.Get(name) - if v == "" { - return def, nil - } - ret, err := strconv.ParseBool(v) - if err != nil { - return false, fmt.Errorf("invalid attribute boolean attribute %s value %q", name, v) - } - return ret, nil - } - ephemeral, err := getBool("ephemeral", true) - if err != nil { - return "", err - } - preauth, err := getBool("preauthorized", false) - if err != nil { - return "", err - } - - baseURL := "https://api.tailscale.com" - if v := attrs.Get("baseURL"); v != "" { - baseURL = v - } - - credentials := clientcredentials.Config{ - ClientID: "some-client-id", // ignored - ClientSecret: clientSecret, - TokenURL: baseURL + "/api/v2/oauth/token", - } - - tsClient := tailscale.NewClient("-", nil) - tsClient.UserAgent = "tailscale-cli" - tsClient.HTTPClient = credentials.Client(ctx) - tsClient.BaseURL = baseURL - - caps := tailscale.KeyCapabilities{ - Devices: tailscale.KeyDeviceCapabilities{ - Create: tailscale.KeyDeviceCreateCapabilities{ - Reusable: false, - Ephemeral: ephemeral, - Preauthorized: preauth, - Tags: strings.Split(tags, ","), - }, - }, - } - - authkey, _, err := tsClient.CreateKey(ctx, caps) - if err != nil { - return "", err - } - return authkey, nil -} - -func warnOnAdvertiseRouts(ctx context.Context, prefs *ipn.Prefs) { - if len(prefs.AdvertiseRoutes) > 0 || prefs.AppConnector.Advertise { +func warnOnAdvertiseRoutes(ctx context.Context, prefs *ipn.Prefs) { + if buildfeatures.HasAdvertiseRoutes && len(prefs.AdvertiseRoutes) > 0 || + buildfeatures.HasAppConnectors && prefs.AppConnector.Advertise { // TODO(jwhited): compress CheckIPForwarding and CheckUDPGROForwarding // into a single HTTP req. if err := localClient.CheckIPForwarding(ctx); err != nil { diff --git a/cmd/tailscale/cli/up_test.go b/cmd/tailscale/cli/up_test.go index eb06f84dce2ea..9af8eae7d9994 100644 --- a/cmd/tailscale/cli/up_test.go +++ b/cmd/tailscale/cli/up_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -35,6 +35,7 @@ var validUpFlags = set.Of( "operator", "report-posture", "qr", + "qr-format", "reset", "shields-up", "snat-subnet-routes", @@ -42,6 +43,10 @@ var validUpFlags = set.Of( "stateful-filtering", "timeout", "unattended", + "client-id", + "client-secret", + "id-token", + "audience", ) // TestUpFlagSetIsFrozen complains when new flags are added to tailscale up. diff --git a/cmd/tailscale/cli/update.go b/cmd/tailscale/cli/update.go index 69d1aa97b43f7..9f2a608965174 100644 --- a/cmd/tailscale/cli/update.go +++ b/cmd/tailscale/cli/update.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_clientupdate + package cli import ( @@ -8,15 +10,27 @@ import ( "errors" "flag" "fmt" + "io" "runtime" - "strings" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/clientupdate" + "tailscale.com/util/prompt" "tailscale.com/version" "tailscale.com/version/distro" ) +func init() { + maybeUpdateCmd = func() *ffcli.Command { return updateCmd } + + clientupdateLatestTailscaleVersion.Set(func(track string) (string, error) { + if track == "" { + return clientupdate.LatestTailscaleVersion(clientupdate.CurrentTrack) + } + return clientupdate.LatestTailscaleVersion(track) + }) +} + var updateCmd = &ffcli.Command{ Name: "update", ShortUsage: "tailscale update", @@ -40,7 +54,7 @@ var updateCmd = &ffcli.Command{ distro.Get() != distro.Synology && runtime.GOOS != "freebsd" && runtime.GOOS != "darwin" { - fs.StringVar(&updateArgs.track, "track", "", `which track to check for updates: "stable" or "unstable" (dev); empty means same as current`) + fs.StringVar(&updateArgs.track, "track", "", `which track to check for updates: "stable", "release-candidate", or "unstable" (dev); empty means same as current`) fs.StringVar(&updateArgs.version, "version", "", `explicit version to update/downgrade to`) } return fs @@ -54,8 +68,19 @@ var updateArgs struct { version string // explicit version; empty means auto } +const gokrazyUpdateFromURLMagicArg = "--gokrazy-update-from-url" + func runUpdate(ctx context.Context, args []string) error { if len(args) > 0 { + if runtime.GOOS == "linux" && distro.Get() == distro.Gokrazy { + gokArgs, err := gokrazyUpdateArgsFromMagicArg(args) + if err != nil { + return err + } + if gokArgs != nil { + return clientupdate.GokrazyUpdateFromURL.Get()(ctx, *gokArgs) + } + } return flag.ErrHelp } if updateArgs.version != "" && updateArgs.track != "" { @@ -87,19 +112,41 @@ func confirmUpdate(ver string) bool { } msg := fmt.Sprintf("This will update Tailscale from %v to %v. Continue?", version.Short(), ver) - return promptYesNo(msg) + return prompt.YesNo(msg, true) } -// PromptYesNo takes a question and prompts the user to answer the -// question with a yes or no. It appends a [y/n] to the message. -func promptYesNo(msg string) bool { - fmt.Print(msg + " [y/n] ") - var resp string - fmt.Scanln(&resp) - resp = strings.ToLower(resp) - switch resp { - case "y", "yes", "sure": - return true +// gokrazyUpdateArgsFromMagicArg parses the Gokrazy update-from-URL command-line +// flow. It returns nil if args do not select that flow. A non-nil result means +// the caller may safely invoke clientupdate.GokrazyUpdateFromURL. +func gokrazyUpdateArgsFromMagicArg(args []string) (*clientupdate.GokrazyUpdateArgs, error) { + var updateURL string + var unsigned bool + + fs := flag.NewFlagSet("gokrazy-update", flag.ContinueOnError) + fs.SetOutput(io.Discard) + // This flag path is exercised end-to-end by TestGokrazyUpdatesItselfToSameImage. + fs.StringVar(&updateURL, gokrazyUpdateFromURLMagicArg[2:], "", "URL of the Gokrazy archive format file to install") + fs.BoolVar(&unsigned, "unsigned", false, "allow an unsigned GAF; for tests only") + if err := fs.Parse(args); err != nil { + return nil, err + } + if fs.NArg() != 0 { + return nil, nil + } + if updateURL == "" { + return nil, nil + } + if !unsigned { + return nil, errors.New("signed GAF verification is not implemented yet; see https://github.com/tailscale/tailscale/issues/20002; pass --unsigned for test updates") + } + if !clientupdate.GokrazyUpdateFromURL.IsSet() { + return nil, errors.New("gokrazy update support is not linked into this binary") } - return false + return &clientupdate.GokrazyUpdateArgs{ + URL: updateURL, + AllowUnsigned: unsigned, + Logf: func(format string, args ...any) { + printf(format+"\n", args...) + }, + }, nil } diff --git a/cmd/tailscale/cli/version.go b/cmd/tailscale/cli/version.go index b25502d5a4be5..3d6590a39bf2e 100644 --- a/cmd/tailscale/cli/version.go +++ b/cmd/tailscale/cli/version.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -10,7 +10,7 @@ import ( "fmt" "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/clientupdate" + "tailscale.com/feature" "tailscale.com/ipn/ipnstate" "tailscale.com/version" ) @@ -24,6 +24,7 @@ var versionCmd = &ffcli.Command{ fs.BoolVar(&versionArgs.daemon, "daemon", false, "also print local node's daemon version") fs.BoolVar(&versionArgs.json, "json", false, "output in JSON format") fs.BoolVar(&versionArgs.upstream, "upstream", false, "fetch and print the latest upstream release version from pkgs.tailscale.com") + fs.StringVar(&versionArgs.track, "track", "", `which track to check for updates: "stable", "release-candidate", or "unstable" (dev); empty means same as current`) return fs })(), Exec: runVersion, @@ -33,8 +34,11 @@ var versionArgs struct { daemon bool // also check local node's daemon version json bool upstream bool + track string } +var clientupdateLatestTailscaleVersion feature.Hook[func(string) (string, error)] + func runVersion(ctx context.Context, args []string) error { if len(args) > 0 { return fmt.Errorf("too many non-flag arguments: %q", args) @@ -51,7 +55,11 @@ func runVersion(ctx context.Context, args []string) error { var upstreamVer string if versionArgs.upstream { - upstreamVer, err = clientupdate.LatestTailscaleVersion(clientupdate.CurrentTrack) + f, ok := clientupdateLatestTailscaleVersion.GetOk() + if !ok { + return fmt.Errorf("fetching latest version not supported in this build") + } + upstreamVer, err = f(versionArgs.track) if err != nil { return err } diff --git a/cmd/tailscale/cli/wait.go b/cmd/tailscale/cli/wait.go new file mode 100644 index 0000000000000..ce9c6aba65b40 --- /dev/null +++ b/cmd/tailscale/cli/wait.go @@ -0,0 +1,157 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "flag" + "fmt" + "net" + "net/netip" + "strings" + "time" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/ipn" + "tailscale.com/types/logger" + "tailscale.com/util/backoff" +) + +var waitCmd = &ffcli.Command{ + Name: "wait", + ShortHelp: "Wait for Tailscale interface/IPs to be ready for binding", + LongHelp: strings.TrimSpace(` +Wait for Tailscale resources to be available. As of 2026-01-02, the only +resource that's available to wait for by is the Tailscale interface and its +IPs. + +With no arguments, this command will block until tailscaled is up, its backend is running, +and the Tailscale interface is up and has a Tailscale IP address assigned to it. + +If running in userspace-networking mode, this command only waits for tailscaled and +the Running state, as no physical network interface exists. + +A future version of this command may support waiting for other types of resources. + +The command returns exit code 0 on success, and non-zero on failure or timeout. + +To wait on a specific type of IP address, use 'tailscale ip' in combination with +the 'tailscale wait' command. For example, to wait for an IPv4 address: + + tailscale wait && tailscale ip --assert= + +Linux systemd users can wait for the "tailscale-online.target" target, which runs +this command. + +More generally, a service that wants to bind to (listen on) a Tailscale interface or IP address +can run it like 'tailscale wait && /path/to/service [...]' to ensure that Tailscale is ready +before the program starts. +`), + + ShortUsage: "tailscale wait", + Exec: runWait, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("wait") + fs.DurationVar(&waitArgs.timeout, "timeout", 0, "how long to wait before giving up (0 means wait indefinitely)") + return fs + })(), +} + +var waitArgs struct { + timeout time.Duration +} + +func runWait(ctx context.Context, args []string) error { + if len(args) > 0 { + return fmt.Errorf("unexpected arguments: %q", args) + } + if waitArgs.timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, waitArgs.timeout) + defer cancel() + } + + bo := backoff.NewBackoff("wait", logger.Discard, 2*time.Second) + for { + _, err := localClient.StatusWithoutPeers(ctx) + bo.BackOff(ctx, err) + if err == nil { + break + } + if ctx.Err() != nil { + return ctx.Err() + } + } + + watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialState) + if err != nil { + return err + } + defer watcher.Close() + var firstIP netip.Addr + for { + not, err := watcher.Next() + if err != nil { + return err + } + if not.State != nil && *not.State == ipn.Running { + + st, err := localClient.StatusWithoutPeers(ctx) + if err != nil { + return err + } + if len(st.TailscaleIPs) > 0 { + firstIP = st.TailscaleIPs[0] + break + } + } + } + + st, err := localClient.StatusWithoutPeers(ctx) + if err != nil { + return err + } + if !st.TUN { + // No TUN; nothing more to wait for. + return nil + } + + // Verify we have an interface using that IP. + for { + err := checkForInterfaceIP(firstIP) + if err == nil { + return nil + } + bo.BackOff(ctx, err) + if ctx.Err() != nil { + return ctx.Err() + } + } +} + +func checkForInterfaceIP(ip netip.Addr) error { + ifs, err := net.Interfaces() + if err != nil { + return err + } + for _, ifi := range ifs { + addrs, err := ifi.Addrs() + if err != nil { + return err + } + for _, addr := range addrs { + var aip netip.Addr + switch v := addr.(type) { + case *net.IPNet: + aip, _ = netip.AddrFromSlice(v.IP) + case *net.IPAddr: + aip, _ = netip.AddrFromSlice(v.IP) + } + if aip.Unmap() == ip { + return nil + } + } + } + return fmt.Errorf("no interface has IP %v", ip) +} diff --git a/cmd/tailscale/cli/web.go b/cmd/tailscale/cli/web.go index 5e1821dd011eb..c13cad2d645ce 100644 --- a/cmd/tailscale/cli/web.go +++ b/cmd/tailscale/cli/web.go @@ -1,6 +1,8 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause +//go:build !ts_omit_webclient + package cli import ( @@ -22,14 +24,20 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/client/web" "tailscale.com/ipn" + "tailscale.com/tsconst" ) -var webCmd = &ffcli.Command{ - Name: "web", - ShortUsage: "tailscale web [flags]", - ShortHelp: "Run a web server for controlling Tailscale", +func init() { + maybeWebCmd = webCmd +} + +func webCmd() *ffcli.Command { + return &ffcli.Command{ + Name: "web", + ShortUsage: "tailscale web [flags]", + ShortHelp: "Run a web server for controlling Tailscale", - LongHelp: strings.TrimSpace(` + LongHelp: strings.TrimSpace(` "tailscale web" runs a webserver for controlling the Tailscale daemon. It's primarily intended for use on Synology, QNAP, and other @@ -37,16 +45,17 @@ NAS devices where a web interface is the natural place to control Tailscale, as opposed to a CLI or a native app. `), - FlagSet: (func() *flag.FlagSet { - webf := newFlagSet("web") - webf.StringVar(&webArgs.listen, "listen", "localhost:8088", "listen address; use port 0 for automatic") - webf.BoolVar(&webArgs.cgi, "cgi", false, "run as CGI script") - webf.StringVar(&webArgs.prefix, "prefix", "", "URL prefix added to requests (for cgi or reverse proxies)") - webf.BoolVar(&webArgs.readonly, "readonly", false, "run web UI in read-only mode") - webf.StringVar(&webArgs.origin, "origin", "", "origin at which the web UI is served (if behind a reverse proxy or used with cgi)") - return webf - })(), - Exec: runWeb, + FlagSet: (func() *flag.FlagSet { + webf := newFlagSet("web") + webf.StringVar(&webArgs.listen, "listen", "localhost:8088", "listen address; use port 0 for automatic") + webf.BoolVar(&webArgs.cgi, "cgi", false, "run as CGI script") + webf.StringVar(&webArgs.prefix, "prefix", "", "URL prefix added to requests (for cgi or reverse proxies)") + webf.BoolVar(&webArgs.readonly, "readonly", false, "run web UI in read-only mode") + webf.StringVar(&webArgs.origin, "origin", "", "origin at which the web UI is served (if behind a reverse proxy or used with cgi)") + return webf + })(), + Exec: runWeb, + } } var webArgs struct { @@ -101,7 +110,7 @@ func runWeb(ctx context.Context, args []string) error { var startedManagementClient bool // we started the management client if !existingWebClient && !webArgs.readonly { // Also start full client in tailscaled. - log.Printf("starting tailscaled web client at http://%s\n", netip.AddrPortFrom(selfIP, web.ListenPort)) + log.Printf("starting tailscaled web client at http://%s\n", netip.AddrPortFrom(selfIP, tsconst.WebListenPort)) if err := setRunWebClient(ctx, true); err != nil { return fmt.Errorf("starting web client in tailscaled: %w", err) } diff --git a/cmd/tailscale/cli/web_test.go b/cmd/tailscale/cli/web_test.go index f2470b364c41e..727c5644be0b4 100644 --- a/cmd/tailscale/cli/web_test.go +++ b/cmd/tailscale/cli/web_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli diff --git a/cmd/tailscale/cli/whoami.go b/cmd/tailscale/cli/whoami.go new file mode 100644 index 0000000000000..495cb0c3d667b --- /dev/null +++ b/cmd/tailscale/cli/whoami.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" +) + +var whoamiCmd = &ffcli.Command{ + Name: "whoami", + ShortUsage: "tailscale whoami [--json]", + ShortHelp: "Show the machine and user identity of the current machine", + LongHelp: strings.TrimSpace(` + 'tailscale whoami' shows the machine and user identity of the current machine. + It is equivalent to running 'tailscale whois' against one of the current machine's own Tailscale IP addresses. + `), + Exec: runWhoami, + FlagSet: func() *flag.FlagSet { + fs := newFlagSet("whoami") + fs.BoolVar(&whoamiArgs.json, "json", false, "output in JSON format") + return fs + }(), +} + +var whoamiArgs struct { + json bool // output in JSON format +} + +func runWhoami(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("too many arguments, expected none") + } + st, err := localClient.StatusWithoutPeers(ctx) + if err != nil { + return err + } + if len(st.TailscaleIPs) == 0 { + return fmt.Errorf("no current Tailscale IP address; state: %v", st.BackendState) + } + who, err := localClient.WhoIsProto(ctx, "", st.TailscaleIPs[0].String()) + if err != nil { + return err + } + return printWhoIs(who, whoamiArgs.json) +} diff --git a/cmd/tailscale/cli/whois.go b/cmd/tailscale/cli/whois.go index 44ff68dec8777..0b8e407d6d008 100644 --- a/cmd/tailscale/cli/whois.go +++ b/cmd/tailscale/cli/whois.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package cli @@ -13,6 +13,7 @@ import ( "text/tabwriter" "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/tailscale/apitype" ) var whoisCmd = &ffcli.Command{ @@ -26,7 +27,7 @@ var whoisCmd = &ffcli.Command{ FlagSet: func() *flag.FlagSet { fs := newFlagSet("whois") fs.BoolVar(&whoIsArgs.json, "json", false, "output in JSON format") - fs.StringVar(&whoIsArgs.proto, "proto", "", `protocol; one of "tcp" or "udp"; empty mans both `) + fs.StringVar(&whoIsArgs.proto, "proto", "", `protocol; one of "tcp" or "udp"; empty means both`) return fs }(), } @@ -46,7 +47,13 @@ func runWhoIs(ctx context.Context, args []string) error { if err != nil { return err } - if whoIsArgs.json { + return printWhoIs(who, whoIsArgs.json) +} + +// printWhoIs prints the WhoIsResponse to Stdout, either as JSON (if asJSON is +// true) or in a human-readable form. +func printWhoIs(who *apitype.WhoIsResponse, asJSON bool) error { + if asJSON { ec := json.NewEncoder(Stdout) ec.SetIndent("", " ") ec.Encode(who) diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 69d054ea42fb6..6fc246bd3c071 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -1,48 +1,148 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/depaware) + đŸ’Ŗ crypto/internal/entropy/v1.0.0 from crypto/internal/fips140/drbg filippo.io/edwards25519 from github.com/hdevalence/ed25519consensus filippo.io/edwards25519/field from filippo.io/edwards25519 + L fyne.io/systray from tailscale.com/client/systray + L fyne.io/systray/internal/generated/menu from fyne.io/systray + L fyne.io/systray/internal/generated/notifier from fyne.io/systray + L github.com/Kodeworks/golang-image-ico from tailscale.com/client/systray W đŸ’Ŗ github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+ W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate W đŸ’Ŗ github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy + L github.com/atotto/clipboard from tailscale.com/client/systray + github.com/aws/aws-sdk-go-v2/aws from github.com/aws/aws-sdk-go-v2/aws/defaults+ + L github.com/aws/aws-sdk-go-v2/aws/arn from tailscale.com/feature/awsparamstore + github.com/aws/aws-sdk-go-v2/aws/defaults from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/aws-sdk-go-v2/aws/middleware from github.com/aws/aws-sdk-go-v2/aws/retry+ + github.com/aws/aws-sdk-go-v2/aws/protocol/query from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/aws/protocol/restjson from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/aws-sdk-go-v2/aws/protocol/xml from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/aws/ratelimit from github.com/aws/aws-sdk-go-v2/aws/retry + github.com/aws/aws-sdk-go-v2/aws/retry from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client+ + github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 from github.com/aws/aws-sdk-go-v2/aws/signer/v4 + github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ + github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/config from tailscale.com/wif+ + github.com/aws/aws-sdk-go-v2/config/internal/ini from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds + github.com/aws/aws-sdk-go-v2/credentials/logincreds from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config + github.com/aws/aws-sdk-go-v2/feature/ec2/imds from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds + github.com/aws/aws-sdk-go-v2/internal/auth from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ + github.com/aws/aws-sdk-go-v2/internal/auth/smithy from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/aws-sdk-go-v2/internal/configsources from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/aws-sdk-go-v2/internal/context from github.com/aws/aws-sdk-go-v2/aws/retry+ + github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints+ + github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ + github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ + github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds + github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4+ + github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws + github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry + github.com/aws/aws-sdk-go-v2/internal/v4a from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/internal/v4a/internal/crypto from github.com/aws/aws-sdk-go-v2/internal/v4a + github.com/aws/aws-sdk-go-v2/internal/v4a/internal/v4 from github.com/aws/aws-sdk-go-v2/internal/v4a + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/service/signin from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/service/signin/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/signin + github.com/aws/aws-sdk-go-v2/service/signin/types from github.com/aws/aws-sdk-go-v2/credentials/logincreds+ + L github.com/aws/aws-sdk-go-v2/service/ssm from tailscale.com/feature/awsparamstore + L github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm + L github.com/aws/aws-sdk-go-v2/service/ssm/types from github.com/aws/aws-sdk-go-v2/service/ssm + github.com/aws/aws-sdk-go-v2/service/sso from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso + github.com/aws/aws-sdk-go-v2/service/sso/types from github.com/aws/aws-sdk-go-v2/service/sso + github.com/aws/aws-sdk-go-v2/service/ssooidc from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssooidc + github.com/aws/aws-sdk-go-v2/service/ssooidc/types from github.com/aws/aws-sdk-go-v2/service/ssooidc + github.com/aws/aws-sdk-go-v2/service/sts from github.com/aws/aws-sdk-go-v2/config+ + github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/aws-sdk-go-v2/service/sts/types from github.com/aws/aws-sdk-go-v2/credentials/stscreds+ + github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ + github.com/aws/smithy-go/auth from github.com/aws/aws-sdk-go-v2/internal/auth+ + github.com/aws/smithy-go/auth/bearer from github.com/aws/aws-sdk-go-v2/aws+ + github.com/aws/smithy-go/context from github.com/aws/smithy-go/auth/bearer + github.com/aws/smithy-go/document from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/smithy-go/encoding from github.com/aws/smithy-go/encoding/json+ + github.com/aws/smithy-go/encoding/httpbinding from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssooidc+ + github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts + github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/sso+ + github.com/aws/smithy-go/endpoints/private/rulesfn from github.com/aws/aws-sdk-go-v2/service/sts+ + github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer + github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ + github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ + github.com/aws/smithy-go/metrics from github.com/aws/aws-sdk-go-v2/aws/retry+ + github.com/aws/smithy-go/middleware from github.com/aws/aws-sdk-go-v2/aws+ + github.com/aws/smithy-go/private/requestcompression from github.com/aws/aws-sdk-go-v2/config + github.com/aws/smithy-go/ptr from github.com/aws/aws-sdk-go-v2/aws+ + github.com/aws/smithy-go/rand from github.com/aws/aws-sdk-go-v2/aws/middleware+ + github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/ssm+ + github.com/aws/smithy-go/tracing from github.com/aws/aws-sdk-go-v2/aws/middleware+ + github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws+ + github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http + L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm github.com/coder/websocket from tailscale.com/util/eventbus github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket - L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw W đŸ’Ŗ github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/pe+ W đŸ’Ŗ github.com/dblohm7/wingoes/pe from tailscale.com/util/winutil/authenticode + L github.com/fogleman/gg from tailscale.com/client/systray github.com/fxamacker/cbor/v2 from tailscale.com/tka + github.com/gaissmai/bart from tailscale.com/net/tsdial + github.com/gaissmai/bart/internal/allot from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/art from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/bitset from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/lpm from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/nodes from github.com/gaissmai/bart + github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/value from github.com/gaissmai/bart+ github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/v1 from tailscale.com/net/routecheck + L đŸ’Ŗ github.com/godbus/dbus/v5 from fyne.io/systray+ + L github.com/godbus/dbus/v5/introspect from fyne.io/systray+ + L github.com/godbus/dbus/v5/prop from fyne.io/systray + L github.com/golang/freetype/raster from github.com/fogleman/gg+ + L github.com/golang/freetype/truetype from github.com/fogleman/gg github.com/golang/groupcache/lru from tailscale.com/net/dnscache - L github.com/google/nftables from tailscale.com/util/linuxfw - L đŸ’Ŗ github.com/google/nftables/alignedbuff from github.com/google/nftables/xt - L đŸ’Ŗ github.com/google/nftables/binaryutil from github.com/google/nftables+ - L github.com/google/nftables/expr from github.com/google/nftables+ - L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ - L github.com/google/nftables/xt from github.com/google/nftables/expr+ DW github.com/google/uuid from tailscale.com/clientupdate+ github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ + github.com/huin/goupnp from github.com/huin/goupnp/dcps/internetgateway2+ + github.com/huin/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper + github.com/huin/goupnp/httpu from github.com/huin/goupnp+ + github.com/huin/goupnp/scpd from github.com/huin/goupnp + github.com/huin/goupnp/soap from github.com/huin/goupnp+ + github.com/huin/goupnp/ssdp from github.com/huin/goupnp + L github.com/jmespath/go-jmespath from github.com/aws/aws-sdk-go-v2/service/ssm L đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink github.com/kballard/go-shellquote from tailscale.com/cmd/tailscale/cli đŸ’Ŗ github.com/mattn/go-colorable from tailscale.com/cmd/tailscale/cli đŸ’Ŗ github.com/mattn/go-isatty from tailscale.com/cmd/tailscale/cli+ - L đŸ’Ŗ github.com/mdlayher/netlink from github.com/google/nftables+ + L đŸ’Ŗ github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+ L đŸ’Ŗ github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ - L github.com/mdlayher/netlink/nltest from github.com/google/nftables L đŸ’Ŗ github.com/mdlayher/socket from github.com/mdlayher/netlink - github.com/miekg/dns from tailscale.com/net/dns/recursive đŸ’Ŗ github.com/mitchellh/go-ps from tailscale.com/cmd/tailscale/cli+ github.com/peterbourgon/ff/v3 from github.com/peterbourgon/ff/v3/ffcli+ github.com/peterbourgon/ff/v3/ffcli from tailscale.com/cmd/tailscale/cli+ github.com/peterbourgon/ff/v3/internal from github.com/peterbourgon/ff/v3 - github.com/skip2/go-qrcode from tailscale.com/cmd/tailscale/cli + github.com/skip2/go-qrcode from tailscale.com/util/qrcodes github.com/skip2/go-qrcode/bitset from github.com/skip2/go-qrcode+ github.com/skip2/go-qrcode/reedsolomon from github.com/skip2/go-qrcode W đŸ’Ŗ github.com/tailscale/go-winio from tailscale.com/safesocket @@ -50,94 +150,106 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ - github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper - github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ - github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp - github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ - github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp - L đŸ’Ŗ github.com/tailscale/netlink from tailscale.com/util/linuxfw - L đŸ’Ŗ github.com/tailscale/netlink/nl from github.com/tailscale/netlink + github.com/tailscale/hujson from tailscale.com/ipn/conffile github.com/tailscale/web-client-prebuilt from tailscale.com/client/web - github.com/toqueteos/webbrowser from tailscale.com/cmd/tailscale/cli - L github.com/vishvananda/netns from github.com/tailscale/netlink+ + github.com/toqueteos/webbrowser from tailscale.com/cmd/tailscale/cli+ github.com/x448/float16 from github.com/fxamacker/cbor/v2 + go.yaml.in/yaml/v2 from sigs.k8s.io/yaml đŸ’Ŗ go4.org/mem from tailscale.com/client/local+ - go4.org/netipx from tailscale.com/net/tsaddr + go4.org/netipx from tailscale.com/net/tsaddr+ W đŸ’Ŗ golang.zx2c4.com/wireguard/windows/tunnel/winipcfg from tailscale.com/net/netmon+ k8s.io/client-go/util/homedir from tailscale.com/cmd/tailscale/cli sigs.k8s.io/yaml from tailscale.com/cmd/tailscale/cli - sigs.k8s.io/yaml/goyaml.v2 from sigs.k8s.io/yaml software.sslmate.com/src/go-pkcs12 from tailscale.com/cmd/tailscale/cli software.sslmate.com/src/go-pkcs12/internal/rc2 from software.sslmate.com/src/go-pkcs12 tailscale.com from tailscale.com/version đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/cmd/tailscale/cli+ + L tailscale.com/client/freedesktop from tailscale.com/client/systray tailscale.com/client/local from tailscale.com/client/tailscale+ - tailscale.com/client/tailscale from tailscale.com/cmd/tailscale/cli+ + L tailscale.com/client/systray from tailscale.com/cmd/tailscale/cli + tailscale.com/client/tailscale from tailscale.com/internal/client/tailscale tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ tailscale.com/client/web from tailscale.com/cmd/tailscale/cli - tailscale.com/clientupdate from tailscale.com/client/web+ + tailscale.com/clientupdate from tailscale.com/cmd/tailscale/cli LW tailscale.com/clientupdate/distsign from tailscale.com/clientupdate tailscale.com/cmd/tailscale/cli from tailscale.com/cmd/tailscale tailscale.com/cmd/tailscale/cli/ffcomplete from tailscale.com/cmd/tailscale/cli tailscale.com/cmd/tailscale/cli/ffcomplete/internal from tailscale.com/cmd/tailscale/cli/ffcomplete + tailscale.com/cmd/tailscale/cli/jsonoutput from tailscale.com/cmd/tailscale/cli tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ - tailscale.com/control/controlhttp from tailscale.com/cmd/tailscale/cli + tailscale.com/control/controlhttp from tailscale.com/control/ts2021 tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp - tailscale.com/control/controlknobs from tailscale.com/net/portmapper + tailscale.com/control/ts2021 from tailscale.com/cmd/tailscale/cli tailscale.com/derp from tailscale.com/derp/derphttp+ - tailscale.com/derp/derpconst from tailscale.com/derp+ + tailscale.com/derp/derpconst from tailscale.com/derp/derphttp+ tailscale.com/derp/derphttp from tailscale.com/net/netcheck - tailscale.com/disco from tailscale.com/derp tailscale.com/drive from tailscale.com/client/local+ tailscale.com/envknob from tailscale.com/client/local+ tailscale.com/envknob/featureknob from tailscale.com/client/web - tailscale.com/feature from tailscale.com/tsweb + tailscale.com/feature from tailscale.com/tsweb+ + L tailscale.com/feature/awsparamstore from tailscale.com/feature/condregister/awsparamstore + tailscale.com/feature/buildfeatures from tailscale.com/cmd/tailscale/cli+ tailscale.com/feature/capture/dissector from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/awsparamstore from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/identityfederation from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/oauthkey from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/portmapper from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/useproxy from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/identityfederation from tailscale.com/feature/condregister/identityfederation + tailscale.com/feature/oauthkey from tailscale.com/feature/condregister/oauthkey + tailscale.com/feature/portmapper from tailscale.com/feature/condregister/portmapper + tailscale.com/feature/syspolicy from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/useproxy from tailscale.com/feature/condregister/useproxy tailscale.com/health from tailscale.com/net/tlsdial+ tailscale.com/health/healthmsg from tailscale.com/cmd/tailscale/cli tailscale.com/hostinfo from tailscale.com/client/web+ - tailscale.com/internal/client/tailscale from tailscale.com/cmd/tailscale/cli - tailscale.com/internal/noiseconn from tailscale.com/cmd/tailscale/cli + tailscale.com/internal/client/tailscale from tailscale.com/cmd/tailscale/cli+ tailscale.com/ipn from tailscale.com/client/local+ + tailscale.com/ipn/conffile from tailscale.com/cmd/tailscale/cli tailscale.com/ipn/ipnstate from tailscale.com/client/local+ tailscale.com/kube/kubetypes from tailscale.com/envknob tailscale.com/licenses from tailscale.com/client/web+ - tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/metrics from tailscale.com/tsweb+ + tailscale.com/net/ace from tailscale.com/cmd/tailscale/cli tailscale.com/net/bakedroots from tailscale.com/net/tlsdial tailscale.com/net/captivedetection from tailscale.com/net/netcheck - tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback tailscale.com/net/dnscache from tailscale.com/control/controlhttp+ tailscale.com/net/dnsfallback from tailscale.com/control/controlhttp+ tailscale.com/net/netaddr from tailscale.com/ipn+ tailscale.com/net/netcheck from tailscale.com/cmd/tailscale/cli tailscale.com/net/neterror from tailscale.com/net/netcheck+ - tailscale.com/net/netknob from tailscale.com/net/netns + tailscale.com/net/netknob from tailscale.com/net/netns+ đŸ’Ŗ tailscale.com/net/netmon from tailscale.com/cmd/tailscale/cli+ đŸ’Ŗ tailscale.com/net/netns from tailscale.com/derp/derphttp+ tailscale.com/net/netutil from tailscale.com/client/local+ tailscale.com/net/netx from tailscale.com/control/controlhttp+ tailscale.com/net/ping from tailscale.com/net/netcheck - tailscale.com/net/portmapper from tailscale.com/cmd/tailscale/cli+ + tailscale.com/net/portmapper from tailscale.com/feature/portmapper + tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ + tailscale.com/net/routecheck from tailscale.com/client/local+ tailscale.com/net/sockstats from tailscale.com/control/controlhttp+ tailscale.com/net/stun from tailscale.com/net/netcheck - L tailscale.com/net/tcpinfo from tailscale.com/derp tailscale.com/net/tlsdial from tailscale.com/cmd/tailscale/cli+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/net/routecheck tailscale.com/net/tsaddr from tailscale.com/client/web+ - đŸ’Ŗ tailscale.com/net/tshttpproxy from tailscale.com/clientupdate/distsign+ + tailscale.com/net/tsdial from tailscale.com/cmd/tailscale/cli+ + đŸ’Ŗ tailscale.com/net/tshttpproxy from tailscale.com/feature/useproxy + tailscale.com/net/udprelay/status from tailscale.com/client/local+ + tailscale.com/omit from tailscale.com/ipn/conffile tailscale.com/paths from tailscale.com/client/local+ đŸ’Ŗ tailscale.com/safesocket from tailscale.com/client/local+ - tailscale.com/syncs from tailscale.com/cmd/tailscale/cli+ + tailscale.com/syncs from tailscale.com/control/controlhttp+ tailscale.com/tailcfg from tailscale.com/client/local+ tailscale.com/tempfork/spf13/cobra from tailscale.com/cmd/tailscale/cli/ffcomplete+ tailscale.com/tka from tailscale.com/client/local+ tailscale.com/tsconst from tailscale.com/net/netmon+ tailscale.com/tstime from tailscale.com/control/controlhttp+ tailscale.com/tstime/mono from tailscale.com/tstime/rate - tailscale.com/tstime/rate from tailscale.com/cmd/tailscale/cli+ - tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tstime/rate from tailscale.com/cmd/tailscale/cli + tailscale.com/tsweb from tailscale.com/util/eventbus+ tailscale.com/tsweb/varz from tailscale.com/util/usermetric+ + tailscale.com/types/appctype from tailscale.com/client/local+ tailscale.com/types/dnstype from tailscale.com/tailcfg+ tailscale.com/types/empty from tailscale.com/ipn tailscale.com/types/ipproto from tailscale.com/ipn+ @@ -147,14 +259,15 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/types/netmap from tailscale.com/ipn+ tailscale.com/types/nettype from tailscale.com/net/netcheck+ tailscale.com/types/opt from tailscale.com/client/tailscale+ - tailscale.com/types/persist from tailscale.com/ipn + tailscale.com/types/persist from tailscale.com/ipn+ tailscale.com/types/preftype from tailscale.com/cmd/tailscale/cli+ - tailscale.com/types/ptr from tailscale.com/hostinfo+ tailscale.com/types/result from tailscale.com/util/lineiter tailscale.com/types/structs from tailscale.com/ipn+ tailscale.com/types/tkatype from tailscale.com/types/key+ tailscale.com/types/views from tailscale.com/tailcfg+ - tailscale.com/util/cibuild from tailscale.com/health + tailscale.com/util/backoff from tailscale.com/cmd/tailscale/cli + tailscale.com/util/bufiox from tailscale.com/types/key + tailscale.com/util/cibuild from tailscale.com/health+ tailscale.com/util/clientmetric from tailscale.com/net/netcheck+ tailscale.com/util/cloudenv from tailscale.com/net/dnscache+ tailscale.com/util/cmpver from tailscale.com/net/tshttpproxy+ @@ -162,27 +275,31 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep đŸ’Ŗ tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting L đŸ’Ŗ tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/cmd/tailscale/cli+ - tailscale.com/util/eventbus from tailscale.com/net/portmapper+ + tailscale.com/util/eventbus from tailscale.com/client/local+ tailscale.com/util/groupmember from tailscale.com/client/web đŸ’Ŗ tailscale.com/util/hashx from tailscale.com/util/deephash tailscale.com/util/httpm from tailscale.com/client/tailscale+ tailscale.com/util/lineiter from tailscale.com/hostinfo+ - L tailscale.com/util/linuxfw from tailscale.com/net/netns tailscale.com/util/mak from tailscale.com/cmd/tailscale/cli+ - tailscale.com/util/multierr from tailscale.com/control/controlhttp+ tailscale.com/util/must from tailscale.com/clientupdate/distsign+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + tailscale.com/util/prompt from tailscale.com/cmd/tailscale/cli + đŸ’Ŗ tailscale.com/util/qrcodes from tailscale.com/cmd/tailscale/cli tailscale.com/util/quarantine from tailscale.com/cmd/tailscale/cli tailscale.com/util/rands from tailscale.com/tsweb - tailscale.com/util/set from tailscale.com/derp+ - tailscale.com/util/singleflight from tailscale.com/net/dnscache+ - tailscale.com/util/slicesx from tailscale.com/net/dns/recursive+ - tailscale.com/util/syspolicy from tailscale.com/ipn + tailscale.com/util/set from tailscale.com/ipn+ + tailscale.com/util/singleflight from tailscale.com/net/dnscache + tailscale.com/util/slicesx from tailscale.com/client/systray+ + L tailscale.com/util/stringsx from tailscale.com/client/systray + tailscale.com/util/syspolicy from tailscale.com/feature/syspolicy tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy/setting+ - tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy/internal/metrics+ + tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy+ tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source + tailscale.com/util/syspolicy/pkey from tailscale.com/ipn+ + tailscale.com/util/syspolicy/policyclient from tailscale.com/client/web+ + tailscale.com/util/syspolicy/ptype from tailscale.com/util/syspolicy/policyclient+ tailscale.com/util/syspolicy/rsop from tailscale.com/util/syspolicy - tailscale.com/util/syspolicy/setting from tailscale.com/util/syspolicy+ + tailscale.com/util/syspolicy/setting from tailscale.com/client/local+ tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy+ tailscale.com/util/testenv from tailscale.com/cmd/tailscale/cli+ tailscale.com/util/truncate from tailscale.com/cmd/tailscale/cli @@ -195,13 +312,12 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/version from tailscale.com/client/web+ tailscale.com/version/distro from tailscale.com/client/web+ tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap + tailscale.com/wif from tailscale.com/feature/identityfederation golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from tailscale.com/clientupdate/distsign+ golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305 - golang.org/x/crypto/chacha20poly1305 from crypto/tls+ - golang.org/x/crypto/cryptobyte from crypto/ecdsa+ - golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + golang.org/x/crypto/chacha20poly1305 from tailscale.com/control/controlbase golang.org/x/crypto/curve25519 from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/hkdf from tailscale.com/control/controlbase golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ @@ -211,29 +327,30 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep golang.org/x/crypto/pbkdf2 from software.sslmate.com/src/go-pkcs12 golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ - golang.org/x/exp/maps from tailscale.com/util/syspolicy/internal/metrics+ + golang.org/x/exp/maps from tailscale.com/util/syspolicy/setting+ + L golang.org/x/image/draw from github.com/fogleman/gg + L golang.org/x/image/font from github.com/fogleman/gg+ + L golang.org/x/image/font/basicfont from github.com/fogleman/gg + L golang.org/x/image/math/f64 from github.com/fogleman/gg+ + L golang.org/x/image/math/fixed from github.com/fogleman/gg+ golang.org/x/net/bpf from github.com/mdlayher/netlink+ - golang.org/x/net/dns/dnsmessage from net+ - golang.org/x/net/http/httpguts from net/http+ - golang.org/x/net/http/httpproxy from net/http+ - golang.org/x/net/http2 from tailscale.com/cmd/tailscale/cli+ - golang.org/x/net/http2/hpack from net/http+ + golang.org/x/net/dns/dnsmessage from tailscale.com/cmd/tailscale/cli+ + golang.org/x/net/http/httpproxy from tailscale.com/net/tshttpproxy golang.org/x/net/icmp from tailscale.com/net/ping - golang.org/x/net/idna from golang.org/x/net/http/httpguts+ - golang.org/x/net/internal/httpcommon from golang.org/x/net/http2 + golang.org/x/net/idna from golang.org/x/net/http/httpproxy+ golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/internal/socks from golang.org/x/net/proxy - golang.org/x/net/ipv4 from github.com/miekg/dns+ - golang.org/x/net/ipv6 from github.com/miekg/dns+ + golang.org/x/net/ipv4 from golang.org/x/net/icmp+ + golang.org/x/net/ipv6 from golang.org/x/net/icmp+ golang.org/x/net/proxy from tailscale.com/net/netns - D golang.org/x/net/route from net+ - golang.org/x/oauth2 from golang.org/x/oauth2/clientcredentials - golang.org/x/oauth2/clientcredentials from tailscale.com/cmd/tailscale/cli + D golang.org/x/net/route from tailscale.com/net/netmon+ + golang.org/x/oauth2 from golang.org/x/oauth2/clientcredentials+ + golang.org/x/oauth2/clientcredentials from tailscale.com/feature/oauthkey golang.org/x/oauth2/internal from golang.org/x/oauth2+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ golang.org/x/sys/cpu from golang.org/x/crypto/argon2+ - LD golang.org/x/sys/unix from github.com/google/nftables+ + LD golang.org/x/sys/unix from github.com/jsimonetti/rtnetlink/internal/unix+ W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ W golang.org/x/sys/windows/svc from golang.org/x/sys/windows/svc/mgr+ @@ -243,7 +360,24 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ golang.org/x/text/unicode/norm from golang.org/x/net/idna golang.org/x/time/rate from tailscale.com/cmd/tailscale/cli+ + vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ + vendor/golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + vendor/golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + vendor/golang.org/x/crypto/internal/alias from vendor/golang.org/x/crypto/chacha20+ + vendor/golang.org/x/crypto/internal/poly1305 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/net/dns/dnsmessage from net + vendor/golang.org/x/net/http/httpguts from net/http+ + vendor/golang.org/x/net/http/httpproxy from net/http + vendor/golang.org/x/net/http2/hpack from net/http+ + vendor/golang.org/x/net/idna from net/http+ + vendor/golang.org/x/sys/cpu from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/text/secure/bidirule from vendor/golang.org/x/net/idna + vendor/golang.org/x/text/transform from vendor/golang.org/x/text/secure/bidirule+ + vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ + vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna archive/tar from tailscale.com/clientupdate + L archive/zip from tailscale.com/clientupdate bufio from compress/flate+ bytes from archive/tar+ cmp from slices+ @@ -253,7 +387,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep container/list from crypto/tls+ context from crypto/tls+ crypto from crypto/ecdh+ - crypto/aes from crypto/internal/hpke+ + crypto/aes from crypto/tls+ crypto/cipher from crypto/aes+ crypto/des from crypto/tls+ crypto/dsa from crypto/x509 @@ -261,11 +395,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/ecdsa from crypto/tls+ crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ + crypto/fips140 from crypto/tls/internal/fips140tls+ + crypto/hkdf from crypto/hpke+ crypto/hmac from crypto/tls+ + crypto/hpke from crypto/tls crypto/internal/boring from crypto/aes+ crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring - crypto/internal/entropy from crypto/internal/fips140/drbg + crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ crypto/internal/fips140 from crypto/internal/fips140/aes+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ @@ -280,9 +417,10 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/mlkem crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec + crypto/internal/fips140/pbkdf2 from crypto/pbkdf2 crypto/internal/fips140/rsa from crypto/rsa crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ crypto/internal/fips140/sha3 from crypto/internal/fips140/hmac+ @@ -290,25 +428,29 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ crypto/internal/fips140/tls12 from crypto/tls crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140cache from crypto/ecdsa+ crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140deps/time from crypto/internal/entropy/v1.0.0 crypto/internal/fips140hash from crypto/ecdsa+ crypto/internal/fips140only from crypto/cipher+ - crypto/internal/hpke from crypto/tls crypto/internal/impl from crypto/internal/fips140/aes+ - crypto/internal/randutil from crypto/dsa+ - crypto/internal/sysrand from crypto/internal/entropy+ + crypto/internal/rand from crypto/dsa+ + crypto/internal/randutil from crypto/internal/rand + crypto/internal/sysrand from crypto/internal/fips140/drbg crypto/md5 from crypto/tls+ + crypto/mlkem from crypto/hpke+ + crypto/pbkdf2 from golang.org/x/crypto/pbkdf2 crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls crypto/rsa from crypto/tls+ crypto/sha1 from crypto/tls+ crypto/sha256 from crypto/tls+ - crypto/sha3 from crypto/internal/fips140hash + crypto/sha3 from crypto/internal/fips140hash+ crypto/sha512 from crypto/ecdsa+ crypto/subtle from crypto/cipher+ - crypto/tls from github.com/miekg/dns+ + crypto/tls from net/http+ crypto/tls/internal/fips140tls from crypto/tls crypto/x509 from crypto/tls+ D crypto/x509/internal/macos from crypto/x509 @@ -325,20 +467,24 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ - encoding/xml from github.com/tailscale/goupnp+ + encoding/xml from github.com/godbus/dbus/v5/introspect+ errors from archive/tar+ - expvar from tailscale.com/derp+ + expvar from tailscale.com/health+ flag from github.com/peterbourgon/ff/v3+ fmt from archive/tar+ hash from compress/zlib+ hash/adler32 from compress/zlib hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic hash/maphash from go4.org/mem html from html/template+ html/template from tailscale.com/util/eventbus image from github.com/skip2/go-qrcode+ image/color from github.com/skip2/go-qrcode+ - image/png from github.com/skip2/go-qrcode + L image/draw from github.com/Kodeworks/golang-image-ico+ + L image/internal/imageutil from image/draw+ + L image/jpeg from github.com/fogleman/gg + image/png from github.com/skip2/go-qrcode+ internal/abi from crypto/x509/internal/macos+ internal/asan from internal/runtime/maps+ internal/bisect from internal/godebug @@ -352,9 +498,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep internal/goarch from crypto/internal/fips140deps/cpu+ internal/godebug from archive/tar+ internal/godebugs from internal/godebug+ - internal/goexperiment from hash/maphash+ + internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ - internal/itoa from internal/poll+ internal/msan from internal/runtime/maps+ internal/nettrace from net+ internal/oserror from io/fs+ @@ -363,22 +508,31 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep internal/profilerecord from runtime+ internal/race from internal/poll+ internal/reflectlite from context+ + D internal/routebsd from net internal/runtime/atomic from internal/runtime/exithook+ + L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime + internal/runtime/gc from runtime+ + internal/runtime/gc/scan from runtime internal/runtime/maps from reflect+ internal/runtime/math from internal/runtime/maps+ + internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ - L internal/runtime/syscall from runtime+ - W internal/saferio from debug/pe + L internal/runtime/syscall/linux from internal/runtime/cgroup+ + W internal/runtime/syscall/windows from internal/syscall/windows+ + internal/saferio from debug/pe+ internal/singleflight from net + internal/strconv from internal/poll+ internal/stringslite from embed+ internal/sync from sync+ + internal/synctest from sync internal/syscall/execenv from os+ LD internal/syscall/unix from crypto/internal/sysrand+ W internal/syscall/windows from crypto/internal/sysrand+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os + internal/trace/tracev2 from runtime+ internal/unsafeheader from internal/reflectlite+ io from archive/tar+ io/fs from archive/tar+ @@ -391,29 +545,30 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep math/big from crypto/dsa+ math/bits from compress/flate+ math/rand from github.com/mdlayher/netlink+ - math/rand/v2 from tailscale.com/derp+ + math/rand/v2 from crypto/ecdsa+ mime from golang.org/x/oauth2/internal+ mime/multipart from net/http mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ net/http/cgi from tailscale.com/cmd/tailscale/cli - net/http/httptrace from golang.org/x/net/http2+ + net/http/httptrace from net/http+ net/http/httputil from tailscale.com/client/web+ net/http/internal from net/http+ net/http/internal/ascii from net/http+ + net/http/internal/httpcommon from net/http net/http/pprof from tailscale.com/tsweb net/netip from go4.org/netipx+ - net/textproto from golang.org/x/net/http/httpguts+ + net/textproto from github.com/coder/websocket+ net/url from crypto/x509+ os from crypto/internal/sysrand+ - os/exec from github.com/coreos/go-iptables/iptables+ - os/signal from tailscale.com/cmd/tailscale/cli + os/exec from github.com/atotto/clipboard+ + os/signal from tailscale.com/cmd/tailscale/cli+ os/user from archive/tar+ path from archive/tar+ path/filepath from archive/tar+ reflect from archive/tar+ - regexp from github.com/coreos/go-iptables/iptables+ + regexp from github.com/huin/goupnp/httpu+ regexp/syntax from regexp runtime from archive/tar+ runtime/debug from tailscale.com+ @@ -423,6 +578,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep sort from compress/flate+ strconv from archive/tar+ strings from archive/tar+ + W structs from internal/syscall/windows sync from archive/tar+ sync/atomic from context+ syscall from archive/tar+ @@ -435,4 +591,4 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep unicode/utf8 from bufio+ unique from net/netip unsafe from bytes+ - weak from unique + weak from unique+ diff --git a/cmd/tailscale/deps_test.go b/cmd/tailscale/deps_test.go new file mode 100644 index 0000000000000..ea7bb15d3a895 --- /dev/null +++ b/cmd/tailscale/deps_test.go @@ -0,0 +1,22 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "testing" + + "tailscale.com/tstest/deptest" +) + +func TestOmitQRCodes(t *testing.T) { + const msg = "unexpected with ts_omit_qrcodes" + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_qrcodes", + BadDeps: map[string]string{ + "github.com/skip2/go-qrcode": msg, + }, + }.Check(t) +} diff --git a/cmd/tailscale/generate.go b/cmd/tailscale/generate.go index 5c2e9be915980..36a4fa671dddb 100644 --- a/cmd/tailscale/generate.go +++ b/cmd/tailscale/generate.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/tailscale/tailscale.go b/cmd/tailscale/tailscale.go index f6adb6c197071..57a51840832b5 100644 --- a/cmd/tailscale/tailscale.go +++ b/cmd/tailscale/tailscale.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // The tailscale command is the Tailscale command-line client. It interacts diff --git a/cmd/tailscale/tailscale_test.go b/cmd/tailscale/tailscale_test.go index dc477fb6e4357..ca064b6b7a28a 100644 --- a/cmd/tailscale/tailscale_test.go +++ b/cmd/tailscale/tailscale_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -19,7 +19,6 @@ func TestDeps(t *testing.T) { "gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756", "tailscale.com/wgengine/filter": "brings in bart, etc", "github.com/bits-and-blooms/bitset": "unneeded in CLI", - "github.com/gaissmai/bart": "unneeded in CLI", "tailscale.com/net/ipset": "unneeded in CLI", }, }.Check(t) diff --git a/cmd/tailscaled/childproc/childproc.go b/cmd/tailscaled/childproc/childproc.go index cc83a06c6ee7c..7d89b314af820 100644 --- a/cmd/tailscaled/childproc/childproc.go +++ b/cmd/tailscaled/childproc/childproc.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package childproc allows other packages to register "tailscaled be-child" diff --git a/cmd/tailscaled/debug.go b/cmd/tailscaled/debug.go index 2f469a0d189f7..360075f5b0e2b 100644 --- a/cmd/tailscaled/debug.go +++ b/cmd/tailscaled/debug.go @@ -1,7 +1,7 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -//go:build go1.19 +//go:build !ts_omit_debug package main @@ -16,17 +16,21 @@ import ( "log" "net/http" "net/http/httptrace" + "net/http/pprof" "net/url" "os" "time" "tailscale.com/derp/derphttp" + "tailscale.com/feature" + "tailscale.com/feature/buildfeatures" "tailscale.com/health" "tailscale.com/ipn" "tailscale.com/net/netmon" - "tailscale.com/net/tshttpproxy" "tailscale.com/tailcfg" + "tailscale.com/tsweb/varz" "tailscale.com/types/key" + "tailscale.com/util/clientmetric" "tailscale.com/util/eventbus" ) @@ -38,7 +42,29 @@ var debugArgs struct { portmap bool } -var debugModeFunc = debugMode // so it can be addressable +func init() { + debugModeFunc := debugMode // to be addressable + subCommands["debug"] = &debugModeFunc + + hookNewDebugMux.Set(newDebugMux) +} + +func newDebugMux() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("/debug/metrics", servePrometheusMetrics) + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + return mux +} + +func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + varz.Handler(w, r) + clientmetric.WritePrometheusExpositionFormat(w) +} func debugMode(args []string) error { fs := flag.NewFlagSet("debug", flag.ExitOnError) @@ -86,14 +112,10 @@ func runMonitor(ctx context.Context, loop bool) error { } defer mon.Close() - mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { - if !delta.Major { - log.Printf("Network monitor fired; not a major change") - return - } - log.Printf("Network monitor fired. New state:") - dump(delta.New) - }) + eventClient := b.Client("debug.runMonitor") + m := eventClient.Monitor(changeDeltaWatcher(eventClient, ctx, dump)) + defer m.Close() + if loop { log.Printf("Starting link change monitor; initial state:") } @@ -106,6 +128,27 @@ func runMonitor(ctx context.Context, loop bool) error { select {} } +func changeDeltaWatcher(ec *eventbus.Client, ctx context.Context, dump func(st *netmon.State)) func(*eventbus.Client) { + changeSub := eventbus.Subscribe[netmon.ChangeDelta](ec) + return func(ec *eventbus.Client) { + for { + select { + case <-ctx.Done(): + return + case <-ec.Done(): + return + case delta := <-changeSub.Events(): + if !delta.RebindLikelyRequired { + log.Printf("Network monitor fired; not a significant change") + return + } + log.Printf("Network monitor fired. New state:") + dump(delta.CurrentState()) + } + } + } +} + func getURL(ctx context.Context, urlStr string) error { if urlStr == "login" { urlStr = "https://login.tailscale.com" @@ -124,9 +167,14 @@ func getURL(ctx context.Context, urlStr string) error { if err != nil { return fmt.Errorf("http.NewRequestWithContext: %v", err) } - proxyURL, err := tshttpproxy.ProxyFromEnvironment(req) - if err != nil { - return fmt.Errorf("tshttpproxy.ProxyFromEnvironment: %v", err) + var proxyURL *url.URL + if buildfeatures.HasUseProxy { + if proxyFromEnv, ok := feature.HookProxyFromEnvironment.GetOk(); ok { + proxyURL, err = proxyFromEnv(req) + if err != nil { + return fmt.Errorf("tshttpproxy.ProxyFromEnvironment: %v", err) + } + } } log.Printf("proxy: %v", proxyURL) tr := &http.Transport{ @@ -135,7 +183,10 @@ func getURL(ctx context.Context, urlStr string) error { DisableKeepAlives: true, } if proxyURL != nil { - auth, err := tshttpproxy.GetAuthHeader(proxyURL) + var auth string + if f, ok := feature.HookProxyGetAuthHeader.GetOk(); ok { + auth, err = f(proxyURL) + } if err == nil && auth != "" { tr.ProxyConnectHeader.Set("Proxy-Authorization", auth) } @@ -161,7 +212,9 @@ func getURL(ctx context.Context, urlStr string) error { } func checkDerp(ctx context.Context, derpRegion string) (err error) { - ht := new(health.Tracker) + bus := eventbus.New() + defer bus.Close() + ht := health.NewTracker(bus) req, err := http.NewRequestWithContext(ctx, "GET", ipn.DefaultControlURL+"/derpmap/default", nil) if err != nil { return fmt.Errorf("create derp map request: %w", err) diff --git a/cmd/tailscaled/debug_forcereflect.go b/cmd/tailscaled/debug_forcereflect.go new file mode 100644 index 0000000000000..088010d7db29a --- /dev/null +++ b/cmd/tailscaled/debug_forcereflect.go @@ -0,0 +1,26 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build ts_debug_forcereflect + +// This file exists for benchmarking binary sizes. When the build tag is +// enabled, it forces use of part of the reflect package that makes the Go +// linker go into conservative retention mode where its deadcode pass can't +// eliminate exported method. + +package main + +import ( + "reflect" + "time" +) + +func init() { + // See Go's src/cmd/compile/internal/walk/expr.go:usemethod for + // why this is isn't a const. + name := []byte("Bar") + if time.Now().Unix()&1 == 0 { + name[0] = 'X' + } + _, _ = reflect.TypeOf(12).MethodByName(string(name)) +} diff --git a/cmd/tailscaled/depaware-min.txt b/cmd/tailscaled/depaware-min.txt new file mode 100644 index 0000000000000..33f7eb7441956 --- /dev/null +++ b/cmd/tailscaled/depaware-min.txt @@ -0,0 +1,429 @@ +tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/depaware) + + đŸ’Ŗ crypto/internal/entropy/v1.0.0 from crypto/internal/fips140/drbg + github.com/gaissmai/bart from tailscale.com/net/ipset+ + github.com/gaissmai/bart/internal/allot from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/art from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/bitset from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/lpm from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/nodes from github.com/gaissmai/bart + github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/value from github.com/gaissmai/bart+ + github.com/go-json-experiment/json from tailscale.com/drive+ + github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/golang/groupcache/lru from tailscale.com/net/dnscache + đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon + github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink + github.com/klauspost/compress from github.com/klauspost/compress/zstd + github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0 + github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd + github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ + đŸ’Ŗ github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+ + github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd + github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe + github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd + đŸ’Ŗ github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+ + đŸ’Ŗ github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ + đŸ’Ŗ github.com/mdlayher/socket from github.com/mdlayher/netlink + đŸ’Ŗ github.com/safchain/ethtool from tailscale.com/net/netkernelconf + đŸ’Ŗ github.com/tailscale/wireguard-go/conn from github.com/tailscale/wireguard-go/device+ + đŸ’Ŗ github.com/tailscale/wireguard-go/device from tailscale.com/net/tstun+ + github.com/tailscale/wireguard-go/ipc from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/ratelimiter from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/replay from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/rwcancel from github.com/tailscale/wireguard-go/device+ + github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device + đŸ’Ŗ github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+ + đŸ’Ŗ go4.org/mem from tailscale.com/control/controlbase+ + go4.org/netipx from tailscale.com/ipn/ipnlocal+ + tailscale.com from tailscale.com/version + tailscale.com/appc from tailscale.com/ipn/ipnlocal + tailscale.com/atomicfile from tailscale.com/ipn+ + tailscale.com/client/tailscale/apitype from tailscale.com/ipn/ipnauth+ + tailscale.com/cmd/tailscaled/childproc from tailscale.com/cmd/tailscaled + tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ + tailscale.com/control/controlclient from tailscale.com/cmd/tailscaled+ + tailscale.com/control/controlhttp from tailscale.com/control/ts2021 + tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp + tailscale.com/control/controlknobs from tailscale.com/control/controlclient+ + tailscale.com/control/ts2021 from tailscale.com/control/controlclient + tailscale.com/derp from tailscale.com/derp/derphttp+ + tailscale.com/derp/derpconst from tailscale.com/derp/derphttp+ + tailscale.com/derp/derphttp from tailscale.com/net/netcheck+ + tailscale.com/disco from tailscale.com/net/tstun+ + tailscale.com/drive from tailscale.com/ipn+ + tailscale.com/envknob from tailscale.com/cmd/tailscaled+ + tailscale.com/envknob/featureknob from tailscale.com/ipn/ipnlocal + tailscale.com/feature from tailscale.com/cmd/tailscaled+ + tailscale.com/feature/buildfeatures from tailscale.com/cmd/tailscaled+ + tailscale.com/feature/condlite/expvar from tailscale.com/wgengine/magicsock + tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled + tailscale.com/feature/condregister/portmapper from tailscale.com/feature/condregister + tailscale.com/feature/condregister/useproxy from tailscale.com/feature/condregister + tailscale.com/health from tailscale.com/control/controlclient+ + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/hostinfo from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn/conffile from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnext+ + tailscale.com/ipn/ipnext from tailscale.com/ipn/ipnlocal+ + tailscale.com/ipn/ipnlocal from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn/ipnlocal/netmapcache from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/ipnserver from tailscale.com/cmd/tailscaled + tailscale.com/ipn/ipnstate from tailscale.com/control/controlclient+ + tailscale.com/ipn/localapi from tailscale.com/ipn/ipnserver + tailscale.com/ipn/store from tailscale.com/cmd/tailscaled + tailscale.com/ipn/store/mem from tailscale.com/ipn/store+ + tailscale.com/kube/kubetypes from tailscale.com/envknob + tailscale.com/log/filelogger from tailscale.com/logpolicy + tailscale.com/log/sockstatlog from tailscale.com/ipn/ipnlocal + tailscale.com/logpolicy from tailscale.com/cmd/tailscaled+ + tailscale.com/logtail from tailscale.com/cmd/tailscaled+ + tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ + tailscale.com/net/bakedroots from tailscale.com/net/tlsdial + đŸ’Ŗ tailscale.com/net/batching from tailscale.com/wgengine/magicsock + tailscale.com/net/dns from tailscale.com/cmd/tailscaled+ + tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ + tailscale.com/net/dns/resolvconffile from tailscale.com/net/dns+ + tailscale.com/net/dns/resolver from tailscale.com/net/dns+ + tailscale.com/net/dnscache from tailscale.com/control/controlclient+ + tailscale.com/net/dnsfallback from tailscale.com/cmd/tailscaled+ + tailscale.com/net/flowtrack from tailscale.com/wgengine/filter + tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/netaddr from tailscale.com/ipn+ + tailscale.com/net/netcheck from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/neterror from tailscale.com/net/batching+ + tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal + tailscale.com/net/netknob from tailscale.com/logpolicy+ + tailscale.com/net/netmon from tailscale.com/cmd/tailscaled+ + tailscale.com/net/netns from tailscale.com/cmd/tailscaled+ + tailscale.com/net/netutil from tailscale.com/control/controlhttp+ + tailscale.com/net/netx from tailscale.com/control/controlclient+ + tailscale.com/net/packet from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/packet/checksum from tailscale.com/net/tstun + tailscale.com/net/ping from tailscale.com/net/netcheck+ + tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ + tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock + tailscale.com/net/sockstats from tailscale.com/control/controlclient+ + tailscale.com/net/stun from tailscale.com/net/netcheck+ + tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ + tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal + tailscale.com/net/tsaddr from tailscale.com/ipn+ + tailscale.com/net/tsdial from tailscale.com/cmd/tailscaled+ + tailscale.com/net/tstun from tailscale.com/cmd/tailscaled+ + tailscale.com/net/udprelay/endpoint from tailscale.com/wgengine/magicsock + tailscale.com/omit from tailscale.com/ipn/conffile + tailscale.com/paths from tailscale.com/cmd/tailscaled+ + tailscale.com/proxymap from tailscale.com/tsd + tailscale.com/safesocket from tailscale.com/cmd/tailscaled+ + tailscale.com/syncs from tailscale.com/cmd/tailscaled+ + tailscale.com/tailcfg from tailscale.com/client/tailscale/apitype+ + tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock + tailscale.com/tka from tailscale.com/control/controlclient+ + tailscale.com/tsconst from tailscale.com/net/netns+ + tailscale.com/tsd from tailscale.com/cmd/tailscaled+ + tailscale.com/tstime from tailscale.com/control/controlclient+ + tailscale.com/tstime/mono from tailscale.com/net/tstun+ + tailscale.com/tstime/rate from tailscale.com/wgengine/filter + tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+ + tailscale.com/types/dnstype from tailscale.com/client/tailscale/apitype+ + tailscale.com/types/empty from tailscale.com/ipn+ + tailscale.com/types/events from tailscale.com/control/controlclient+ + tailscale.com/types/flagtype from tailscale.com/cmd/tailscaled + tailscale.com/types/ipproto from tailscale.com/ipn+ + tailscale.com/types/key from tailscale.com/control/controlbase+ + tailscale.com/types/lazy from tailscale.com/hostinfo+ + tailscale.com/types/logger from tailscale.com/appc+ + tailscale.com/types/logid from tailscale.com/cmd/tailscaled+ + tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ + tailscale.com/types/netmap from tailscale.com/control/controlclient+ + tailscale.com/types/nettype from tailscale.com/net/batching+ + tailscale.com/types/opt from tailscale.com/control/controlknobs+ + tailscale.com/types/persist from tailscale.com/control/controlclient+ + tailscale.com/types/preftype from tailscale.com/ipn+ + tailscale.com/types/result from tailscale.com/util/lineiter + tailscale.com/types/structs from tailscale.com/control/controlclient+ + tailscale.com/types/tkatype from tailscale.com/control/controlclient+ + tailscale.com/types/views from tailscale.com/appc+ + tailscale.com/util/backoff from tailscale.com/control/controlclient+ + tailscale.com/util/bufiox from tailscale.com/types/key + tailscale.com/util/checkchange from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/cibuild from tailscale.com/health+ + tailscale.com/util/clientmetric from tailscale.com/appc+ + tailscale.com/util/cloudenv from tailscale.com/hostinfo+ + tailscale.com/util/cloudinfo from tailscale.com/wgengine/magicsock + tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ + tailscale.com/util/dnsname from tailscale.com/appc+ + tailscale.com/util/eventbus from tailscale.com/control/controlclient+ + tailscale.com/util/execqueue from tailscale.com/appc+ + tailscale.com/util/goroutines from tailscale.com/ipn/ipnlocal + tailscale.com/util/groupmember from tailscale.com/ipn/ipnauth + tailscale.com/util/httpm from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/lineiter from tailscale.com/hostinfo+ + tailscale.com/util/mak from tailscale.com/control/controlclient+ + tailscale.com/util/must from tailscale.com/logpolicy+ + tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + tailscale.com/util/osdiag from tailscale.com/ipn/localapi + tailscale.com/util/osshare from tailscale.com/cmd/tailscaled + tailscale.com/util/osuser from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/race from tailscale.com/net/dns/resolver + tailscale.com/util/racebuild from tailscale.com/logpolicy + tailscale.com/util/rands from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/ringlog from tailscale.com/wgengine/magicsock + tailscale.com/util/set from tailscale.com/control/controlclient+ + tailscale.com/util/singleflight from tailscale.com/control/controlclient+ + tailscale.com/util/slicesx from tailscale.com/appc+ + tailscale.com/util/syspolicy/pkey from tailscale.com/cmd/tailscaled+ + tailscale.com/util/syspolicy/policyclient from tailscale.com/cmd/tailscaled+ + tailscale.com/util/syspolicy/ptype from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/testenv from tailscale.com/control/controlclient+ + tailscale.com/util/usermetric from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/vizerror from tailscale.com/tailcfg+ + tailscale.com/util/winutil from tailscale.com/ipn/ipnauth + tailscale.com/util/zstdframe from tailscale.com/control/controlclient + tailscale.com/version from tailscale.com/cmd/tailscaled+ + tailscale.com/version/distro from tailscale.com/cmd/tailscaled+ + tailscale.com/wgengine from tailscale.com/cmd/tailscaled+ + tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ + tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ + đŸ’Ŗ tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/netlog from tailscale.com/wgengine + tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ + tailscale.com/wgengine/router from tailscale.com/cmd/tailscaled+ + tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal + đŸ’Ŗ tailscale.com/wgengine/wgint from tailscale.com/wgengine+ + tailscale.com/wgengine/wglog from tailscale.com/wgengine + golang.org/x/crypto/blake2b from golang.org/x/crypto/nacl/box + golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305 + golang.org/x/crypto/chacha20poly1305 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/curve25519 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/hkdf from tailscale.com/control/controlbase + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/nacl/box from tailscale.com/types/key + golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box + golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device + golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ + golang.org/x/exp/constraints from tailscale.com/util/set + golang.org/x/exp/maps from tailscale.com/ipn/store/mem + golang.org/x/net/bpf from github.com/mdlayher/netlink+ + golang.org/x/net/dns/dnsmessage from tailscale.com/ipn/ipnlocal+ + golang.org/x/net/http/httpguts from tailscale.com/ipn/ipnlocal + golang.org/x/net/icmp from tailscale.com/net/ping + golang.org/x/net/idna from golang.org/x/net/http/httpguts + golang.org/x/net/internal/iana from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ + golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/sync/errgroup from github.com/mdlayher/socket + golang.org/x/sys/cpu from github.com/tailscale/wireguard-go/tun+ + golang.org/x/sys/unix from github.com/jsimonetti/rtnetlink/internal/unix+ + golang.org/x/term from tailscale.com/logpolicy + golang.org/x/text/secure/bidirule from golang.org/x/net/idna + golang.org/x/text/transform from golang.org/x/text/secure/bidirule+ + golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ + golang.org/x/text/unicode/norm from golang.org/x/net/idna + golang.org/x/time/rate from tailscale.com/derp + vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ + vendor/golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + vendor/golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + vendor/golang.org/x/crypto/internal/alias from vendor/golang.org/x/crypto/chacha20+ + vendor/golang.org/x/crypto/internal/poly1305 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/net/dns/dnsmessage from net + vendor/golang.org/x/net/http/httpguts from net/http+ + vendor/golang.org/x/net/http/httpproxy from net/http + vendor/golang.org/x/net/http2/hpack from net/http+ + vendor/golang.org/x/net/idna from net/http+ + vendor/golang.org/x/sys/cpu from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/text/secure/bidirule from vendor/golang.org/x/net/idna + vendor/golang.org/x/text/transform from vendor/golang.org/x/text/secure/bidirule+ + vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ + vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna + bufio from compress/flate+ + bytes from bufio+ + cmp from encoding/json+ + compress/flate from compress/gzip+ + compress/gzip from net/http + container/list from crypto/tls+ + context from crypto/tls+ + crypto from crypto/ecdh+ + crypto/aes from crypto/tls+ + crypto/cipher from crypto/aes+ + crypto/des from crypto/tls+ + crypto/dsa from crypto/x509 + crypto/ecdh from crypto/ecdsa+ + crypto/ecdsa from crypto/tls+ + crypto/ed25519 from crypto/tls+ + crypto/elliptic from crypto/ecdsa+ + crypto/fips140 from crypto/tls/internal/fips140tls+ + crypto/hkdf from crypto/hpke+ + crypto/hmac from crypto/tls+ + crypto/hpke from crypto/tls + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ + crypto/internal/fips140 from crypto/fips140+ + crypto/internal/fips140/aes from crypto/aes+ + crypto/internal/fips140/aes/gcm from crypto/cipher+ + crypto/internal/fips140/alias from crypto/cipher+ + crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/check from crypto/fips140+ + crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/ecdh from crypto/ecdh + crypto/internal/fips140/ecdsa from crypto/ecdsa + crypto/internal/fips140/ed25519 from crypto/ed25519 + crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 + crypto/internal/fips140/edwards25519/field from crypto/ecdh+ + crypto/internal/fips140/hkdf from crypto/hkdf+ + crypto/internal/fips140/hmac from crypto/hmac+ + crypto/internal/fips140/mlkem from crypto/mlkem + crypto/internal/fips140/nistec from crypto/ecdsa+ + crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec + crypto/internal/fips140/rsa from crypto/rsa + crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ + crypto/internal/fips140/sha3 from crypto/internal/fips140/hmac+ + crypto/internal/fips140/sha512 from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ + crypto/internal/fips140/tls12 from crypto/tls + crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140cache from crypto/ecdsa+ + crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140deps/time from crypto/internal/entropy/v1.0.0 + crypto/internal/fips140hash from crypto/ecdsa+ + crypto/internal/fips140only from crypto/cipher+ + crypto/internal/impl from crypto/internal/fips140/aes+ + crypto/internal/rand from crypto/dsa+ + crypto/internal/randutil from crypto/internal/rand + crypto/internal/sysrand from crypto/internal/fips140/drbg + crypto/md5 from crypto/tls+ + crypto/mlkem from crypto/hpke+ + crypto/rand from crypto/ed25519+ + crypto/rc4 from crypto/tls + crypto/rsa from crypto/tls+ + crypto/sha1 from crypto/tls+ + crypto/sha256 from crypto/tls+ + crypto/sha3 from crypto/internal/fips140hash+ + crypto/sha512 from crypto/ecdsa+ + crypto/subtle from crypto/cipher+ + crypto/tls from net/http+ + crypto/tls/internal/fips140tls from crypto/tls + crypto/x509 from crypto/tls+ + crypto/x509/pkix from crypto/x509 + embed from tailscale.com+ + encoding from encoding/json+ + encoding/asn1 from crypto/x509+ + encoding/base32 from github.com/go-json-experiment/json + encoding/base64 from encoding/json+ + encoding/binary from compress/gzip+ + encoding/hex from crypto/x509+ + encoding/json from github.com/gaissmai/bart+ + encoding/pem from crypto/tls+ + errors from bufio+ + flag from tailscale.com/cmd/tailscaled+ + fmt from compress/flate+ + hash from crypto+ + hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic + hash/maphash from go4.org/mem + html from tailscale.com/ipn/ipnlocal+ + internal/abi from hash/maphash+ + internal/asan from internal/runtime/maps+ + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/cipher+ + internal/chacha8rand from math/rand/v2+ + internal/coverage/rtcov from runtime + internal/cpu from crypto/internal/fips140deps/cpu+ + internal/filepathlite from os+ + internal/fmtsort from fmt + internal/goarch from crypto/internal/fips140deps/cpu+ + internal/godebug from crypto/internal/fips140deps/godebug+ + internal/godebugs from internal/godebug+ + internal/goexperiment from runtime + internal/goos from crypto/x509+ + internal/msan from internal/runtime/maps+ + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profilerecord from runtime + internal/race from internal/runtime/maps+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/cgroup from runtime + internal/runtime/exithook from runtime + internal/runtime/gc from runtime+ + internal/runtime/gc/scan from runtime + internal/runtime/maps from reflect+ + internal/runtime/math from internal/runtime/maps+ + internal/runtime/pprof/label from runtime + internal/runtime/sys from crypto/subtle+ + internal/runtime/syscall/linux from internal/runtime/cgroup+ + internal/saferio from encoding/asn1 + internal/singleflight from net + internal/strconv from internal/poll+ + internal/stringslite from embed+ + internal/sync from sync+ + internal/synctest from sync + internal/syscall/execenv from os+ + internal/syscall/unix from crypto/internal/sysrand+ + internal/testlog from os + internal/trace/tracev2 from runtime + internal/unsafeheader from internal/reflectlite+ + io from bufio+ + io/fs from crypto/x509+ + iter from bytes+ + log from github.com/klauspost/compress/zstd+ + log/internal from log + maps from crypto/x509+ + math from compress/flate+ + math/big from crypto/dsa+ + math/bits from bytes+ + math/rand from github.com/mdlayher/netlink+ + math/rand/v2 from crypto/ecdsa+ + mime from mime/multipart+ + mime/multipart from net/http + mime/quotedprintable from mime/multipart + net from crypto/tls+ + net/http from tailscale.com/cmd/tailscaled+ + net/http/httptrace from net/http+ + net/http/internal from net/http + net/http/internal/ascii from net/http + net/http/internal/httpcommon from net/http + net/netip from crypto/x509+ + net/textproto from golang.org/x/net/http/httpguts+ + net/url from crypto/x509+ + os from crypto/internal/sysrand+ + os/exec from tailscale.com/hostinfo+ + os/signal from tailscale.com/cmd/tailscaled + os/user from tailscale.com/ipn/ipnauth+ + path from io/fs+ + path/filepath from crypto/x509+ + reflect from encoding/asn1+ + runtime from crypto/internal/fips140+ + runtime/debug from github.com/klauspost/compress/zstd+ + slices from crypto/tls+ + sort from compress/flate+ + strconv from compress/flate+ + strings from bufio+ + sync from compress/flate+ + sync/atomic from context+ + syscall from crypto/internal/sysrand+ + time from compress/gzip+ + unicode from bytes+ + unicode/utf16 from crypto/x509+ + unicode/utf8 from bufio+ + unique from net/netip + unsafe from bytes+ + weak from crypto/internal/fips140cache+ diff --git a/cmd/tailscaled/depaware-minbox.txt b/cmd/tailscaled/depaware-minbox.txt new file mode 100644 index 0000000000000..e0b07a2c257bd --- /dev/null +++ b/cmd/tailscaled/depaware-minbox.txt @@ -0,0 +1,452 @@ +tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/depaware) + + đŸ’Ŗ crypto/internal/entropy/v1.0.0 from crypto/internal/fips140/drbg + github.com/gaissmai/bart from tailscale.com/net/ipset+ + github.com/gaissmai/bart/internal/allot from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/art from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/bitset from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/lpm from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/nodes from github.com/gaissmai/bart + github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/value from github.com/gaissmai/bart+ + github.com/go-json-experiment/json from tailscale.com/drive+ + github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + github.com/golang/groupcache/lru from tailscale.com/net/dnscache + đŸ’Ŗ github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon + github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink + github.com/kballard/go-shellquote from tailscale.com/cmd/tailscale/cli + github.com/klauspost/compress from github.com/klauspost/compress/zstd + github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0 + github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd + github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ + đŸ’Ŗ github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+ + github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd + github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe + github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd + github.com/mattn/go-isatty from tailscale.com/util/prompt + đŸ’Ŗ github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+ + đŸ’Ŗ github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ + đŸ’Ŗ github.com/mdlayher/socket from github.com/mdlayher/netlink + github.com/peterbourgon/ff/v3 from github.com/peterbourgon/ff/v3/ffcli+ + github.com/peterbourgon/ff/v3/ffcli from tailscale.com/cmd/tailscale/cli+ + github.com/peterbourgon/ff/v3/internal from github.com/peterbourgon/ff/v3 + đŸ’Ŗ github.com/safchain/ethtool from tailscale.com/net/netkernelconf + đŸ’Ŗ github.com/tailscale/wireguard-go/conn from github.com/tailscale/wireguard-go/device+ + đŸ’Ŗ github.com/tailscale/wireguard-go/device from tailscale.com/net/tstun+ + github.com/tailscale/wireguard-go/ipc from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/ratelimiter from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/replay from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/rwcancel from github.com/tailscale/wireguard-go/device+ + github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device + đŸ’Ŗ github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+ + đŸ’Ŗ go4.org/mem from tailscale.com/control/controlbase+ + go4.org/netipx from tailscale.com/ipn/ipnlocal+ + tailscale.com from tailscale.com/version + tailscale.com/appc from tailscale.com/ipn/ipnlocal + tailscale.com/atomicfile from tailscale.com/ipn+ + tailscale.com/client/local from tailscale.com/client/tailscale+ + tailscale.com/client/tailscale from tailscale.com/internal/client/tailscale + tailscale.com/client/tailscale/apitype from tailscale.com/ipn/ipnauth+ + tailscale.com/cmd/tailscale/cli from tailscale.com/cmd/tailscaled + tailscale.com/cmd/tailscale/cli/ffcomplete from tailscale.com/cmd/tailscale/cli + tailscale.com/cmd/tailscale/cli/ffcomplete/internal from tailscale.com/cmd/tailscale/cli/ffcomplete + tailscale.com/cmd/tailscale/cli/jsonoutput from tailscale.com/cmd/tailscale/cli + tailscale.com/cmd/tailscaled/childproc from tailscale.com/cmd/tailscaled + tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ + tailscale.com/control/controlclient from tailscale.com/cmd/tailscaled+ + tailscale.com/control/controlhttp from tailscale.com/control/ts2021 + tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp + tailscale.com/control/controlknobs from tailscale.com/control/controlclient+ + tailscale.com/control/ts2021 from tailscale.com/control/controlclient+ + tailscale.com/derp from tailscale.com/derp/derphttp+ + tailscale.com/derp/derpconst from tailscale.com/derp/derphttp+ + tailscale.com/derp/derphttp from tailscale.com/net/netcheck+ + tailscale.com/disco from tailscale.com/net/tstun+ + tailscale.com/drive from tailscale.com/ipn+ + tailscale.com/envknob from tailscale.com/cmd/tailscaled+ + tailscale.com/envknob/featureknob from tailscale.com/ipn/ipnlocal + tailscale.com/feature from tailscale.com/cmd/tailscaled+ + tailscale.com/feature/buildfeatures from tailscale.com/ipn/ipnlocal+ + tailscale.com/feature/condlite/expvar from tailscale.com/wgengine/magicsock + tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled + tailscale.com/feature/condregister/awsparamstore from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/identityfederation from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/oauthkey from tailscale.com/cmd/tailscale/cli + tailscale.com/feature/condregister/portmapper from tailscale.com/feature/condregister+ + tailscale.com/feature/condregister/useproxy from tailscale.com/cmd/tailscale/cli+ + tailscale.com/health from tailscale.com/control/controlclient+ + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ + tailscale.com/hostinfo from tailscale.com/cmd/tailscaled+ + tailscale.com/internal/client/tailscale from tailscale.com/cmd/tailscale/cli + tailscale.com/ipn from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn/conffile from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnext+ + tailscale.com/ipn/ipnext from tailscale.com/ipn/ipnlocal+ + tailscale.com/ipn/ipnlocal from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn/ipnlocal/netmapcache from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/ipnserver from tailscale.com/cmd/tailscaled + tailscale.com/ipn/ipnstate from tailscale.com/control/controlclient+ + tailscale.com/ipn/localapi from tailscale.com/ipn/ipnserver + tailscale.com/ipn/store from tailscale.com/cmd/tailscaled + tailscale.com/ipn/store/mem from tailscale.com/ipn/store+ + tailscale.com/kube/kubetypes from tailscale.com/envknob + tailscale.com/licenses from tailscale.com/cmd/tailscale/cli + tailscale.com/log/filelogger from tailscale.com/logpolicy + tailscale.com/log/sockstatlog from tailscale.com/ipn/ipnlocal + tailscale.com/logpolicy from tailscale.com/cmd/tailscaled+ + tailscale.com/logtail from tailscale.com/cmd/tailscaled+ + tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ + tailscale.com/net/ace from tailscale.com/cmd/tailscale/cli + tailscale.com/net/bakedroots from tailscale.com/net/tlsdial + đŸ’Ŗ tailscale.com/net/batching from tailscale.com/wgengine/magicsock + tailscale.com/net/dns from tailscale.com/cmd/tailscaled+ + tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ + tailscale.com/net/dns/resolvconffile from tailscale.com/net/dns+ + tailscale.com/net/dns/resolver from tailscale.com/net/dns+ + tailscale.com/net/dnscache from tailscale.com/control/controlclient+ + tailscale.com/net/dnsfallback from tailscale.com/cmd/tailscaled+ + tailscale.com/net/flowtrack from tailscale.com/wgengine/filter + tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/netaddr from tailscale.com/ipn+ + tailscale.com/net/netcheck from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/neterror from tailscale.com/net/batching+ + tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal + tailscale.com/net/netknob from tailscale.com/logpolicy+ + tailscale.com/net/netmon from tailscale.com/cmd/tailscaled+ + tailscale.com/net/netns from tailscale.com/cmd/tailscaled+ + tailscale.com/net/netutil from tailscale.com/client/local+ + tailscale.com/net/netx from tailscale.com/control/controlclient+ + tailscale.com/net/packet from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/packet/checksum from tailscale.com/net/tstun + tailscale.com/net/ping from tailscale.com/net/netcheck+ + tailscale.com/net/portmapper/portmappertype from tailscale.com/net/netcheck+ + tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock + tailscale.com/net/sockstats from tailscale.com/control/controlclient+ + tailscale.com/net/stun from tailscale.com/net/netcheck+ + tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ + tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal + tailscale.com/net/tsaddr from tailscale.com/ipn+ + tailscale.com/net/tsdial from tailscale.com/cmd/tailscaled+ + tailscale.com/net/tstun from tailscale.com/cmd/tailscaled+ + tailscale.com/net/udprelay/endpoint from tailscale.com/wgengine/magicsock + tailscale.com/net/udprelay/status from tailscale.com/client/local + tailscale.com/omit from tailscale.com/ipn/conffile + tailscale.com/paths from tailscale.com/cmd/tailscaled+ + tailscale.com/proxymap from tailscale.com/tsd + tailscale.com/safesocket from tailscale.com/cmd/tailscaled+ + tailscale.com/syncs from tailscale.com/cmd/tailscaled+ + tailscale.com/tailcfg from tailscale.com/client/tailscale/apitype+ + tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock + tailscale.com/tempfork/spf13/cobra from tailscale.com/cmd/tailscale/cli/ffcomplete+ + tailscale.com/tka from tailscale.com/control/controlclient+ + tailscale.com/tsconst from tailscale.com/net/netns+ + tailscale.com/tsd from tailscale.com/cmd/tailscaled+ + tailscale.com/tstime from tailscale.com/control/controlclient+ + tailscale.com/tstime/mono from tailscale.com/net/tstun+ + tailscale.com/tstime/rate from tailscale.com/wgengine/filter + tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+ + tailscale.com/types/dnstype from tailscale.com/client/tailscale/apitype+ + tailscale.com/types/empty from tailscale.com/ipn+ + tailscale.com/types/events from tailscale.com/control/controlclient+ + tailscale.com/types/flagtype from tailscale.com/cmd/tailscaled + tailscale.com/types/ipproto from tailscale.com/ipn+ + tailscale.com/types/key from tailscale.com/client/local+ + tailscale.com/types/lazy from tailscale.com/hostinfo+ + tailscale.com/types/logger from tailscale.com/appc+ + tailscale.com/types/logid from tailscale.com/cmd/tailscaled+ + tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ + tailscale.com/types/netmap from tailscale.com/control/controlclient+ + tailscale.com/types/nettype from tailscale.com/net/batching+ + tailscale.com/types/opt from tailscale.com/control/controlknobs+ + tailscale.com/types/persist from tailscale.com/control/controlclient+ + tailscale.com/types/preftype from tailscale.com/ipn+ + tailscale.com/types/result from tailscale.com/util/lineiter + tailscale.com/types/structs from tailscale.com/control/controlclient+ + tailscale.com/types/tkatype from tailscale.com/control/controlclient+ + tailscale.com/types/views from tailscale.com/appc+ + tailscale.com/util/backoff from tailscale.com/control/controlclient+ + tailscale.com/util/bufiox from tailscale.com/types/key + tailscale.com/util/checkchange from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/cibuild from tailscale.com/health+ + tailscale.com/util/clientmetric from tailscale.com/appc+ + tailscale.com/util/cloudenv from tailscale.com/hostinfo+ + tailscale.com/util/cloudinfo from tailscale.com/wgengine/magicsock + tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ + tailscale.com/util/dnsname from tailscale.com/appc+ + tailscale.com/util/eventbus from tailscale.com/client/local+ + tailscale.com/util/execqueue from tailscale.com/appc+ + tailscale.com/util/goroutines from tailscale.com/ipn/ipnlocal + tailscale.com/util/groupmember from tailscale.com/ipn/ipnauth + tailscale.com/util/httpm from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/lineiter from tailscale.com/hostinfo+ + tailscale.com/util/mak from tailscale.com/control/controlclient+ + tailscale.com/util/must from tailscale.com/logpolicy+ + tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + tailscale.com/util/osdiag from tailscale.com/ipn/localapi + tailscale.com/util/osshare from tailscale.com/cmd/tailscaled + tailscale.com/util/osuser from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/prompt from tailscale.com/cmd/tailscale/cli + tailscale.com/util/qrcodes from tailscale.com/cmd/tailscale/cli + tailscale.com/util/race from tailscale.com/net/dns/resolver + tailscale.com/util/racebuild from tailscale.com/logpolicy + tailscale.com/util/rands from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/ringlog from tailscale.com/wgengine/magicsock + tailscale.com/util/set from tailscale.com/control/controlclient+ + tailscale.com/util/singleflight from tailscale.com/control/controlclient+ + tailscale.com/util/slicesx from tailscale.com/appc+ + tailscale.com/util/syspolicy/pkey from tailscale.com/cmd/tailscaled+ + tailscale.com/util/syspolicy/policyclient from tailscale.com/cmd/tailscaled+ + tailscale.com/util/syspolicy/ptype from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/testenv from tailscale.com/control/controlclient+ + tailscale.com/util/usermetric from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/vizerror from tailscale.com/tailcfg+ + tailscale.com/util/winutil from tailscale.com/ipn/ipnauth + tailscale.com/util/zstdframe from tailscale.com/control/controlclient + tailscale.com/version from tailscale.com/cmd/tailscaled+ + tailscale.com/version/distro from tailscale.com/cmd/tailscaled+ + tailscale.com/wgengine from tailscale.com/cmd/tailscaled+ + tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ + tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ + đŸ’Ŗ tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/netlog from tailscale.com/wgengine + tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ + tailscale.com/wgengine/router from tailscale.com/cmd/tailscaled+ + tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal + đŸ’Ŗ tailscale.com/wgengine/wgint from tailscale.com/wgengine+ + tailscale.com/wgengine/wglog from tailscale.com/wgengine + golang.org/x/crypto/blake2b from golang.org/x/crypto/nacl/box + golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305 + golang.org/x/crypto/chacha20poly1305 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/curve25519 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/hkdf from tailscale.com/control/controlbase + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/nacl/box from tailscale.com/types/key + golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box + golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device + golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ + golang.org/x/exp/constraints from tailscale.com/util/set + golang.org/x/exp/maps from tailscale.com/ipn/store/mem + golang.org/x/net/bpf from github.com/mdlayher/netlink+ + golang.org/x/net/dns/dnsmessage from tailscale.com/cmd/tailscale/cli+ + golang.org/x/net/http/httpguts from tailscale.com/ipn/ipnlocal + golang.org/x/net/icmp from tailscale.com/net/ping + golang.org/x/net/idna from golang.org/x/net/http/httpguts+ + golang.org/x/net/internal/iana from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ + golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/sync/errgroup from github.com/mdlayher/socket + golang.org/x/sys/cpu from github.com/tailscale/wireguard-go/tun+ + golang.org/x/sys/unix from github.com/jsimonetti/rtnetlink/internal/unix+ + golang.org/x/term from tailscale.com/logpolicy + golang.org/x/text/secure/bidirule from golang.org/x/net/idna + golang.org/x/text/transform from golang.org/x/text/secure/bidirule+ + golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ + golang.org/x/text/unicode/norm from golang.org/x/net/idna + golang.org/x/time/rate from tailscale.com/derp + vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ + vendor/golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + vendor/golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + vendor/golang.org/x/crypto/internal/alias from vendor/golang.org/x/crypto/chacha20+ + vendor/golang.org/x/crypto/internal/poly1305 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/net/dns/dnsmessage from net + vendor/golang.org/x/net/http/httpguts from net/http+ + vendor/golang.org/x/net/http/httpproxy from net/http + vendor/golang.org/x/net/http2/hpack from net/http+ + vendor/golang.org/x/net/idna from net/http+ + vendor/golang.org/x/sys/cpu from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/text/secure/bidirule from vendor/golang.org/x/net/idna + vendor/golang.org/x/text/transform from vendor/golang.org/x/text/secure/bidirule+ + vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ + vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna + bufio from compress/flate+ + bytes from bufio+ + cmp from encoding/json+ + compress/flate from compress/gzip+ + compress/gzip from net/http+ + container/list from crypto/tls+ + context from crypto/tls+ + crypto from crypto/ecdh+ + crypto/aes from crypto/tls+ + crypto/cipher from crypto/aes+ + crypto/des from crypto/tls+ + crypto/dsa from crypto/x509 + crypto/ecdh from crypto/ecdsa+ + crypto/ecdsa from crypto/tls+ + crypto/ed25519 from crypto/tls+ + crypto/elliptic from crypto/ecdsa+ + crypto/fips140 from crypto/tls/internal/fips140tls+ + crypto/hkdf from crypto/hpke+ + crypto/hmac from crypto/tls+ + crypto/hpke from crypto/tls + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ + crypto/internal/fips140 from crypto/fips140+ + crypto/internal/fips140/aes from crypto/aes+ + crypto/internal/fips140/aes/gcm from crypto/cipher+ + crypto/internal/fips140/alias from crypto/cipher+ + crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/check from crypto/fips140+ + crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/ecdh from crypto/ecdh + crypto/internal/fips140/ecdsa from crypto/ecdsa + crypto/internal/fips140/ed25519 from crypto/ed25519 + crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 + crypto/internal/fips140/edwards25519/field from crypto/ecdh+ + crypto/internal/fips140/hkdf from crypto/hkdf+ + crypto/internal/fips140/hmac from crypto/hmac+ + crypto/internal/fips140/mlkem from crypto/mlkem + crypto/internal/fips140/nistec from crypto/ecdsa+ + crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec + crypto/internal/fips140/rsa from crypto/rsa + crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ + crypto/internal/fips140/sha3 from crypto/internal/fips140/hmac+ + crypto/internal/fips140/sha512 from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ + crypto/internal/fips140/tls12 from crypto/tls + crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140cache from crypto/ecdsa+ + crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140deps/time from crypto/internal/entropy/v1.0.0 + crypto/internal/fips140hash from crypto/ecdsa+ + crypto/internal/fips140only from crypto/cipher+ + crypto/internal/impl from crypto/internal/fips140/aes+ + crypto/internal/rand from crypto/dsa+ + crypto/internal/randutil from crypto/internal/rand + crypto/internal/sysrand from crypto/internal/fips140/drbg + crypto/md5 from crypto/tls+ + crypto/mlkem from crypto/hpke+ + crypto/rand from crypto/ed25519+ + crypto/rc4 from crypto/tls + crypto/rsa from crypto/tls+ + crypto/sha1 from crypto/tls+ + crypto/sha256 from crypto/tls+ + crypto/sha3 from crypto/internal/fips140hash+ + crypto/sha512 from crypto/ecdsa+ + crypto/subtle from crypto/cipher+ + crypto/tls from net/http+ + crypto/tls/internal/fips140tls from crypto/tls + crypto/x509 from crypto/tls+ + crypto/x509/pkix from crypto/x509 + embed from tailscale.com+ + encoding from encoding/json+ + encoding/asn1 from crypto/x509+ + encoding/base32 from github.com/go-json-experiment/json + encoding/base64 from encoding/json+ + encoding/binary from compress/gzip+ + encoding/hex from crypto/x509+ + encoding/json from github.com/gaissmai/bart+ + encoding/pem from crypto/tls+ + errors from bufio+ + flag from tailscale.com/cmd/tailscaled+ + fmt from compress/flate+ + hash from crypto+ + hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic + hash/maphash from go4.org/mem + html from tailscale.com/ipn/ipnlocal+ + internal/abi from hash/maphash+ + internal/asan from internal/runtime/maps+ + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/cipher+ + internal/chacha8rand from math/rand/v2+ + internal/coverage/rtcov from runtime + internal/cpu from crypto/internal/fips140deps/cpu+ + internal/filepathlite from os+ + internal/fmtsort from fmt + internal/goarch from crypto/internal/fips140deps/cpu+ + internal/godebug from crypto/internal/fips140deps/godebug+ + internal/godebugs from internal/godebug+ + internal/goexperiment from runtime + internal/goos from crypto/x509+ + internal/msan from internal/runtime/maps+ + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profilerecord from runtime + internal/race from internal/runtime/maps+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/cgroup from runtime + internal/runtime/exithook from runtime + internal/runtime/gc from runtime+ + internal/runtime/gc/scan from runtime + internal/runtime/maps from reflect+ + internal/runtime/math from internal/runtime/maps+ + internal/runtime/pprof/label from runtime + internal/runtime/sys from crypto/subtle+ + internal/runtime/syscall/linux from internal/runtime/cgroup+ + internal/saferio from encoding/asn1 + internal/singleflight from net + internal/strconv from internal/poll+ + internal/stringslite from embed+ + internal/sync from sync+ + internal/synctest from sync + internal/syscall/execenv from os+ + internal/syscall/unix from crypto/internal/sysrand+ + internal/testlog from os + internal/trace/tracev2 from runtime + internal/unsafeheader from internal/reflectlite+ + io from bufio+ + io/fs from crypto/x509+ + iter from bytes+ + log from github.com/klauspost/compress/zstd+ + log/internal from log + maps from crypto/x509+ + math from compress/flate+ + math/big from crypto/dsa+ + math/bits from bytes+ + math/rand from github.com/mdlayher/netlink+ + math/rand/v2 from crypto/ecdsa+ + mime from mime/multipart+ + mime/multipart from net/http + mime/quotedprintable from mime/multipart + net from crypto/tls+ + net/http from net/http/httputil+ + net/http/httptrace from net/http+ + net/http/httputil from tailscale.com/cmd/tailscale/cli + net/http/internal from net/http+ + net/http/internal/ascii from net/http+ + net/http/internal/httpcommon from net/http + net/netip from crypto/x509+ + net/textproto from golang.org/x/net/http/httpguts+ + net/url from crypto/x509+ + os from crypto/internal/sysrand+ + os/exec from tailscale.com/hostinfo+ + os/signal from tailscale.com/cmd/tailscaled+ + os/user from tailscale.com/ipn/ipnauth+ + path from io/fs+ + path/filepath from crypto/x509+ + reflect from encoding/asn1+ + runtime from crypto/internal/fips140+ + runtime/debug from github.com/klauspost/compress/zstd+ + slices from crypto/tls+ + sort from compress/flate+ + strconv from compress/flate+ + strings from bufio+ + sync from compress/flate+ + sync/atomic from context+ + syscall from crypto/internal/sysrand+ + text/tabwriter from github.com/peterbourgon/ff/v3/ffcli+ + time from compress/gzip+ + unicode from bytes+ + unicode/utf16 from crypto/x509+ + unicode/utf8 from bufio+ + unique from net/netip + unsafe from bytes+ + weak from crypto/internal/fips140cache+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 7c4885a4be4c4..0915068a55765 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -1,11 +1,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/depaware) + đŸ’Ŗ crypto/internal/entropy/v1.0.0 from crypto/internal/fips140/drbg filippo.io/edwards25519 from github.com/hdevalence/ed25519consensus filippo.io/edwards25519/field from filippo.io/edwards25519 W đŸ’Ŗ github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+ W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate W đŸ’Ŗ github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy - LD github.com/anmitsu/go-shlex from tailscale.com/tempfork/gliderlabs/ssh + LD github.com/anmitsu/go-shlex from github.com/tailscale/gliderssh L github.com/aws/aws-sdk-go-v2/aws from github.com/aws/aws-sdk-go-v2/aws/defaults+ L github.com/aws/aws-sdk-go-v2/aws/arn from tailscale.com/ipn/store/awsstore L github.com/aws/aws-sdk-go-v2/aws/defaults from github.com/aws/aws-sdk-go-v2/service/ssm+ @@ -19,10 +20,12 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/service/internal/presigned-url+ L github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ L github.com/aws/aws-sdk-go-v2/config from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/config/internal/ini from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds + L github.com/aws/aws-sdk-go-v2/credentials/logincreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config L github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config @@ -35,17 +38,21 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ L github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/ssm+ L github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+ - L github.com/aws/aws-sdk-go-v2/internal/ini from github.com/aws/aws-sdk-go-v2/config - L github.com/aws/aws-sdk-go-v2/internal/middleware from github.com/aws/aws-sdk-go-v2/service/sso+ L github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ L github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ L github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds L github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ - L github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 + L github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4+ L github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws L github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry + L github.com/aws/aws-sdk-go-v2/internal/v4a from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/internal/v4a/internal/crypto from github.com/aws/aws-sdk-go-v2/internal/v4a + L github.com/aws/aws-sdk-go-v2/internal/v4a/internal/v4 from github.com/aws/aws-sdk-go-v2/internal/v4a L github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts L github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/signin from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/signin/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/signin + L github.com/aws/aws-sdk-go-v2/service/signin/types from github.com/aws/aws-sdk-go-v2/credentials/logincreds+ L github.com/aws/aws-sdk-go-v2/service/ssm from tailscale.com/ipn/store/awsstore L github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm L github.com/aws/aws-sdk-go-v2/service/ssm/types from github.com/aws/aws-sdk-go-v2/service/ssm+ @@ -68,6 +75,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssm+ L github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts L github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/endpoints/private/rulesfn from github.com/aws/aws-sdk-go-v2/service/sts+ L github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer L github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ L github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ @@ -84,8 +92,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/coder/websocket from tailscale.com/util/eventbus github.com/coder/websocket/internal/errd from github.com/coder/websocket github.com/coder/websocket/internal/util from github.com/coder/websocket - github.com/coder/websocket/internal/xsync from github.com/coder/websocket L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw + github.com/creachadair/msync/trigger from tailscale.com/logtail LD đŸ’Ŗ github.com/creack/pty from tailscale.com/ssh/tailssh W đŸ’Ŗ github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/com+ W đŸ’Ŗ github.com/dblohm7/wingoes/com from tailscale.com/cmd/tailscaled+ @@ -96,19 +104,25 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de đŸ’Ŗ github.com/djherbis/times from tailscale.com/drive/driveimpl github.com/fxamacker/cbor/v2 from tailscale.com/tka github.com/gaissmai/bart from tailscale.com/net/tstun+ + github.com/gaissmai/bart/internal/allot from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/art from github.com/gaissmai/bart+ github.com/gaissmai/bart/internal/bitset from github.com/gaissmai/bart+ - github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart + github.com/gaissmai/bart/internal/lpm from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/nodes from github.com/gaissmai/bart + github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart/internal/nodes + github.com/gaissmai/bart/internal/value from github.com/gaissmai/bart+ github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json/internal/jsonflags+ github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json/internal/jsonopts+ github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json/jsontext+ github.com/go-json-experiment/json/jsontext from tailscale.com/logtail+ + github.com/go-json-experiment/json/v1 from tailscale.com/feature/routecheck+ W đŸ’Ŗ github.com/go-ole/go-ole from github.com/go-ole/go-ole/oleutil+ W đŸ’Ŗ github.com/go-ole/go-ole/oleutil from tailscale.com/wgengine/winnet L đŸ’Ŗ github.com/godbus/dbus/v5 from tailscale.com/net/dns+ github.com/golang/groupcache/lru from tailscale.com/net/dnscache - github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+ + github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/transport/tcp github.com/google/go-tpm/legacy/tpm2 from github.com/google/go-tpm/tpm2/transport+ github.com/google/go-tpm/tpm2 from tailscale.com/feature/tpm github.com/google/go-tpm/tpm2/transport from github.com/google/go-tpm/tpm2/transport/linuxtpm+ @@ -122,9 +136,15 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/google/nftables/expr from github.com/google/nftables+ L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ - DW github.com/google/uuid from tailscale.com/clientupdate+ + W github.com/google/uuid from tailscale.com/clientupdate github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ - L đŸ’Ŗ github.com/illarion/gonotify/v3 from tailscale.com/net/dns + github.com/huin/goupnp from github.com/huin/goupnp/dcps/internetgateway2+ + github.com/huin/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper + github.com/huin/goupnp/httpu from github.com/huin/goupnp+ + github.com/huin/goupnp/scpd from github.com/huin/goupnp + github.com/huin/goupnp/soap from github.com/huin/goupnp+ + github.com/huin/goupnp/ssdp from github.com/huin/goupnp + L đŸ’Ŗ github.com/illarion/gonotify/v3 from tailscale.com/feature/linuxdnsfight L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/feature/tap L github.com/insomniacslk/dhcp/iana from github.com/insomniacslk/dhcp/dhcpv4 @@ -137,45 +157,40 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de github.com/klauspost/compress from github.com/klauspost/compress/zstd github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0 github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd - github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ + github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/zstd+ + đŸ’Ŗ github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+ github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd github.com/kortschak/wol from tailscale.com/feature/wakeonlan LD github.com/kr/fs from github.com/pkg/sftp - L github.com/mdlayher/genetlink from tailscale.com/net/tstun + L github.com/mdlayher/genetlink from tailscale.com/feature/linkspeed L đŸ’Ŗ github.com/mdlayher/netlink from github.com/google/nftables+ L đŸ’Ŗ github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ L github.com/mdlayher/netlink/nltest from github.com/google/nftables - L github.com/mdlayher/sdnotify from tailscale.com/util/systemd + L github.com/mdlayher/sdnotify from tailscale.com/feature/sdnotify L đŸ’Ŗ github.com/mdlayher/socket from github.com/mdlayher/netlink+ - github.com/miekg/dns from tailscale.com/net/dns/recursive đŸ’Ŗ github.com/mitchellh/go-ps from tailscale.com/safesocket L github.com/pierrec/lz4/v4 from github.com/u-root/uio/uio L github.com/pierrec/lz4/v4/internal/lz4block from github.com/pierrec/lz4/v4+ L github.com/pierrec/lz4/v4/internal/lz4errors from github.com/pierrec/lz4/v4+ L github.com/pierrec/lz4/v4/internal/lz4stream from github.com/pierrec/lz4/v4 L github.com/pierrec/lz4/v4/internal/xxh32 from github.com/pierrec/lz4/v4/internal/lz4stream + github.com/pires/go-proxyproto from tailscale.com/ipn/ipnlocal LD github.com/pkg/sftp from tailscale.com/ssh/tailssh LD github.com/pkg/sftp/internal/encoding/ssh/filexfer from github.com/pkg/sftp - D github.com/prometheus-community/pro-bing from tailscale.com/wgengine/netstack L đŸ’Ŗ github.com/safchain/ethtool from tailscale.com/net/netkernelconf+ - W đŸ’Ŗ github.com/tailscale/certstore from tailscale.com/control/controlclient + DW đŸ’Ŗ github.com/tailscale/certstore from tailscale.com/control/controlclient + LD github.com/tailscale/gliderssh from tailscale.com/ssh/tailssh W đŸ’Ŗ github.com/tailscale/go-winio from tailscale.com/safesocket W đŸ’Ŗ github.com/tailscale/go-winio/internal/fs from github.com/tailscale/go-winio W đŸ’Ŗ github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ - github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ - github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper - github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ - github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp - github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ - github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp github.com/tailscale/hujson from tailscale.com/ipn/conffile L đŸ’Ŗ github.com/tailscale/netlink from tailscale.com/net/routetable+ L đŸ’Ŗ github.com/tailscale/netlink/nl from github.com/tailscale/netlink - github.com/tailscale/peercred from tailscale.com/ipn/ipnauth + LD github.com/tailscale/peercred from tailscale.com/ipn/ipnauth github.com/tailscale/web-client-prebuilt from tailscale.com/client/web W đŸ’Ŗ github.com/tailscale/wf from tailscale.com/wf đŸ’Ŗ github.com/tailscale/wireguard-go/conn from github.com/tailscale/wireguard-go/device+ @@ -212,7 +227,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de đŸ’Ŗ gvisor.dev/gvisor/pkg/state from gvisor.dev/gvisor/pkg/atomicbitops+ gvisor.dev/gvisor/pkg/state/wire from gvisor.dev/gvisor/pkg/state đŸ’Ŗ gvisor.dev/gvisor/pkg/sync from gvisor.dev/gvisor/pkg/atomicbitops+ - đŸ’Ŗ gvisor.dev/gvisor/pkg/sync/locking from gvisor.dev/gvisor/pkg/tcpip/stack + đŸ’Ŗ gvisor.dev/gvisor/pkg/sync/locking from gvisor.dev/gvisor/pkg/tcpip/stack+ gvisor.dev/gvisor/pkg/tcpip from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ gvisor.dev/gvisor/pkg/tcpip/adapters/gonet from tailscale.com/wgengine/netstack đŸ’Ŗ gvisor.dev/gvisor/pkg/tcpip/checksum from gvisor.dev/gvisor/pkg/buffer+ @@ -241,30 +256,31 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de gvisor.dev/gvisor/pkg/tcpip/transport/udp from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ gvisor.dev/gvisor/pkg/waiter from gvisor.dev/gvisor/pkg/context+ tailscale.com from tailscale.com/version - tailscale.com/appc from tailscale.com/ipn/ipnlocal + tailscale.com/appc from tailscale.com/ipn/ipnlocal+ đŸ’Ŗ tailscale.com/atomicfile from tailscale.com/ipn+ LD tailscale.com/chirp from tailscale.com/cmd/tailscaled - tailscale.com/client/local from tailscale.com/client/tailscale+ - tailscale.com/client/tailscale from tailscale.com/derp - tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+ + tailscale.com/client/local from tailscale.com/client/web+ + tailscale.com/client/tailscale/apitype from tailscale.com/client/local+ tailscale.com/client/web from tailscale.com/ipn/ipnlocal - tailscale.com/clientupdate from tailscale.com/client/web+ + tailscale.com/clientupdate from tailscale.com/feature/clientupdate LW tailscale.com/clientupdate/distsign from tailscale.com/clientupdate + tailscale.com/cmd/tailscale/cli/jsonoutput from tailscale.com/feature/tailnetlock tailscale.com/cmd/tailscaled/childproc from tailscale.com/cmd/tailscaled+ tailscale.com/cmd/tailscaled/tailscaledhooks from tailscale.com/cmd/tailscaled+ tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ tailscale.com/control/controlclient from tailscale.com/cmd/tailscaled+ - tailscale.com/control/controlhttp from tailscale.com/control/controlclient + tailscale.com/control/controlhttp from tailscale.com/control/ts2021+ tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp tailscale.com/control/controlknobs from tailscale.com/control/controlclient+ + tailscale.com/control/ts2021 from tailscale.com/control/controlclient tailscale.com/derp from tailscale.com/derp/derphttp+ - tailscale.com/derp/derpconst from tailscale.com/derp+ + tailscale.com/derp/derpconst from tailscale.com/derp/derphttp+ tailscale.com/derp/derphttp from tailscale.com/cmd/tailscaled+ - tailscale.com/disco from tailscale.com/derp+ - tailscale.com/doctor from tailscale.com/ipn/ipnlocal - tailscale.com/doctor/ethtool from tailscale.com/ipn/ipnlocal - đŸ’Ŗ tailscale.com/doctor/permissions from tailscale.com/ipn/ipnlocal - tailscale.com/doctor/routetable from tailscale.com/ipn/ipnlocal + tailscale.com/disco from tailscale.com/feature/relayserver+ + tailscale.com/doctor from tailscale.com/feature/doctor + tailscale.com/doctor/ethtool from tailscale.com/feature/doctor + đŸ’Ŗ tailscale.com/doctor/permissions from tailscale.com/feature/doctor + tailscale.com/doctor/routetable from tailscale.com/feature/doctor tailscale.com/drive from tailscale.com/client/local+ tailscale.com/drive/driveimpl from tailscale.com/cmd/tailscaled tailscale.com/drive/driveimpl/compositedav from tailscale.com/drive/driveimpl @@ -273,31 +289,56 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/envknob from tailscale.com/client/local+ tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/feature/wakeonlan+ + tailscale.com/feature/ace from tailscale.com/feature/condregister + tailscale.com/feature/appconnectors from tailscale.com/feature/condregister + tailscale.com/feature/buildfeatures from tailscale.com/wgengine/magicsock+ + tailscale.com/feature/c2n from tailscale.com/feature/condregister tailscale.com/feature/capture from tailscale.com/feature/condregister + tailscale.com/feature/clientupdate from tailscale.com/feature/condregister + tailscale.com/feature/condlite/expvar from tailscale.com/wgengine/magicsock tailscale.com/feature/condregister from tailscale.com/cmd/tailscaled + tailscale.com/feature/condregister/portmapper from tailscale.com/feature/condregister + tailscale.com/feature/condregister/useproxy from tailscale.com/feature/condregister + tailscale.com/feature/conn25 from tailscale.com/feature/condregister + tailscale.com/feature/debugportmapper from tailscale.com/feature/condregister + tailscale.com/feature/doctor from tailscale.com/feature/condregister + tailscale.com/feature/drive from tailscale.com/feature/condregister + L tailscale.com/feature/linkspeed from tailscale.com/feature/condregister + L tailscale.com/feature/linuxdnsfight from tailscale.com/feature/condregister + tailscale.com/feature/portlist from tailscale.com/feature/condregister + tailscale.com/feature/portmapper from tailscale.com/feature/condregister/portmapper + tailscale.com/feature/posture from tailscale.com/feature/condregister tailscale.com/feature/relayserver from tailscale.com/feature/condregister + tailscale.com/feature/routecheck from tailscale.com/feature/condregister + tailscale.com/feature/runtimemetrics from tailscale.com/feature/condregister + L tailscale.com/feature/sdnotify from tailscale.com/feature/condregister + LD tailscale.com/feature/ssh from tailscale.com/cmd/tailscaled + tailscale.com/feature/syspolicy from tailscale.com/feature/condregister tailscale.com/feature/taildrop from tailscale.com/feature/condregister + tailscale.com/feature/tailnetlock from tailscale.com/feature/condregister L tailscale.com/feature/tap from tailscale.com/feature/condregister tailscale.com/feature/tpm from tailscale.com/feature/condregister + L đŸ’Ŗ tailscale.com/feature/tundevstats from tailscale.com/feature/condregister + tailscale.com/feature/useproxy from tailscale.com/feature/condregister/useproxy tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal tailscale.com/hostinfo from tailscale.com/client/web+ - tailscale.com/internal/noiseconn from tailscale.com/control/controlclient tailscale.com/ipn from tailscale.com/client/local+ W tailscale.com/ipn/auditlog from tailscale.com/cmd/tailscaled tailscale.com/ipn/conffile from tailscale.com/cmd/tailscaled+ - W đŸ’Ŗ tailscale.com/ipn/desktop from tailscale.com/cmd/tailscaled + W đŸ’Ŗ tailscale.com/ipn/desktop from tailscale.com/feature/condregister đŸ’Ŗ tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnlocal+ tailscale.com/ipn/ipnext from tailscale.com/ipn/auditlog+ tailscale.com/ipn/ipnlocal from tailscale.com/cmd/tailscaled+ + tailscale.com/ipn/ipnlocal/netmapcache from tailscale.com/ipn/ipnlocal tailscale.com/ipn/ipnserver from tailscale.com/cmd/tailscaled tailscale.com/ipn/ipnstate from tailscale.com/client/local+ tailscale.com/ipn/localapi from tailscale.com/ipn/ipnserver+ - tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/policy from tailscale.com/feature/portlist tailscale.com/ipn/store from tailscale.com/cmd/tailscaled+ - L tailscale.com/ipn/store/awsstore from tailscale.com/ipn/store - L tailscale.com/ipn/store/kubestore from tailscale.com/ipn/store + L tailscale.com/ipn/store/awsstore from tailscale.com/feature/condregister + L tailscale.com/ipn/store/kubestore from tailscale.com/feature/condregister tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+ L tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ L tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore @@ -307,20 +348,19 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/log/sockstatlog from tailscale.com/ipn/ipnlocal tailscale.com/logpolicy from tailscale.com/cmd/tailscaled+ tailscale.com/logtail from tailscale.com/cmd/tailscaled+ - tailscale.com/logtail/backoff from tailscale.com/cmd/tailscaled+ tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ - tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/metrics from tailscale.com/tsweb+ + tailscale.com/net/ace from tailscale.com/feature/ace tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+ + đŸ’Ŗ tailscale.com/net/batching from tailscale.com/wgengine/magicsock+ tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ - tailscale.com/net/connstats from tailscale.com/net/tstun+ tailscale.com/net/dns from tailscale.com/cmd/tailscaled+ tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ - tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback tailscale.com/net/dns/resolvconffile from tailscale.com/net/dns+ tailscale.com/net/dns/resolver from tailscale.com/net/dns+ tailscale.com/net/dnscache from tailscale.com/control/controlclient+ tailscale.com/net/dnsfallback from tailscale.com/cmd/tailscaled+ - tailscale.com/net/flowtrack from tailscale.com/net/packet+ + tailscale.com/net/flowtrack from tailscale.com/wgengine+ tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ tailscale.com/net/netaddr from tailscale.com/ipn+ tailscale.com/net/netcheck from tailscale.com/wgengine/magicsock+ @@ -332,49 +372,54 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W đŸ’Ŗ tailscale.com/net/netstat from tailscale.com/portlist tailscale.com/net/netutil from tailscale.com/client/local+ tailscale.com/net/netx from tailscale.com/control/controlclient+ - tailscale.com/net/packet from tailscale.com/net/connstats+ - tailscale.com/net/packet/checksum from tailscale.com/net/tstun + tailscale.com/net/packet from tailscale.com/feature/capture+ + tailscale.com/net/packet/checksum from tailscale.com/net/tstun+ tailscale.com/net/ping from tailscale.com/net/netcheck+ - tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ + tailscale.com/net/portmapper from tailscale.com/feature/portmapper+ + tailscale.com/net/portmapper/portmappertype from tailscale.com/feature/portmapper+ tailscale.com/net/proxymux from tailscale.com/cmd/tailscaled + tailscale.com/net/routecheck from tailscale.com/feature/routecheck+ tailscale.com/net/routetable from tailscale.com/doctor/routetable + đŸ’Ŗ tailscale.com/net/sockopts from tailscale.com/wgengine/magicsock+ tailscale.com/net/socks5 from tailscale.com/cmd/tailscaled tailscale.com/net/sockstats from tailscale.com/control/controlclient+ tailscale.com/net/stun from tailscale.com/ipn/localapi+ - L tailscale.com/net/tcpinfo from tailscale.com/derp tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/traffic from tailscale.com/ipn/ipnlocal+ tailscale.com/net/tsaddr from tailscale.com/client/web+ tailscale.com/net/tsdial from tailscale.com/cmd/tailscaled+ - đŸ’Ŗ tailscale.com/net/tshttpproxy from tailscale.com/clientupdate/distsign+ + đŸ’Ŗ tailscale.com/net/tshttpproxy from tailscale.com/feature/useproxy tailscale.com/net/tstun from tailscale.com/cmd/tailscaled+ tailscale.com/net/udprelay from tailscale.com/feature/relayserver - tailscale.com/net/udprelay/endpoint from tailscale.com/feature/relayserver+ + tailscale.com/net/udprelay/endpoint from tailscale.com/net/udprelay+ + tailscale.com/net/udprelay/status from tailscale.com/client/local+ tailscale.com/omit from tailscale.com/ipn/conffile tailscale.com/paths from tailscale.com/client/local+ - đŸ’Ŗ tailscale.com/portlist from tailscale.com/ipn/ipnlocal - tailscale.com/posture from tailscale.com/ipn/ipnlocal + đŸ’Ŗ tailscale.com/portlist from tailscale.com/feature/portlist + tailscale.com/posture from tailscale.com/feature/posture tailscale.com/proxymap from tailscale.com/tsd+ đŸ’Ŗ tailscale.com/safesocket from tailscale.com/client/local+ LD tailscale.com/sessionrecording from tailscale.com/ssh/tailssh - LD đŸ’Ŗ tailscale.com/ssh/tailssh from tailscale.com/cmd/tailscaled + LD đŸ’Ŗ tailscale.com/ssh/tailssh from tailscale.com/feature/ssh tailscale.com/syncs from tailscale.com/cmd/tailscaled+ tailscale.com/tailcfg from tailscale.com/client/local+ tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal - LD tailscale.com/tempfork/gliderlabs/ssh from tailscale.com/ssh/tailssh tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock - tailscale.com/tempfork/httprec from tailscale.com/control/controlclient + tailscale.com/tempfork/httprec from tailscale.com/feature/c2n tailscale.com/tka from tailscale.com/client/local+ tailscale.com/tsconst from tailscale.com/net/netmon+ tailscale.com/tsd from tailscale.com/cmd/tailscaled+ tailscale.com/tstime from tailscale.com/control/controlclient+ tailscale.com/tstime/mono from tailscale.com/net/tstun+ - tailscale.com/tstime/rate from tailscale.com/derp+ - tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tstime/rate from tailscale.com/wgengine/filter + tailscale.com/tsweb from tailscale.com/util/eventbus+ tailscale.com/tsweb/varz from tailscale.com/cmd/tailscaled+ - tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal + tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+ + tailscale.com/types/bools from tailscale.com/wgengine/netlog tailscale.com/types/dnstype from tailscale.com/ipn/ipnlocal+ tailscale.com/types/empty from tailscale.com/ipn+ + tailscale.com/types/events from tailscale.com/control/controlclient+ tailscale.com/types/flagtype from tailscale.com/cmd/tailscaled tailscale.com/types/ipproto from tailscale.com/net/flowtrack+ tailscale.com/types/key from tailscale.com/client/local+ @@ -382,23 +427,27 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/types/logger from tailscale.com/appc+ tailscale.com/types/logid from tailscale.com/cmd/tailscaled+ tailscale.com/types/mapx from tailscale.com/ipn/ipnext - tailscale.com/types/netlogtype from tailscale.com/net/connstats+ + tailscale.com/types/netlogfunc from tailscale.com/net/tstun+ + tailscale.com/types/netlogtype from tailscale.com/wgengine/netlog tailscale.com/types/netmap from tailscale.com/control/controlclient+ tailscale.com/types/nettype from tailscale.com/ipn/localapi+ - tailscale.com/types/opt from tailscale.com/client/tailscale+ + tailscale.com/types/opt from tailscale.com/control/controlknobs+ tailscale.com/types/persist from tailscale.com/control/controlclient+ tailscale.com/types/preftype from tailscale.com/ipn+ - tailscale.com/types/ptr from tailscale.com/control/controlclient+ tailscale.com/types/result from tailscale.com/util/lineiter tailscale.com/types/structs from tailscale.com/control/controlclient+ tailscale.com/types/tkatype from tailscale.com/tka+ tailscale.com/types/views from tailscale.com/ipn/ipnlocal+ - tailscale.com/util/cibuild from tailscale.com/health + tailscale.com/util/backoff from tailscale.com/cmd/tailscaled+ + tailscale.com/util/bufiox from tailscale.com/types/key + tailscale.com/util/checkchange from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/cibuild from tailscale.com/health+ tailscale.com/util/clientmetric from tailscale.com/control/controlclient+ tailscale.com/util/cloudenv from tailscale.com/net/dns/resolver+ + tailscale.com/util/cloudinfo from tailscale.com/wgengine/magicsock+ tailscale.com/util/cmpver from tailscale.com/net/dns+ tailscale.com/util/ctxkey from tailscale.com/ipn/ipnlocal+ - đŸ’Ŗ tailscale.com/util/deephash from tailscale.com/ipn/ipnlocal+ + đŸ’Ŗ tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting L đŸ’Ŗ tailscale.com/util/dirwalk from tailscale.com/metrics+ tailscale.com/util/dnsname from tailscale.com/appc+ tailscale.com/util/eventbus from tailscale.com/tsd+ @@ -407,11 +456,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/groupmember from tailscale.com/client/web+ đŸ’Ŗ tailscale.com/util/hashx from tailscale.com/util/deephash tailscale.com/util/httphdr from tailscale.com/feature/taildrop - tailscale.com/util/httpm from tailscale.com/client/tailscale+ + tailscale.com/util/httpm from tailscale.com/client/web+ tailscale.com/util/lineiter from tailscale.com/hostinfo+ - L tailscale.com/util/linuxfw from tailscale.com/net/netns+ + L tailscale.com/util/linuxfw from tailscale.com/wgengine/router/osrouter tailscale.com/util/mak from tailscale.com/control/controlclient+ - tailscale.com/util/multierr from tailscale.com/cmd/tailscaled+ + tailscale.com/util/multierr from tailscale.com/feature/taildrop tailscale.com/util/must from tailscale.com/clientupdate/distsign+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto đŸ’Ŗ tailscale.com/util/osdiag from tailscale.com/cmd/tailscaled+ @@ -422,23 +471,24 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/util/race from tailscale.com/net/dns/resolver tailscale.com/util/racebuild from tailscale.com/logpolicy tailscale.com/util/rands from tailscale.com/ipn/ipnlocal+ - tailscale.com/util/ringbuffer from tailscale.com/wgengine/magicsock - tailscale.com/util/set from tailscale.com/derp+ + tailscale.com/util/ringlog from tailscale.com/wgengine/magicsock + tailscale.com/util/set from tailscale.com/control/controlclient+ tailscale.com/util/singleflight from tailscale.com/control/controlclient+ - tailscale.com/util/slicesx from tailscale.com/net/dns/recursive+ - tailscale.com/util/syspolicy from tailscale.com/cmd/tailscaled+ + tailscale.com/util/slicesx from tailscale.com/appc+ + tailscale.com/util/syspolicy from tailscale.com/feature/syspolicy tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy/setting+ tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy/internal/metrics+ tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source + tailscale.com/util/syspolicy/pkey from tailscale.com/cmd/tailscaled+ + tailscale.com/util/syspolicy/policyclient from tailscale.com/control/controlclient+ + tailscale.com/util/syspolicy/ptype from tailscale.com/util/syspolicy+ tailscale.com/util/syspolicy/rsop from tailscale.com/util/syspolicy+ tailscale.com/util/syspolicy/setting from tailscale.com/util/syspolicy+ tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy+ - tailscale.com/util/sysresources from tailscale.com/wgengine/magicsock - tailscale.com/util/systemd from tailscale.com/control/controlclient+ tailscale.com/util/testenv from tailscale.com/ipn/ipnlocal+ tailscale.com/util/truncate from tailscale.com/logtail tailscale.com/util/usermetric from tailscale.com/health+ - tailscale.com/util/vizerror from tailscale.com/tailcfg+ + tailscale.com/util/vizerror from tailscale.com/tsweb+ đŸ’Ŗ tailscale.com/util/winutil from tailscale.com/clientupdate+ W đŸ’Ŗ tailscale.com/util/winutil/authenticode from tailscale.com/clientupdate+ W đŸ’Ŗ tailscale.com/util/winutil/gp from tailscale.com/net/dns+ @@ -446,7 +496,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W đŸ’Ŗ tailscale.com/util/winutil/winenv from tailscale.com/hostinfo+ tailscale.com/util/zstdframe from tailscale.com/control/controlclient+ tailscale.com/version from tailscale.com/client/web+ - tailscale.com/version/distro from tailscale.com/client/web+ + tailscale.com/version/distro from tailscale.com/hostinfo+ W tailscale.com/wf from tailscale.com/cmd/tailscaled tailscale.com/wgengine from tailscale.com/cmd/tailscaled+ tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ @@ -456,19 +506,20 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/wgengine/netstack from tailscale.com/cmd/tailscaled tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ tailscale.com/wgengine/router from tailscale.com/cmd/tailscaled+ + tailscale.com/wgengine/router/osrouter from tailscale.com/feature/condregister tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal đŸ’Ŗ tailscale.com/wgengine/wgint from tailscale.com/wgengine+ tailscale.com/wgengine/wglog from tailscale.com/wgengine - W đŸ’Ŗ tailscale.com/wgengine/winnet from tailscale.com/wgengine/router + W đŸ’Ŗ tailscale.com/wgengine/winnet from tailscale.com/wgengine/router/osrouter golang.org/x/crypto/argon2 from tailscale.com/tka golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305+ - golang.org/x/crypto/chacha20poly1305 from crypto/tls+ - golang.org/x/crypto/cryptobyte from crypto/ecdsa+ - golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + golang.org/x/crypto/chacha20poly1305 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/cryptobyte from tailscale.com/feature/tpm + golang.org/x/crypto/cryptobyte/asn1 from golang.org/x/crypto/cryptobyte+ golang.org/x/crypto/curve25519 from golang.org/x/crypto/ssh+ golang.org/x/crypto/hkdf from tailscale.com/control/controlbase golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ @@ -482,22 +533,18 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ golang.org/x/exp/maps from tailscale.com/ipn/store/mem+ golang.org/x/net/bpf from github.com/mdlayher/genetlink+ - golang.org/x/net/dns/dnsmessage from net+ - golang.org/x/net/http/httpguts from golang.org/x/net/http2+ - golang.org/x/net/http/httpproxy from net/http+ - golang.org/x/net/http2 from golang.org/x/net/http2/h2c+ - golang.org/x/net/http2/h2c from tailscale.com/ipn/ipnlocal - golang.org/x/net/http2/hpack from golang.org/x/net/http2+ - golang.org/x/net/icmp from tailscale.com/net/ping+ + golang.org/x/net/dns/dnsmessage from tailscale.com/appc+ + golang.org/x/net/http/httpguts from tailscale.com/ipn/ipnlocal + golang.org/x/net/http/httpproxy from tailscale.com/net/tshttpproxy + golang.org/x/net/icmp from tailscale.com/net/ping golang.org/x/net/idna from golang.org/x/net/http/httpguts+ - golang.org/x/net/internal/httpcommon from golang.org/x/net/http2 golang.org/x/net/internal/iana from golang.org/x/net/icmp+ - golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/ipv4+ golang.org/x/net/internal/socks from golang.org/x/net/proxy - golang.org/x/net/ipv4 from github.com/miekg/dns+ - golang.org/x/net/ipv6 from github.com/miekg/dns+ + golang.org/x/net/ipv4 from github.com/tailscale/wireguard-go/conn+ + golang.org/x/net/ipv6 from github.com/tailscale/wireguard-go/conn+ golang.org/x/net/proxy from tailscale.com/net/netns - D golang.org/x/net/route from net+ + D golang.org/x/net/route from tailscale.com/net/netmon+ golang.org/x/sync/errgroup from github.com/mdlayher/socket+ golang.org/x/sync/singleflight from github.com/jellydator/ttlcache/v3 golang.org/x/sys/cpu from github.com/tailscale/certstore+ @@ -513,18 +560,35 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ golang.org/x/text/unicode/norm from golang.org/x/net/idna golang.org/x/time/rate from gvisor.dev/gvisor/pkg/log+ + vendor/golang.org/x/crypto/chacha20 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/crypto/chacha20poly1305 from crypto/hpke+ + vendor/golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + vendor/golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + vendor/golang.org/x/crypto/internal/alias from vendor/golang.org/x/crypto/chacha20+ + vendor/golang.org/x/crypto/internal/poly1305 from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/net/dns/dnsmessage from net + vendor/golang.org/x/net/http/httpguts from net/http+ + vendor/golang.org/x/net/http/httpproxy from net/http + vendor/golang.org/x/net/http2/hpack from net/http+ + vendor/golang.org/x/net/idna from net/http+ + vendor/golang.org/x/sys/cpu from vendor/golang.org/x/crypto/chacha20poly1305 + vendor/golang.org/x/text/secure/bidirule from vendor/golang.org/x/net/idna + vendor/golang.org/x/text/transform from vendor/golang.org/x/text/secure/bidirule+ + vendor/golang.org/x/text/unicode/bidi from vendor/golang.org/x/net/idna+ + vendor/golang.org/x/text/unicode/norm from vendor/golang.org/x/net/idna archive/tar from tailscale.com/clientupdate + L archive/zip from tailscale.com/clientupdate bufio from compress/flate+ bytes from archive/tar+ cmp from slices+ compress/flate from compress/gzip+ - compress/gzip from golang.org/x/net/http2+ + compress/gzip from github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding+ W compress/zlib from debug/pe container/heap from github.com/jellydator/ttlcache/v3+ container/list from crypto/tls+ context from crypto/tls+ crypto from crypto/ecdh+ - crypto/aes from crypto/internal/hpke+ + crypto/aes from crypto/tls+ crypto/cipher from crypto/aes+ crypto/des from crypto/tls+ crypto/dsa from crypto/x509+ @@ -532,27 +596,30 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/ecdsa from crypto/tls+ crypto/ed25519 from crypto/tls+ crypto/elliptic from crypto/ecdsa+ + crypto/fips140 from crypto/tls/internal/fips140tls+ + crypto/hkdf from crypto/hpke+ crypto/hmac from crypto/tls+ + crypto/hpke from crypto/tls crypto/internal/boring from crypto/aes+ crypto/internal/boring/bbig from crypto/ecdsa+ crypto/internal/boring/sig from crypto/internal/boring - crypto/internal/entropy from crypto/internal/fips140/drbg - crypto/internal/fips140 from crypto/internal/fips140/aes+ + crypto/internal/constanttime from crypto/internal/fips140/edwards25519+ + crypto/internal/fips140 from crypto/fips140+ crypto/internal/fips140/aes from crypto/aes+ crypto/internal/fips140/aes/gcm from crypto/cipher+ crypto/internal/fips140/alias from crypto/cipher+ crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ - crypto/internal/fips140/check from crypto/internal/fips140/aes+ - crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/check from crypto/fips140+ + crypto/internal/fips140/drbg from crypto/hpke+ crypto/internal/fips140/ecdh from crypto/ecdh crypto/internal/fips140/ecdsa from crypto/ecdsa crypto/internal/fips140/ed25519 from crypto/ed25519 crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 crypto/internal/fips140/edwards25519/field from crypto/ecdh+ - crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ + crypto/internal/fips140/hkdf from crypto/hkdf+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls+ - crypto/internal/fips140/nistec from crypto/elliptic+ + crypto/internal/fips140/mlkem from crypto/mlkem + crypto/internal/fips140/nistec from crypto/ecdsa+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ @@ -561,23 +628,25 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ crypto/internal/fips140/tls12 from crypto/tls crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140cache from crypto/ecdsa+ crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140deps/time from crypto/internal/entropy/v1.0.0 crypto/internal/fips140hash from crypto/ecdsa+ crypto/internal/fips140only from crypto/cipher+ - crypto/internal/hpke from crypto/tls crypto/internal/impl from crypto/internal/fips140/aes+ - crypto/internal/randutil from crypto/dsa+ - crypto/internal/sysrand from crypto/internal/entropy+ + crypto/internal/rand from crypto/dsa+ + crypto/internal/randutil from crypto/internal/rand + crypto/internal/sysrand from crypto/internal/fips140/drbg crypto/md5 from crypto/tls+ - LD crypto/mlkem from golang.org/x/crypto/ssh + crypto/mlkem from golang.org/x/crypto/ssh+ crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ crypto/rsa from crypto/tls+ crypto/sha1 from crypto/tls+ crypto/sha256 from crypto/tls+ - crypto/sha3 from crypto/internal/fips140hash + crypto/sha3 from crypto/internal/fips140hash+ crypto/sha512 from crypto/ecdsa+ crypto/subtle from crypto/cipher+ crypto/tls from github.com/aws/aws-sdk-go-v2/aws/transport/http+ @@ -585,7 +654,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/x509 from crypto/tls+ D crypto/x509/internal/macos from crypto/x509 crypto/x509/pkix from crypto/x509+ - DW database/sql/driver from github.com/google/uuid + W database/sql/driver from github.com/google/uuid W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ @@ -599,12 +668,13 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de encoding/pem from crypto/tls+ encoding/xml from github.com/aws/aws-sdk-go-v2/aws/protocol/xml+ errors from archive/tar+ - expvar from tailscale.com/derp+ + expvar from tailscale.com/cmd/tailscaled+ flag from tailscale.com/cmd/tailscaled+ fmt from archive/tar+ hash from compress/zlib+ hash/adler32 from compress/zlib+ hash/crc32 from compress/gzip+ + hash/fnv from tailscale.com/net/traffic hash/maphash from go4.org/mem html from html/template+ html/template from tailscale.com/util/eventbus @@ -621,37 +691,45 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de internal/goarch from crypto/internal/fips140deps/cpu+ internal/godebug from archive/tar+ internal/godebugs from internal/godebug+ - internal/goexperiment from hash/maphash+ + internal/goexperiment from net/http/pprof+ internal/goos from crypto/x509+ - internal/itoa from internal/poll+ internal/msan from internal/runtime/maps+ internal/nettrace from net+ - internal/oserror from io/fs+ + internal/oserror from internal/syscall/windows+ internal/poll from net+ internal/profile from net/http/pprof internal/profilerecord from runtime+ internal/race from internal/poll+ internal/reflectlite from context+ + D internal/routebsd from net internal/runtime/atomic from internal/runtime/exithook+ + L internal/runtime/cgroup from runtime internal/runtime/exithook from runtime - internal/runtime/maps from reflect+ + internal/runtime/gc from internal/runtime/gc/scan+ + internal/runtime/gc/scan from runtime + internal/runtime/maps from hash/maphash+ internal/runtime/math from internal/runtime/maps+ + internal/runtime/pprof/label from runtime+ internal/runtime/sys from crypto/subtle+ - L internal/runtime/syscall from runtime+ - W internal/saferio from debug/pe + L internal/runtime/syscall/linux from internal/runtime/cgroup+ + W internal/runtime/syscall/windows from internal/syscall/windows+ + internal/saferio from debug/pe+ internal/singleflight from net + internal/strconv from internal/poll+ internal/stringslite from embed+ internal/sync from sync+ + internal/synctest from sync internal/syscall/execenv from os+ LD internal/syscall/unix from crypto/internal/sysrand+ - W internal/syscall/windows from crypto/internal/sysrand+ + W internal/syscall/windows from crypto/internal/fips140deps/time+ W internal/syscall/windows/registry from mime+ W internal/syscall/windows/sysdll from internal/syscall/windows+ internal/testlog from os + internal/trace/tracev2 from runtime+ internal/unsafeheader from internal/reflectlite+ io from archive/tar+ io/fs from archive/tar+ - io/ioutil from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + io/ioutil from github.com/aws/aws-sdk-go-v2/service/sso+ iter from maps+ log from expvar+ log/internal from log @@ -667,10 +745,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de mime/quotedprintable from mime/multipart net from crypto/tls+ net/http from expvar+ - net/http/httptrace from github.com/prometheus-community/pro-bing+ + net/http/httptrace from github.com/aws/smithy-go/transport/http+ net/http/httputil from github.com/aws/smithy-go/transport/http+ net/http/internal from net/http+ net/http/internal/ascii from net/http+ + net/http/internal/httpcommon from net/http net/http/pprof from tailscale.com/cmd/tailscaled+ net/netip from github.com/tailscale/wireguard-go/conn+ net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ @@ -686,12 +765,14 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de regexp/syntax from regexp runtime from archive/tar+ runtime/debug from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+ + runtime/metrics from tailscale.com/feature/runtimemetrics runtime/pprof from net/http/pprof+ runtime/trace from net/http/pprof slices from tailscale.com/appc+ sort from compress/flate+ strconv from archive/tar+ strings from archive/tar+ + W structs from internal/syscall/windows sync from archive/tar+ sync/atomic from context+ syscall from archive/tar+ @@ -704,4 +785,4 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de unicode/utf8 from bufio+ unique from net/netip unsafe from bytes+ - weak from unique + weak from unique+ diff --git a/cmd/tailscaled/deps_test.go b/cmd/tailscaled/deps_test.go index 7f06abc6c5ba1..e91509765259a 100644 --- a/cmd/tailscaled/deps_test.go +++ b/cmd/tailscaled/deps_test.go @@ -1,11 +1,15 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main import ( + "maps" + "slices" + "strings" "testing" + "tailscale.com/feature/featuretags" "tailscale.com/tstest/deptest" ) @@ -14,8 +18,9 @@ func TestOmitSSH(t *testing.T) { deptest.DepChecker{ GOOS: "linux", GOARCH: "amd64", - Tags: "ts_omit_ssh", + Tags: "ts_omit_ssh,ts_include_cli", BadDeps: map[string]string{ + "golang.org/x/crypto/ssh": msg, "tailscale.com/ssh/tailssh": msg, "tailscale.com/sessionrecording": msg, "github.com/anmitsu/go-shlex": msg, @@ -27,3 +32,285 @@ func TestOmitSSH(t *testing.T) { }, }.Check(t) } + +func TestOmitSyspolicy(t *testing.T) { + const msg = "unexpected syspolicy usage with ts_omit_syspolicy" + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_syspolicy,ts_include_cli", + BadDeps: map[string]string{ + "tailscale.com/util/syspolicy": msg, + "tailscale.com/util/syspolicy/setting": msg, + "tailscale.com/util/syspolicy/rsop": msg, + }, + }.Check(t) +} + +func TestOmitLocalClient(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_webclient,ts_omit_relayserver,ts_omit_oauthkey,ts_omit_acme", + BadDeps: map[string]string{ + "tailscale.com/client/local": "unexpected", + }, + }.Check(t) +} + +// Test that we can build a binary without reflect.MethodByName. +// See https://github.com/tailscale/tailscale/issues/17063 +func TestOmitReflectThings(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_include_cli,ts_omit_systray,ts_omit_debugeventbus,ts_omit_webclient", + BadDeps: map[string]string{ + "text/template": "unexpected text/template usage", + "html/template": "unexpected text/template usage", + }, + OnDep: func(dep string) { + if strings.Contains(dep, "systray") { + t.Errorf("unexpected systray dep %q", dep) + } + }, + }.Check(t) +} + +func TestOmitDrive(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_drive,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "driveimpl") { + t.Errorf("unexpected dep with ts_omit_drive: %q", dep) + } + if strings.Contains(dep, "webdav") { + t.Errorf("unexpected dep with ts_omit_drive: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitPortmapper(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_portmapper,ts_include_cli,ts_omit_debugportmapper", + OnDep: func(dep string) { + if dep == "tailscale.com/net/portmapper" { + t.Errorf("unexpected dep with ts_omit_portmapper: %q", dep) + return + } + if strings.Contains(dep, "goupnp") || strings.Contains(dep, "/soap") || + strings.Contains(dep, "internetgateway2") { + t.Errorf("unexpected dep with ts_omit_portmapper: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitACME(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_acme,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "/acme") { + t.Errorf("unexpected dep with ts_omit_acme: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitCaptivePortal(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_captiveportal,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "captive") { + t.Errorf("unexpected dep with ts_omit_captiveportal: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitAuth(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_oauthkey,ts_omit_identityfederation,ts_include_cli", + OnDep: func(dep string) { + if strings.HasPrefix(dep, "golang.org/x/oauth2") { + t.Errorf("unexpected oauth2 dep: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitOutboundProxy(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_outboundproxy,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "socks5") || strings.Contains(dep, "proxymux") { + t.Errorf("unexpected dep with ts_omit_outboundproxy: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitDBus(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_networkmanager,ts_omit_dbus,ts_omit_resolved,ts_omit_systray,ts_omit_ssh,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "dbus") { + t.Errorf("unexpected DBus dep: %q", dep) + } + }, + }.Check(t) +} + +func TestNetstack(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_gro,ts_omit_netstack,ts_omit_outboundproxy,ts_omit_serve,ts_omit_ssh,ts_omit_webclient,ts_omit_tap", + OnDep: func(dep string) { + if strings.Contains(dep, "gvisor") { + t.Errorf("unexpected gvisor dep: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitPortlist(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_portlist,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "portlist") { + t.Errorf("unexpected dep: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitRouteCheck(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_routecheck,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "routecheck") { + t.Errorf("unexpected dep: %q", dep) + } + }, + }.Check(t) +} + +func TestOmitGRO(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_gro,ts_include_cli", + BadDeps: map[string]string{ + "gvisor.dev/gvisor/pkg/tcpip/stack/gro": "unexpected dep with ts_omit_gro", + }, + }.Check(t) +} + +func TestOmitUseProxy(t *testing.T) { + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: "ts_omit_useproxy,ts_include_cli", + OnDep: func(dep string) { + if strings.Contains(dep, "tshttproxy") { + t.Errorf("unexpected dep: %q", dep) + } + }, + }.Check(t) +} + +func minTags() string { + var tags []string + for _, f := range slices.Sorted(maps.Keys(featuretags.Features)) { + if f.IsOmittable() { + tags = append(tags, f.OmitTag()) + } + } + return strings.Join(tags, ",") +} + +func TestMinTailscaledNoCLI(t *testing.T) { + badSubstrs := []string{ + "cbor", + "regexp", + "golang.org/x/net/proxy", + "internal/socks", + "github.com/tailscale/peercred", + "tailscale.com/types/netlogtype", + "deephash", + "util/hashx", + } + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: minTags(), + OnDep: func(dep string) { + for _, bad := range badSubstrs { + if strings.Contains(dep, bad) { + t.Errorf("unexpected dep: %q", dep) + } + } + }, + }.Check(t) +} + +func TestMinTailscaledWithCLI(t *testing.T) { + badSubstrs := []string{ + "cbor", + "hujson", + "multierr", // https://github.com/tailscale/tailscale/pull/17379 + "tailscale.com/metrics", + "tailscale.com/tsweb/varz", + "dirwalk", + "deephash", + "util/hashx", + } + deptest.DepChecker{ + GOOS: "linux", + GOARCH: "amd64", + Tags: minTags() + ",ts_include_cli", + OnDep: func(dep string) { + for _, bad := range badSubstrs { + if strings.Contains(dep, bad) { + t.Errorf("unexpected dep: %q", dep) + } + } + }, + BadDeps: map[string]string{ + "golang.org/x/net/http2": "unexpected x/net/http2 dep; tailscale/tailscale#17305", + "expvar": "unexpected expvar dep", + "runtime/pprof": "unexpected runtime/pprof dep", + "net/http/pprof": "unexpected net/http/pprof dep", + "github.com/mdlayher/genetlink": "unexpected genetlink dep", + "tailscale.com/clientupdate": "unexpected clientupdate dep", + "filippo.io/edwards25519": "unexpected edwards25519 dep", + "github.com/hdevalence/ed25519consensus": "unexpected ed25519consensus dep", + "tailscale.com/clientupdate/distsign": "unexpected distsign dep", + "archive/tar": "unexpected archive/tar dep", + "tailscale.com/feature/conn25": "unexpected conn25 dep", + "regexp": "unexpected regexp dep; bloats binary", + "github.com/toqueteos/webbrowser": "unexpected webbrowser dep with ts_omit_webbrowser", + "github.com/mattn/go-colorable": "unexpected go-colorable dep with ts_omit_colorable", + }, + }.Check(t) +} diff --git a/cmd/tailscaled/flag.go b/cmd/tailscaled/flag.go new file mode 100644 index 0000000000000..357210a29c426 --- /dev/null +++ b/cmd/tailscaled/flag.go @@ -0,0 +1,31 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import "strconv" + +// boolFlag is a flag.Value that tracks whether it was ever set. +type boolFlag struct { + set bool + v bool +} + +func (b *boolFlag) String() string { + if b == nil || !b.set { + return "unset" + } + return strconv.FormatBool(b.v) +} + +func (b *boolFlag) Set(s string) error { + v, err := strconv.ParseBool(s) + if err != nil { + return err + } + b.v = v + b.set = true + return nil +} + +func (b *boolFlag) IsBoolFlag() bool { return true } diff --git a/cmd/tailscaled/generate.go b/cmd/tailscaled/generate.go index 5c2e9be915980..36a4fa671dddb 100644 --- a/cmd/tailscaled/generate.go +++ b/cmd/tailscaled/generate.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/tailscaled/install_darwin.go b/cmd/tailscaled/install_darwin.go index 05e5eaed8af90..15d9e54621181 100644 --- a/cmd/tailscaled/install_darwin.go +++ b/cmd/tailscaled/install_darwin.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 diff --git a/cmd/tailscaled/install_windows.go b/cmd/tailscaled/install_windows.go index c667539b04d4f..d0f40b37d1156 100644 --- a/cmd/tailscaled/install_windows.go +++ b/cmd/tailscaled/install_windows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 @@ -16,8 +16,8 @@ import ( "golang.org/x/sys/windows/svc" "golang.org/x/sys/windows/svc/mgr" "tailscale.com/cmd/tailscaled/tailscaledhooks" - "tailscale.com/logtail/backoff" "tailscale.com/types/logger" + "tailscale.com/util/backoff" ) func init() { @@ -25,6 +25,16 @@ func init() { uninstallSystemDaemon = uninstallSystemDaemonWindows } +// serviceDependencies lists all system services that tailscaled depends on. +// This list must be kept in sync with the TailscaledDependencies preprocessor +// variable in the installer. +var serviceDependencies = []string{ + "Dnscache", + "iphlpsvc", + "netprofm", + "WinHttpAutoProxySvc", +} + func installSystemDaemonWindows(args []string) (err error) { m, err := mgr.Connect() if err != nil { @@ -48,6 +58,7 @@ func installSystemDaemonWindows(args []string) (err error) { ServiceType: windows.SERVICE_WIN32_OWN_PROCESS, StartType: mgr.StartAutomatic, ErrorControl: mgr.ErrorNormal, + Dependencies: serviceDependencies, DisplayName: serviceName, Description: "Connects this computer to others on the Tailscale network.", } diff --git a/cmd/tailscaled/netstack.go b/cmd/tailscaled/netstack.go new file mode 100644 index 0000000000000..d896f384fcc98 --- /dev/null +++ b/cmd/tailscaled/netstack.go @@ -0,0 +1,75 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_netstack + +package main + +import ( + "context" + "expvar" + "net" + "net/netip" + + "tailscale.com/tsd" + "tailscale.com/types/logger" + "tailscale.com/wgengine/netstack" +) + +func init() { + hookNewNetstack.Set(newNetstack) +} + +func newNetstack(logf logger.Logf, sys *tsd.System, onlyNetstack bool) (tsd.NetstackImpl, error) { + ns, err := netstack.Create(logf, + sys.Tun.Get(), + sys.Engine.Get(), + sys.MagicSock.Get(), + sys.Dialer.Get(), + sys.DNSManager.Get(), + sys.ProxyMapper(), + ) + if err != nil { + return nil, err + } + // Only register debug info if we have a debug mux + if debugMux != nil { + expvar.Publish("netstack", ns.ExpVar()) + } + + sys.Set(ns) + ns.ProcessLocalIPs = onlyNetstack + ns.ProcessSubnets = onlyNetstack || handleSubnetsInNetstack() + + dialer := sys.Dialer.Get() // must be set by caller already + + if onlyNetstack { + e := sys.Engine.Get() + dialer.UseNetstackForIP = func(ip netip.Addr) bool { + _, ok := e.PeerForIP(ip) + return ok + } + dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { + // Note: don't just return ns.DialContextTCP or we'll return + // *gonet.TCPConn(nil) instead of a nil interface which trips up + // callers. + tcpConn, err := ns.DialContextTCP(ctx, dst) + if err != nil { + return nil, err + } + return tcpConn, nil + } + dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { + // Note: don't just return ns.DialContextUDP or we'll return + // *gonet.UDPConn(nil) instead of a nil interface which trips up + // callers. + udpConn, err := ns.DialContextUDP(ctx, dst) + if err != nil { + return nil, err + } + return udpConn, nil + } + } + + return ns, nil +} diff --git a/cmd/tailscaled/proxy.go b/cmd/tailscaled/proxy.go index a91c62bfa44ac..ea9f54a479dc5 100644 --- a/cmd/tailscaled/proxy.go +++ b/cmd/tailscaled/proxy.go @@ -1,7 +1,7 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -//go:build go1.19 +//go:build !ts_omit_outboundproxy // HTTP proxy code @@ -9,13 +9,107 @@ package main import ( "context" + "flag" "io" + "log" "net" "net/http" "net/http/httputil" "strings" + + "tailscale.com/feature" + "tailscale.com/net/proxymux" + "tailscale.com/net/socks5" + "tailscale.com/net/tsdial" + "tailscale.com/types/logger" ) +func init() { + hookRegisterOutboundProxyFlags.Set(registerOutboundProxyFlags) + hookOutboundProxyListen.Set(outboundProxyListen) +} + +func registerOutboundProxyFlags() { + flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`) + flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`) +} + +// outboundProxyListen creates listeners for local SOCKS and HTTP proxies, if +// the respective addresses are not empty. args.socksAddr and args.httpProxyAddr +// can be the same, in which case the SOCKS5 Listener will receive connections +// that look like they're speaking SOCKS and httpListener will receive +// everything else. +// +// socksListener and httpListener can be nil, if their respective addrs are +// empty. +// +// The returned func closes over those two (possibly nil) listeners and +// starts the respective servers on the listener when called. +func outboundProxyListen() proxyStartFunc { + socksAddr, httpAddr := args.socksAddr, args.httpProxyAddr + + if socksAddr == httpAddr && socksAddr != "" && !strings.HasSuffix(socksAddr, ":0") { + ln, err := net.Listen("tcp", socksAddr) + if err != nil { + log.Fatalf("proxy listener: %v", err) + } + return mkProxyStartFunc(proxymux.SplitSOCKSAndHTTP(ln)) + } + + var socksListener, httpListener net.Listener + var err error + if socksAddr != "" { + socksListener, err = net.Listen("tcp", socksAddr) + if err != nil { + log.Fatalf("SOCKS5 listener: %v", err) + } + if strings.HasSuffix(socksAddr, ":0") { + // Log kernel-selected port number so integration tests + // can find it portably. + log.Printf("SOCKS5 listening on %v", socksListener.Addr()) + } + } + if httpAddr != "" { + httpListener, err = net.Listen("tcp", httpAddr) + if err != nil { + log.Fatalf("HTTP proxy listener: %v", err) + } + if strings.HasSuffix(httpAddr, ":0") { + // Log kernel-selected port number so integration tests + // can find it portably. + log.Printf("HTTP proxy listening on %v", httpListener.Addr()) + } + } + + return mkProxyStartFunc(socksListener, httpListener) +} + +func mkProxyStartFunc(socksListener, httpListener net.Listener) proxyStartFunc { + return func(logf logger.Logf, dialer *tsdial.Dialer) { + var addrs []string + if httpListener != nil { + hs := &http.Server{Handler: httpProxyHandler(dialer.UserDial)} + go func() { + log.Fatalf("HTTP proxy exited: %v", hs.Serve(httpListener)) + }() + addrs = append(addrs, httpListener.Addr().String()) + } + if socksListener != nil { + ss := &socks5.Server{ + Logf: logger.WithPrefix(logf, "socks5: "), + Dialer: dialer.UserDial, + } + go func() { + log.Fatalf("SOCKS5 server exited: %v", ss.Serve(socksListener)) + }() + addrs = append(addrs, socksListener.Addr().String()) + } + if set, ok := feature.HookProxySetSelfProxy.GetOk(); ok { + set(addrs...) + } + } +} + // httpProxyHandler returns an HTTP proxy http.Handler using the // provided backend dialer. func httpProxyHandler(dialer func(ctx context.Context, netw, addr string) (net.Conn, error)) http.Handler { diff --git a/cmd/tailscaled/required_version.go b/cmd/tailscaled/required_version.go index 3acb3d52e4d8c..bfde77cd8474b 100644 --- a/cmd/tailscaled/required_version.go +++ b/cmd/tailscaled/required_version.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !go1.23 diff --git a/cmd/tailscaled/sigpipe.go b/cmd/tailscaled/sigpipe.go index 2fcdab2a4660e..ba69fcd2a0632 100644 --- a/cmd/tailscaled/sigpipe.go +++ b/cmd/tailscaled/sigpipe.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.21 && !plan9 diff --git a/cmd/tailscaled/ssh.go b/cmd/tailscaled/ssh.go index 59a1ddd0df461..8de3117944431 100644 --- a/cmd/tailscaled/ssh.go +++ b/cmd/tailscaled/ssh.go @@ -1,9 +1,9 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build (linux || darwin || freebsd || openbsd || plan9) && !ts_omit_ssh package main -// Force registration of tailssh with LocalBackend. -import _ "tailscale.com/ssh/tailssh" +// Register implementations of various SSH hooks. +import _ "tailscale.com/feature/ssh" diff --git a/cmd/tailscaled/tailscale-online.target b/cmd/tailscaled/tailscale-online.target new file mode 100644 index 0000000000000..a8ee7db475378 --- /dev/null +++ b/cmd/tailscaled/tailscale-online.target @@ -0,0 +1,4 @@ +[Unit] +Description=Tailscale is online +Requires=tailscale-wait-online.service +After=tailscale-wait-online.service diff --git a/cmd/tailscaled/tailscale-wait-online.service b/cmd/tailscaled/tailscale-wait-online.service new file mode 100644 index 0000000000000..eb46a18bf92d2 --- /dev/null +++ b/cmd/tailscaled/tailscale-wait-online.service @@ -0,0 +1,12 @@ +[Unit] +Description=Wait for Tailscale to be online +After=tailscaled.service +Requires=tailscaled.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/tailscale wait +RemainAfterExit=yes + +[Install] +WantedBy=tailscale-online.target diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 61b811c129454..69f4ff5bc43ed 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.23 @@ -13,14 +13,11 @@ package main // import "tailscale.com/cmd/tailscaled" import ( "context" "errors" - "expvar" "flag" "fmt" "log" "net" "net/http" - "net/http/pprof" - "net/netip" "os" "os/signal" "path/filepath" @@ -30,44 +27,42 @@ import ( "syscall" "time" - "tailscale.com/client/local" "tailscale.com/cmd/tailscaled/childproc" "tailscale.com/control/controlclient" - "tailscale.com/drive/driveimpl" "tailscale.com/envknob" + "tailscale.com/feature" + "tailscale.com/feature/buildfeatures" _ "tailscale.com/feature/condregister" + "tailscale.com/health" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/conffile" "tailscale.com/ipn/ipnlocal" "tailscale.com/ipn/ipnserver" "tailscale.com/ipn/store" + "tailscale.com/ipn/store/mem" "tailscale.com/logpolicy" "tailscale.com/logtail" "tailscale.com/net/dns" "tailscale.com/net/dnsfallback" "tailscale.com/net/netmon" "tailscale.com/net/netns" - "tailscale.com/net/proxymux" - "tailscale.com/net/socks5" "tailscale.com/net/tsdial" - "tailscale.com/net/tshttpproxy" "tailscale.com/net/tstun" "tailscale.com/paths" "tailscale.com/safesocket" "tailscale.com/syncs" "tailscale.com/tsd" - "tailscale.com/tsweb/varz" "tailscale.com/types/flagtype" + "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/logid" - "tailscale.com/util/clientmetric" - "tailscale.com/util/multierr" "tailscale.com/util/osshare" + "tailscale.com/util/syspolicy/pkey" + "tailscale.com/util/syspolicy/policyclient" "tailscale.com/version" "tailscale.com/version/distro" "tailscale.com/wgengine" - "tailscale.com/wgengine/netstack" "tailscale.com/wgengine/router" ) @@ -87,13 +82,21 @@ func defaultTunName() string { case "aix", "solaris", "illumos": return "userspace-networking" case "linux": - switch distro.Get() { - case distro.Synology: + if buildfeatures.HasSynology && buildfeatures.HasNetstack && distro.Get() == distro.Synology { // Try TUN, but fall back to userspace networking if needed. // See https://github.com/tailscale/tailscale-synology/issues/35 return "tailscale0,userspace-networking" } - + if buildfeatures.HasNetstack && distro.Get() == distro.Crostini { + // cros-garcon NULL-derefs on cold-boot netlink interface + // enumeration when tailscale0 is present, preventing the + // Crostini container and ChromeOS Terminal from starting + // cleanly. Default to userspace-networking until the + // upstream ChromiumOS bug is fixed. + // See https://github.com/tailscale/tailscale/issues/12090 + // See https://issuetracker.google.com/issues/517069318 + return "userspace-networking" + } } return "tailscale0" } @@ -121,18 +124,20 @@ var args struct { // or comma-separated list thereof. tunname string - cleanUp bool - confFile string // empty, file path, or "vm:user-data" - debug string - port uint16 - statepath string - statedir string - socketpath string - birdSocketPath string - verbose int - socksAddr string // listen address for SOCKS5 server - httpProxyAddr string // listen address for HTTP proxy server - disableLogs bool + cleanUp bool + confFile string // empty, file path, or "vm:user-data" + debug string + port uint16 + statepath string + encryptState boolFlag + statedir string + socketpath string + birdSocketPath string + verbose int + socksAddr string // listen address for SOCKS5 server + httpProxyAddr string // listen address for HTTP proxy server + disableLogs bool + hardwareAttestation boolFlag } var ( @@ -148,9 +153,7 @@ var ( var subCommands = map[string]*func([]string) error{ "install-system-daemon": &installSystemDaemon, "uninstall-system-daemon": &uninstallSystemDaemon, - "debug": &debugModeFunc, "be-child": &beChildFunc, - "serve-taildrive": &serveDriveFunc, } var beCLI func() // non-nil if CLI is linked in with the "ts_include_cli" build tag @@ -174,6 +177,17 @@ func shouldRunCLI() bool { return false } +// Outbound Proxy hooks +var ( + hookRegisterOutboundProxyFlags feature.Hook[func()] + hookOutboundProxyListen feature.Hook[func() proxyStartFunc] +) + +// proxyStartFunc is the type of the function returned by +// outboundProxyListen, to start the servers on the Listeners +// started by hookOutboundProxyListen. +type proxyStartFunc = func(logf logger.Logf, dialer *tsdial.Dialer) + func main() { envknob.PanicIfAnyEnvCheckedInInit() if shouldRunCLI() { @@ -187,18 +201,32 @@ func main() { printVersion := false flag.IntVar(&args.verbose, "verbose", defaultVerbosity(), "log verbosity level; 0 is default, 1 or higher are increasingly verbose") flag.BoolVar(&args.cleanUp, "cleanup", false, "clean up system state and exit") - flag.StringVar(&args.debug, "debug", "", "listen address ([ip]:port) of optional debug server") - flag.StringVar(&args.socksAddr, "socks5-server", "", `optional [ip]:port to run a SOCK5 server (e.g. "localhost:1080")`) - flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`) + if buildfeatures.HasDebug { + flag.StringVar(&args.debug, "debug", "", "listen address ([ip]:port) of optional debug server") + } flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`) flag.Var(flagtype.PortValue(&args.port, defaultPort()), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select") flag.StringVar(&args.statepath, "state", "", "absolute path of state file; use 'kube:' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM; use 'mem:' to not store state and register as an ephemeral node. If empty and --statedir is provided, the default is /tailscaled.state. Default: "+paths.DefaultTailscaledStateFile()) + if buildfeatures.HasTPM { + flag.Var(&args.encryptState, "encrypt-state", `encrypt the state file on disk; when not set encryption will be enabled if supported on this platform; uses TPM on Linux and Windows, on all other platforms this flag is not supported`) + } flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.") flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket") - flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket") + if buildfeatures.HasBird { + flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket") + } flag.BoolVar(&printVersion, "version", false, "print version information and exit") flag.BoolVar(&args.disableLogs, "no-logs-no-support", false, "disable log uploads; this also disables any technical support") flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2)") + if buildfeatures.HasTPM { + flag.Var(&args.hardwareAttestation, "hardware-attestation", `use hardware-backed keys to bind node identity to this device when supported +by the OS and hardware. Uses TPM 2.0 on Linux and Windows; SecureEnclave on +macOS and iOS; and Keystore on Android. Only supported for Tailscale nodes that +store state on filesystem.`) + } + if f, ok := hookRegisterOutboundProxyFlags.GetOk(); ok { + f() + } if runtime.GOOS == "plan9" && os.Getenv("_NETSHELL_CHILD_") != "" { os.Args = []string{"tailscaled", "be-child", "plan9-netshell"} @@ -246,7 +274,7 @@ func main() { log.Fatalf("--socket is required") } - if args.birdSocketPath != "" && createBIRDClient == nil { + if buildfeatures.HasBird && args.birdSocketPath != "" && createBIRDClient == nil { log.SetFlags(0) log.Fatalf("--bird-socket is not supported on %s", runtime.GOOS) } @@ -254,20 +282,23 @@ func main() { // Only apply a default statepath when neither have been provided, so that a // user may specify only --statedir if they wish. if args.statepath == "" && args.statedir == "" { - if runtime.GOOS == "plan9" { - home, err := os.UserHomeDir() - if err != nil { - log.Fatalf("failed to get home directory: %v", err) - } - args.statedir = filepath.Join(home, "tailscale-state") - if err := os.MkdirAll(args.statedir, 0700); err != nil { - log.Fatalf("failed to create state directory: %v", err) + if paths.MakeAutomaticStateDir() { + d := paths.DefaultTailscaledStateDir() + if d != "" { + args.statedir = d + if err := os.MkdirAll(d, 0700); err != nil { + log.Fatalf("failed to create state directory: %v", err) + } } } else { args.statepath = paths.DefaultTailscaledStateFile() } } + if buildfeatures.HasTPM { + handleTPMFlags() + } + if args.disableLogs { envknob.SetNoLogsNoSupport() } @@ -278,8 +309,10 @@ func main() { err := run() - // Remove file sharing from Windows shell (noop in non-windows) - osshare.SetFileSharingEnabled(false, logger.Discard) + if buildfeatures.HasTaildrop { + // Remove file sharing from Windows shell (noop in non-windows) + osshare.SetFileSharingEnabled(false, logger.Discard) + } if err != nil { log.Fatal(err) @@ -315,13 +348,17 @@ func trySynologyMigration(p string) error { } func statePathOrDefault() string { + var path string if args.statepath != "" { - return args.statepath + path = args.statepath + } + if path == "" && args.statedir != "" { + path = filepath.Join(args.statedir, "tailscaled.state") } - if args.statedir != "" { - return filepath.Join(args.statedir, "tailscaled.state") + if path != "" && !store.HasKnownProviderPrefix(path) && args.encryptState.v { + path = store.TPMPrefix + path } - return "" + return path } // serverOptions is the configuration of the Tailscale node agent. @@ -368,7 +405,7 @@ func ipnServerOpts() (o serverOptions) { return o } -var logPol *logpolicy.Policy +var logPol *logpolicy.Policy // or nil if not used var debugMux *http.ServeMux func run() (err error) { @@ -377,6 +414,7 @@ func run() (err error) { // Install an event bus as early as possible, so that it's // available universally when setting up everything else. sys := tsd.NewSystem() + sys.SocketPath = args.socketpath // Parse config, if specified, to fail early if it's invalid. var conf *conffile.Config @@ -398,15 +436,29 @@ func run() (err error) { sys.Set(netMon) } - pol := logpolicy.New(logtail.CollectionNode, netMon, sys.HealthTracker(), nil /* use log.Printf */) - pol.SetVerbosityLevel(args.verbose) - logPol = pol - defer func() { - // Finish uploading logs after closing everything else. - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - pol.Shutdown(ctx) - }() + var publicLogID logid.PublicID + if buildfeatures.HasLogTail { + logpolicy.GetLogTarget.Set(func() string { + target, _ := sys.PolicyClientOrDefault().GetString(pkey.LogTarget, "") + return target + }) + + pol := logpolicy.Options{ + Collection: logtail.CollectionNode, + NetMon: netMon, + Health: sys.HealthTracker.Get(), + Bus: sys.Bus.Get(), + }.New() + pol.SetVerbosityLevel(args.verbose) + publicLogID = pol.PublicID + logPol = pol + defer func() { + // Finish uploading logs after closing everything else. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + pol.Shutdown(ctx) + }() + } if err := envknob.ApplyDiskConfigError(); err != nil { log.Printf("Error reading environment config: %v", err) @@ -415,16 +467,13 @@ func run() (err error) { if isWinSvc { // Run the IPN server from the Windows service manager. log.Printf("Running service...") - if err := runWindowsService(pol); err != nil { + if err := runWindowsService(logPol); err != nil { log.Printf("runservice: %v", err) } log.Printf("Service ended.") return nil } - if envknob.Bool("TS_DEBUG_MEMORY") { - logf = logger.RusagePrefixLog(logf) - } logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100) if envknob.Bool("TS_PLEASE_PANIC") { @@ -433,7 +482,7 @@ func run() (err error) { // Always clean up, even if we're going to run the server. This covers cases // such as when a system was rebooted without shutting down, or tailscaled // crashed, and would for example restore system DNS configuration. - dns.CleanUp(logf, netMon, sys.HealthTracker(), args.tunname) + dns.CleanUp(logf, netMon, sys.Bus.Get(), sys.HealthTracker.Get(), args.tunname) router.CleanUp(logf, netMon, args.tunname) // If the cleanUp flag was passed, then exit. if args.cleanUp { @@ -447,21 +496,29 @@ func run() (err error) { log.Printf("error in synology migration: %v", err) } - if args.debug != "" { - debugMux = newDebugMux() + if buildfeatures.HasDebug && args.debug != "" { + debugMux = hookNewDebugMux.Get()() } - sys.Set(driveimpl.NewFileSystemForRemote(logf)) + if f, ok := hookSetSysDrive.GetOk(); ok { + f(sys, logf) + } if app := envknob.App(); app != "" { hostinfo.SetApp(app) } - return startIPNServer(context.Background(), logf, pol.PublicID, sys) + return startIPNServer(context.Background(), logf, publicLogID, sys) } +var ( + hookSetSysDrive feature.Hook[func(*tsd.System, logger.Logf)] + hookSetWgEnginConfigDrive feature.Hook[func(*wgengine.Config, logger.Logf)] +) + var sigPipe os.Signal // set by sigpipe.go +// logID may be the zero value if logging is not in use. func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID, sys *tsd.System) error { ln, err := safesocket.Listen(args.socketpath) if err != nil { @@ -503,8 +560,8 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID, } }() - srv := ipnserver.New(logf, logID, sys.NetMon.Get()) - if debugMux != nil { + srv := ipnserver.New(logf, logID, sys.Bus.Get(), sys.NetMon.Get()) + if buildfeatures.HasDebug && debugMux != nil { debugMux.HandleFunc("/debug/ipn", srv.ServeHTMLStatus) } var lbErr syncs.AtomicValue[error] @@ -555,89 +612,65 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID, return nil } +var ( + hookNewNetstack feature.Hook[func(_ logger.Logf, _ *tsd.System, onlyNetstack bool) (tsd.NetstackImpl, error)] +) + +// logID may be the zero value if logging is not in use. func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID, sys *tsd.System) (_ *ipnlocal.LocalBackend, retErr error) { if logPol != nil { logPol.Logtail.SetNetMon(sys.NetMon.Get()) } - socksListener, httpProxyListener := mustStartProxyListeners(args.socksAddr, args.httpProxyAddr) + var startProxy proxyStartFunc + if listen, ok := hookOutboundProxyListen.GetOk(); ok { + startProxy = listen() + } dialer := &tsdial.Dialer{Logf: logf} // mutated below (before used) + dialer.SetBus(sys.Bus.Get()) sys.Set(dialer) onlyNetstack, err := createEngine(logf, sys) if err != nil { return nil, fmt.Errorf("createEngine: %w", err) } - if debugMux != nil { + if onlyNetstack && !buildfeatures.HasNetstack { + return nil, errors.New("userspace-networking support is not compiled in to this binary") + } + if buildfeatures.HasDebug && debugMux != nil { if ms, ok := sys.MagicSock.GetOK(); ok { debugMux.HandleFunc("/debug/magicsock", ms.ServeHTTPDebug) } go runDebugServer(logf, debugMux, args.debug) } - ns, err := newNetstack(logf, sys) - if err != nil { - return nil, fmt.Errorf("newNetstack: %w", err) + var ns tsd.NetstackImpl // or nil if not linked in + if newNetstack, ok := hookNewNetstack.GetOk(); ok { + ns, err = newNetstack(logf, sys, onlyNetstack) + if err != nil { + return nil, fmt.Errorf("newNetstack: %w", err) + } } - sys.Set(ns) - ns.ProcessLocalIPs = onlyNetstack - ns.ProcessSubnets = onlyNetstack || handleSubnetsInNetstack() - if onlyNetstack { - e := sys.Engine.Get() - dialer.UseNetstackForIP = func(ip netip.Addr) bool { - _, ok := e.PeerForIP(ip) - return ok - } - dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { - // Note: don't just return ns.DialContextTCP or we'll return - // *gonet.TCPConn(nil) instead of a nil interface which trips up - // callers. - tcpConn, err := ns.DialContextTCP(ctx, dst) - if err != nil { - return nil, err - } - return tcpConn, nil - } - dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { - // Note: don't just return ns.DialContextUDP or we'll return - // *gonet.UDPConn(nil) instead of a nil interface which trips up - // callers. - udpConn, err := ns.DialContextUDP(ctx, dst) - if err != nil { - return nil, err - } - return udpConn, nil - } - } - if socksListener != nil || httpProxyListener != nil { - var addrs []string - if httpProxyListener != nil { - hs := &http.Server{Handler: httpProxyHandler(dialer.UserDial)} - go func() { - log.Fatalf("HTTP proxy exited: %v", hs.Serve(httpProxyListener)) - }() - addrs = append(addrs, httpProxyListener.Addr().String()) - } - if socksListener != nil { - ss := &socks5.Server{ - Logf: logger.WithPrefix(logf, "socks5: "), - Dialer: dialer.UserDial, - } - go func() { - log.Fatalf("SOCKS5 server exited: %v", ss.Serve(socksListener)) - }() - addrs = append(addrs, socksListener.Addr().String()) - } - tshttpproxy.SetSelfProxy(addrs...) + if startProxy != nil { + go startProxy(logf, dialer) } opts := ipnServerOpts() store, err := store.New(logf, statePathOrDefault()) if err != nil { - return nil, fmt.Errorf("store.New: %w", err) + // If we can't create the store (for example if it's TPM-sealed and the + // TPM is reset), create a dummy in-memory store to propagate the error + // to the user. + ht, ok := sys.HealthTracker.GetOK() + if !ok { + return nil, fmt.Errorf("store.New: %w", err) + } + logf("store.New failed: %v; starting with in-memory store with a health warning", err) + store = new(mem.Store) + ht.SetUnhealthy(ipn.StateStoreHealth, health.Args{health.ArgError: err.Error()}) } sys.Set(store) @@ -656,16 +689,23 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID if root := lb.TailscaleVarRoot(); root != "" { dnsfallback.SetCachePath(filepath.Join(root, "derpmap.cached.json"), logf) } - lb.ConfigureWebClient(&local.Client{ - Socket: args.socketpath, - UseSocketOnly: args.socketpath != paths.DefaultTailscaledSocket(), - }) - if err := ns.Start(lb); err != nil { - log.Fatalf("failed to start netstack: %v", err) + if f, ok := hookConfigureWebClient.GetOk(); ok { + f(lb) + } + + if ns != nil { + if err := ns.Start(lb); err != nil { + log.Fatalf("failed to start netstack: %v", err) + } + } + if buildfeatures.HasTPM && args.hardwareAttestation.v { + lb.SetHardwareAttested() } return lb, nil } +var hookConfigureWebClient feature.Hook[func(*ipnlocal.LocalBackend)] + // createEngine tries to the wgengine.Engine based on the order of tunnels // specified in the command line flags. // @@ -685,7 +725,7 @@ func createEngine(logf logger.Logf, sys *tsd.System) (onlyNetstack bool, err err logf("wgengine.NewUserspaceEngine(tun %q) error: %v", name, err) errs = append(errs, err) } - return false, multierr.New(errs...) + return false, errors.Join(errs...) } // handleSubnetsInNetstack reports whether netstack should handle subnet routers @@ -714,16 +754,19 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo conf := wgengine.Config{ ListenPort: args.port, NetMon: sys.NetMon.Get(), - HealthTracker: sys.HealthTracker(), + HealthTracker: sys.HealthTracker.Get(), + ExtraRootCAs: sys.ExtraRootCAs, Metrics: sys.UserMetricsRegistry(), Dialer: sys.Dialer.Get(), SetSubsystem: sys.Set, ControlKnobs: sys.ControlKnobs(), EventBus: sys.Bus.Get(), - DriveForLocal: driveimpl.NewFileSystemForLocal(logf), + } + if f, ok := hookSetWgEnginConfigDrive.GetOk(); ok { + f(&conf, logf) } - sys.HealthTracker().SetMetricsRegistry(sys.UserMetricsRegistry()) + sys.HealthTracker.Get().SetMetricsRegistry(sys.UserMetricsRegistry()) onlyNetstack = name == "userspace-networking" netstackSubnetRouter := onlyNetstack // but mutated later on some platforms @@ -744,7 +787,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo // configuration being unavailable (from the noop // manager). More in Issue 4017. // TODO(bradfitz): add a Synology-specific DNS manager. - conf.DNS, err = dns.NewOSConfigurator(logf, sys.HealthTracker(), sys.ControlKnobs(), "") // empty interface name + conf.DNS, err = dns.NewOSConfigurator(logf, sys.HealthTracker.Get(), sys.Bus.Get(), sys.PolicyClientOrDefault(), sys.ControlKnobs(), "") // empty interface name if err != nil { return false, fmt.Errorf("dns.NewOSConfigurator: %w", err) } @@ -768,17 +811,18 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo if runtime.GOOS == "plan9" { // TODO(bradfitz): why don't we do this on all platforms? + // TODO(barnstar): we do it on sandboxed darwin now // We should. Doing it just on plan9 for now conservatively. - sys.NetMon.Get().SetTailscaleInterfaceName(devName) + netmon.SetTailscaleInterfaceProps(devName, 0) } - r, err := router.New(logf, dev, sys.NetMon.Get(), sys.HealthTracker()) + r, err := router.New(logf, dev, sys.NetMon.Get(), sys.HealthTracker.Get(), sys.Bus.Get()) if err != nil { dev.Close() return false, fmt.Errorf("creating router: %w", err) } - d, err := dns.NewOSConfigurator(logf, sys.HealthTracker(), sys.ControlKnobs(), devName) + d, err := dns.NewOSConfigurator(logf, sys.HealthTracker.Get(), sys.Bus.Get(), sys.PolicyClientOrDefault(), sys.ControlKnobs(), devName) if err != nil { dev.Close() r.Close() @@ -795,31 +839,18 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo if err != nil { return onlyNetstack, err } - e = wgengine.NewWatchdog(e) sys.Set(e) sys.NetstackRouter.Set(netstackSubnetRouter) return onlyNetstack, nil } -func newDebugMux() *http.ServeMux { - mux := http.NewServeMux() - mux.HandleFunc("/debug/metrics", servePrometheusMetrics) - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - return mux -} - -func servePrometheusMetrics(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - varz.Handler(w, r) - clientmetric.WritePrometheusExpositionFormat(w) -} +var hookNewDebugMux feature.Hook[func() *http.ServeMux] func runDebugServer(logf logger.Logf, mux *http.ServeMux, addr string) { + if !buildfeatures.HasDebug { + return + } ln, err := net.Listen("tcp", addr) if err != nil { log.Fatalf("debug server: %v", err) @@ -837,69 +868,6 @@ func runDebugServer(logf logger.Logf, mux *http.ServeMux, addr string) { } } -func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) { - ret, err := netstack.Create(logf, - sys.Tun.Get(), - sys.Engine.Get(), - sys.MagicSock.Get(), - sys.Dialer.Get(), - sys.DNSManager.Get(), - sys.ProxyMapper(), - ) - if err != nil { - return nil, err - } - // Only register debug info if we have a debug mux - if debugMux != nil { - expvar.Publish("netstack", ret.ExpVar()) - } - return ret, nil -} - -// mustStartProxyListeners creates listeners for local SOCKS and HTTP -// proxies, if the respective addresses are not empty. socksAddr and -// httpAddr can be the same, in which case socksListener will receive -// connections that look like they're speaking SOCKS and httpListener -// will receive everything else. -// -// socksListener and httpListener can be nil, if their respective -// addrs are empty. -func mustStartProxyListeners(socksAddr, httpAddr string) (socksListener, httpListener net.Listener) { - if socksAddr == httpAddr && socksAddr != "" && !strings.HasSuffix(socksAddr, ":0") { - ln, err := net.Listen("tcp", socksAddr) - if err != nil { - log.Fatalf("proxy listener: %v", err) - } - return proxymux.SplitSOCKSAndHTTP(ln) - } - - var err error - if socksAddr != "" { - socksListener, err = net.Listen("tcp", socksAddr) - if err != nil { - log.Fatalf("SOCKS5 listener: %v", err) - } - if strings.HasSuffix(socksAddr, ":0") { - // Log kernel-selected port number so integration tests - // can find it portably. - log.Printf("SOCKS5 listening on %v", socksListener.Addr()) - } - } - if httpAddr != "" { - httpListener, err = net.Listen("tcp", httpAddr) - if err != nil { - log.Fatalf("HTTP proxy listener: %v", err) - } - if strings.HasSuffix(httpAddr, ":0") { - // Log kernel-selected port number so integration tests - // can find it portably. - log.Printf("HTTP proxy listening on %v", httpListener.Addr()) - } - } - - return socksListener, httpListener -} - var beChildFunc = beChild func beChild(args []string) error { @@ -914,35 +882,6 @@ func beChild(args []string) error { return f(args[1:]) } -var serveDriveFunc = serveDrive - -// serveDrive serves one or more Taildrives on localhost using the WebDAV -// protocol. On UNIX and MacOS tailscaled environment, Taildrive spawns child -// tailscaled processes in serve-taildrive mode in order to access the fliesystem -// as specific (usually unprivileged) users. -// -// serveDrive prints the address on which it's listening to stdout so that the -// parent process knows where to connect to. -func serveDrive(args []string) error { - if len(args) == 0 { - return errors.New("missing shares") - } - if len(args)%2 != 0 { - return errors.New("need pairs") - } - s, err := driveimpl.NewFileServer() - if err != nil { - return fmt.Errorf("unable to start Taildrive file server: %v", err) - } - shares := make(map[string]string) - for i := 0; i < len(args); i += 2 { - shares[args[i]] = args[i+1] - } - s.SetShares(shares) - fmt.Printf("%v\n", s.Addr()) - return s.Serve() -} - // dieOnPipeReadErrorOfFD reads from the pipe named by fd and exit the process // when the pipe becomes readable. We use this in tests as a somewhat more // portable mechanism for the Linux PR_SET_PDEATHSIG, which we wish existed on @@ -974,3 +913,95 @@ func applyIntegrationTestEnvKnob() { } } } + +// handleTPMFlags validates the --encrypt-state and --hardware-attestation flags +// if set, and defaults both to on if supported and compatible with other +// settings. +func handleTPMFlags() { + switch { + case args.hardwareAttestation.v: + if err := canUseHardwareAttestation(); err != nil { + log.SetFlags(0) + log.Fatal(err) + } + case !args.hardwareAttestation.set: + policyHWAttestation, _ := policyclient.Get().GetBoolean(pkey.HardwareAttestation, false) + if err := canUseHardwareAttestation(); err != nil { + if policyHWAttestation { + log.Printf("[unexpected] policy requires hardware attestation, but device does not support it: %v", err) + } + args.hardwareAttestation.v = false + } else { + args.hardwareAttestation.v = policyHWAttestation + } + } + + switch { + case args.encryptState.v: + // Explicitly enabled, validate. + if err := canEncryptState(); err != nil { + log.SetFlags(0) + log.Fatal(err) + } + case !args.encryptState.set: + policyEncrypt, _ := policyclient.Get().GetBoolean(pkey.EncryptState, false) + if err := canEncryptState(); policyEncrypt && err == nil { + args.encryptState.v = true + } + } +} + +// canUseHardwareAttestation returns an error if hardware attestation can't be +// enabled, either due to availability or compatibility with other settings. +func canUseHardwareAttestation() error { + if _, err := key.NewEmptyHardwareAttestationKey(); err == key.ErrUnsupported { + return errors.New("--hardware-attestation is not supported on this platform or in this build of tailscaled") + } + // Hardware attestation keys are TPM-bound and cannot be migrated between + // machines. Disable when using portable state stores like kube: or arn: + // where state may be loaded on a different machine. + if args.statepath != "" && isPortableStore(args.statepath) { + return errors.New("--hardware-attestation cannot be used with portable state stores (kube:, arn:) because TPM-bound keys cannot be migrated between machines") + } + return nil +} + +// isPortableStore reports whether the given state path refers to a portable +// state store where state may be loaded on different machines. +// All stores apart from file store and TPM store are portable. +func isPortableStore(path string) bool { + if store.HasKnownProviderPrefix(path) && !strings.HasPrefix(path, store.TPMPrefix) { + return true + } + // In most cases Kubernetes Secret and AWS SSM stores would have been caught + // by the earlier check - but that check relies on those stores having been + // registered. This additional check is here to ensure that if we ever + // produce a faulty build that failed to register some store, users who + // upgraded to that don't get hardware keys generated. + if strings.HasPrefix(path, "kube:") || strings.HasPrefix(path, "arn:") { + return true + } + return false +} + +// canEncryptState returns an error if state encryption can't be enabled, +// either due to availability or compatibility with other settings. +func canEncryptState() error { + if runtime.GOOS != "windows" && runtime.GOOS != "linux" { + // TPM encryption is only configurable on Windows and Linux. Other + // platforms either use system APIs and are not configurable + // (Android/Apple), or don't support any form of encryption yet + // (plan9/FreeBSD/etc). + return fmt.Errorf("--encrypt-state is not supported on %s", runtime.GOOS) + } + // Check if we have TPM access. + if !feature.TPMAvailable() { + return errors.New("--encrypt-state is not supported on this device or a TPM is not accessible") + } + // Check for conflicting prefix in --state, like arn: or kube:. + if args.statepath != "" && store.HasKnownProviderPrefix(args.statepath) { + return errors.New("--encrypt-state can only be used with --state set to a local file path") + } + + return nil +} diff --git a/cmd/tailscaled/tailscaled.service b/cmd/tailscaled/tailscaled.service index 719a3c0c96398..9950891a35fa4 100644 --- a/cmd/tailscaled/tailscaled.service +++ b/cmd/tailscaled/tailscaled.service @@ -1,6 +1,6 @@ [Unit] Description=Tailscale node agent -Documentation=https://tailscale.com/kb/ +Documentation=https://tailscale.com/docs/ Wants=network-pre.target After=network-pre.target NetworkManager.service systemd-resolved.service diff --git a/cmd/tailscaled/tailscaled_bird.go b/cmd/tailscaled/tailscaled_bird.go index c76f77bec6e36..c1c32d2bb493d 100644 --- a/cmd/tailscaled/tailscaled_bird.go +++ b/cmd/tailscaled/tailscaled_bird.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 && (linux || darwin || freebsd || openbsd) && !ts_omit_bird diff --git a/cmd/tailscaled/tailscaled_drive.go b/cmd/tailscaled/tailscaled_drive.go new file mode 100644 index 0000000000000..6a8590bb82217 --- /dev/null +++ b/cmd/tailscaled/tailscaled_drive.go @@ -0,0 +1,56 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_drive + +package main + +import ( + "errors" + "fmt" + + "tailscale.com/drive/driveimpl" + "tailscale.com/tsd" + "tailscale.com/types/logger" + "tailscale.com/wgengine" +) + +func init() { + subCommands["serve-taildrive"] = &serveDriveFunc + + hookSetSysDrive.Set(func(sys *tsd.System, logf logger.Logf) { + sys.Set(driveimpl.NewFileSystemForRemote(logf)) + }) + hookSetWgEnginConfigDrive.Set(func(conf *wgengine.Config, logf logger.Logf) { + conf.DriveForLocal = driveimpl.NewFileSystemForLocal(logf) + }) +} + +var serveDriveFunc = serveDrive + +// serveDrive serves one or more Taildrives on localhost using the WebDAV +// protocol. On UNIX and MacOS tailscaled environment, Taildrive spawns child +// tailscaled processes in serve-taildrive mode in order to access the fliesystem +// as specific (usually unprivileged) users. +// +// serveDrive prints the address on which it's listening to stdout so that the +// parent process knows where to connect to. +func serveDrive(args []string) error { + if len(args) == 0 { + return errors.New("missing shares") + } + if len(args)%2 != 0 { + return errors.New("need pairs") + } + s, err := driveimpl.NewFileServer() + if err != nil { + return fmt.Errorf("unable to start Taildrive file server: %v", err) + } + shares := make(map[string]string) + for i := 0; i < len(args); i += 2 { + shares[args[i]] = args[i+1] + } + s.SetShares(shares) + fmt.Printf("%v\n", s.Addr()) + return s.Serve() +} diff --git a/cmd/tailscaled/tailscaled_notwindows.go b/cmd/tailscaled/tailscaled_notwindows.go index d5361cf286d3d..735facc37b861 100644 --- a/cmd/tailscaled/tailscaled_notwindows.go +++ b/cmd/tailscaled/tailscaled_notwindows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !windows && go1.19 diff --git a/cmd/tailscaled/tailscaled_test.go b/cmd/tailscaled/tailscaled_test.go index c50c237591170..ab6482293687b 100644 --- a/cmd/tailscaled/tailscaled_test.go +++ b/cmd/tailscaled/tailscaled_test.go @@ -1,12 +1,20 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main // import "tailscale.com/cmd/tailscaled" import ( + "os" + "strings" "testing" + "tailscale.com/envknob" + "tailscale.com/ipn" + "tailscale.com/net/netmon" + "tailscale.com/tsd" "tailscale.com/tstest/deptest" + "tailscale.com/types/logid" + "tailscale.com/util/must" ) func TestNothing(t *testing.T) { @@ -38,3 +46,98 @@ func TestDeps(t *testing.T) { }, }.Check(t) } + +func TestStateStoreError(t *testing.T) { + logID, err := logid.NewPrivateID() + if err != nil { + t.Fatal(err) + } + // Don't upload any logs from tests. + envknob.SetNoLogsNoSupport() + + args.statedir = t.TempDir() + args.tunname = "userspace-networking" + + t.Run("new-state", func(t *testing.T) { + sys := tsd.NewSystem() + sys.NetMon.Set(must.Get(netmon.New(sys.Bus.Get(), t.Logf))) + lb, err := getLocalBackend(t.Context(), t.Logf, logID.Public(), sys) + if err != nil { + t.Fatal(err) + } + defer lb.Shutdown() + if lb.HealthTracker().IsUnhealthy(ipn.StateStoreHealth) { + t.Errorf("StateStoreHealth is unhealthy on fresh LocalBackend:\n%s", strings.Join(lb.HealthTracker().Strings(), "\n")) + } + }) + t.Run("corrupt-state", func(t *testing.T) { + sys := tsd.NewSystem() + sys.NetMon.Set(must.Get(netmon.New(sys.Bus.Get(), t.Logf))) + // Populate the state file with something that will fail to parse to + // trigger an error from store.New. + if err := os.WriteFile(statePathOrDefault(), []byte("bad json"), 0644); err != nil { + t.Fatal(err) + } + lb, err := getLocalBackend(t.Context(), t.Logf, logID.Public(), sys) + if err != nil { + t.Fatal(err) + } + defer lb.Shutdown() + if !lb.HealthTracker().IsUnhealthy(ipn.StateStoreHealth) { + t.Errorf("StateStoreHealth is healthy when state file is corrupt") + } + }) +} + +func TestIsPortableStore(t *testing.T) { + tests := []struct { + name string + path string + want bool + }{ + { + name: "kube_store", + path: "kube:my-secret", + want: true, + }, + { + name: "aws_arn_store", + path: "arn:aws:ssm:us-east-1:123456789012:parameter/tailscale/state", + want: true, + }, + { + name: "tpm_store", + path: "tpmseal:/var/lib/tailscale/tailscaled.state", + want: false, + }, + { + name: "local_file_store", + path: "/var/lib/tailscale/tailscaled.state", + want: false, + }, + { + name: "empty_path", + path: "", + want: false, + }, + { + name: "mem_store", + path: "mem:", + want: true, + }, + { + name: "windows_file_store", + path: `C:\ProgramData\Tailscale\server-state.conf`, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isPortableStore(tt.path) + if got != tt.want { + t.Errorf("isPortableStore(%q) = %v, want %v", tt.path, got, tt.want) + } + }) + } +} diff --git a/cmd/tailscaled/tailscaled_windows.go b/cmd/tailscaled/tailscaled_windows.go index 1b50688922968..0ad550d4cc0cd 100644 --- a/cmd/tailscaled/tailscaled_windows.go +++ b/cmd/tailscaled/tailscaled_windows.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build go1.19 @@ -45,17 +45,17 @@ import ( "tailscale.com/drive/driveimpl" "tailscale.com/envknob" _ "tailscale.com/ipn/auditlog" - _ "tailscale.com/ipn/desktop" "tailscale.com/logpolicy" - "tailscale.com/logtail/backoff" "tailscale.com/net/dns" "tailscale.com/net/netmon" "tailscale.com/net/tstun" "tailscale.com/tsd" "tailscale.com/types/logger" "tailscale.com/types/logid" + "tailscale.com/util/backoff" "tailscale.com/util/osdiag" - "tailscale.com/util/syspolicy" + "tailscale.com/util/syspolicy/pkey" + "tailscale.com/util/syspolicy/policyclient" "tailscale.com/util/winutil" "tailscale.com/util/winutil/gp" "tailscale.com/version" @@ -148,6 +148,8 @@ var syslogf logger.Logf = logger.Discard // // At this point we're still the parent process that // Windows started. +// +// pol may be nil. func runWindowsService(pol *logpolicy.Policy) error { go func() { logger.Logf(log.Printf).JSON(1, "SupportInfo", osdiag.SupportInfo(osdiag.LogSupportInfoReasonStartup)) @@ -155,7 +157,7 @@ func runWindowsService(pol *logpolicy.Policy) error { if syslog, err := eventlog.Open(serviceName); err == nil { syslogf = func(format string, args ...any) { - if logSCMInteractions, _ := syspolicy.GetBoolean(syspolicy.LogSCMInteractions, false); logSCMInteractions { + if logSCMInteractions, _ := policyclient.Get().GetBoolean(pkey.LogSCMInteractions, false); logSCMInteractions { syslog.Info(0, fmt.Sprintf(format, args...)) } } @@ -168,7 +170,7 @@ func runWindowsService(pol *logpolicy.Policy) error { } type ipnService struct { - Policy *logpolicy.Policy + Policy *logpolicy.Policy // or nil if logging not in use } // Called by Windows to execute the windows service. @@ -185,7 +187,11 @@ func (service *ipnService) Execute(args []string, r <-chan svc.ChangeRequest, ch doneCh := make(chan struct{}) go func() { defer close(doneCh) - args := []string{"/subproc", service.Policy.PublicID.String()} + publicID := "none" + if service.Policy != nil { + publicID = service.Policy.PublicID.String() + } + args := []string{"/subproc", publicID} // Make a logger without a date prefix, as filelogger // and logtail both already add their own. All we really want // from the log package is the automatic newline. @@ -389,8 +395,7 @@ func handleSessionChange(chgRequest svc.ChangeRequest) { if chgRequest.Cmd != svc.SessionChange || chgRequest.EventType != windows.WTS_SESSION_UNLOCK { return } - - if flushDNSOnSessionUnlock, _ := syspolicy.GetBoolean(syspolicy.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock { + if flushDNSOnSessionUnlock, _ := policyclient.Get().GetBoolean(pkey.FlushDNSOnSessionUnlock, false); flushDNSOnSessionUnlock { log.Printf("Received WTS_SESSION_UNLOCK event, initiating DNS flush.") go func() { err := dns.Flush() diff --git a/cmd/tailscaled/tailscaledhooks/tailscaledhooks.go b/cmd/tailscaled/tailscaledhooks/tailscaledhooks.go index 6ea662d39230c..42009d02bf6af 100644 --- a/cmd/tailscaled/tailscaledhooks/tailscaledhooks.go +++ b/cmd/tailscaled/tailscaledhooks/tailscaledhooks.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package tailscaledhooks provides hooks for optional features diff --git a/cmd/tailscaled/webclient.go b/cmd/tailscaled/webclient.go new file mode 100644 index 0000000000000..e031277abfc27 --- /dev/null +++ b/cmd/tailscaled/webclient.go @@ -0,0 +1,21 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !ts_omit_webclient + +package main + +import ( + "tailscale.com/client/local" + "tailscale.com/ipn/ipnlocal" + "tailscale.com/paths" +) + +func init() { + hookConfigureWebClient.Set(func(lb *ipnlocal.LocalBackend) { + lb.ConfigureWebClient(&local.Client{ + Socket: args.socketpath, + UseSocketOnly: args.socketpath != paths.DefaultTailscaledSocket(), + }) + }) +} diff --git a/cmd/tailscaled/with_cli.go b/cmd/tailscaled/with_cli.go index a8554eb8ce9dc..33da1f448e727 100644 --- a/cmd/tailscaled/with_cli.go +++ b/cmd/tailscaled/with_cli.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build ts_include_cli diff --git a/cmd/testcontrol/testcontrol.go b/cmd/testcontrol/testcontrol.go index b05b3128df0ef..49e7e429e63e9 100644 --- a/cmd/testcontrol/testcontrol.go +++ b/cmd/testcontrol/testcontrol.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Program testcontrol runs a simple test control server. diff --git a/cmd/testwrapper/args.go b/cmd/testwrapper/args.go index 95157bc34efee..22e5d4c902f8e 100644 --- a/cmd/testwrapper/args.go +++ b/cmd/testwrapper/args.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main @@ -8,6 +8,7 @@ import ( "io" "os" "slices" + "strconv" "strings" "testing" ) @@ -60,6 +61,9 @@ func splitArgs(args []string) (pre, pkgs, post []string, _ error) { return nil, nil, nil, err } fs.Visit(func(f *flag.Flag) { + if f.Name == "cachelink" && !cacheLink.enabled { + return + } if f.Value.String() != f.DefValue && f.DefValue != "false" { pre = append(pre, "-"+f.Name, f.Value.String()) } else { @@ -79,6 +83,37 @@ func splitArgs(args []string) (pre, pkgs, post []string, _ error) { return pre, pkgs, post, nil } +// cacheLink is whether the -cachelink flag is enabled. +// +// The -cachelink flag is Tailscale-specific addition to the "go test" command; +// see https://github.com/tailscale/go/issues/149 and +// https://github.com/golang/go/issues/77349. +// +// In that PR, it's only a boolean, but we implement a custom flag type +// so we can support -cachelink=auto, which enables cachelink if GOCACHEPROG +// is set, which is a behavior we want in our CI environment. +var cacheLink cacheLinkVal + +type cacheLinkVal struct { + enabled bool +} + +func (c *cacheLinkVal) String() string { + return strconv.FormatBool(c.enabled) +} + +func (c *cacheLinkVal) Set(s string) error { + if s == "auto" { + c.enabled = os.Getenv("GOCACHEPROG") != "" + return nil + } + var err error + c.enabled, err = strconv.ParseBool(s) + return err +} + +func (*cacheLinkVal) IsBoolFlag() bool { return true } + func newTestFlagSet() *flag.FlagSet { fs := flag.NewFlagSet("testwrapper", flag.ContinueOnError) fs.SetOutput(io.Discard) @@ -89,6 +124,9 @@ func newTestFlagSet() *flag.FlagSet { // TODO(maisem): figure out what other flags we need to register explicitly. fs.String("exec", "", "Command to run tests with") fs.Bool("race", false, "build with race detector") + fs.String("vet", "", "vet checks to run, or 'off' or 'all'") + + fs.Var(&cacheLink, "cachelink", "Go -cachelink value (bool); or 'auto' to enable if GOCACHEPROG is set") return fs } diff --git a/cmd/testwrapper/args_test.go b/cmd/testwrapper/args_test.go index 10063d7bcf6e1..25364fb96d6a1 100644 --- a/cmd/testwrapper/args_test.go +++ b/cmd/testwrapper/args_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package main diff --git a/cmd/testwrapper/flakytest/flakytest.go b/cmd/testwrapper/flakytest/flakytest.go index 6302900cbd3ab..ff9b8e5bbec91 100644 --- a/cmd/testwrapper/flakytest/flakytest.go +++ b/cmd/testwrapper/flakytest/flakytest.go @@ -1,9 +1,14 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause -// Package flakytest contains test helpers for marking a test as flaky. For -// tests run using cmd/testwrapper, a failed flaky test will cause tests to be -// re-run a few time until they succeed or exceed our iteration limit. +// Package flakytest contains test helpers for marking a test as flaky. +// +// Marking a test with [Mark] is not required for cmd/testwrapper to retry +// failed tests; the wrapper retries any failure within a per-test time +// budget and reports a test as flaky if it ever passes on retry. Mark is +// useful for tracking a known-flaky test against a GitHub issue and for the +// TS_SKIP_FLAKY_TESTS skip behavior used to keep CI green when a flake is +// being investigated. package flakytest import ( @@ -11,6 +16,7 @@ import ( "os" "path" "regexp" + "strconv" "sync" "testing" @@ -27,7 +33,7 @@ const FlakyTestLogMessage = "flakytest: this is a known flaky test" // starting at 1. const FlakeAttemptEnv = "TS_TESTWRAPPER_ATTEMPT" -var issueRegexp = regexp.MustCompile(`\Ahttps://github\.com/tailscale/[a-zA-Z0-9_.-]+/issues/\d+\z`) +var issueRegexp = regexp.MustCompile(`\Ahttps://github\.com/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/issues/\d+\z`) var ( rootFlakesMu sync.Mutex @@ -49,8 +55,21 @@ func Mark(t testing.TB, issue string) { // spamming people running tests without the wrapper) fmt.Fprintf(os.Stderr, "%s: %s\n", FlakyTestLogMessage, issue) } + t.Attr("flaky-test-issue-url", issue) + + // The Attr method above also emits human-readable output, so this t.Logf + // is somewhat redundant, but we keep it for compatibility with + // old test runs, so cmd/testwrapper doesn't need to be modified. + // TODO(bradfitz): switch testwrapper to look for Action "attr" + // instead: + // "Action":"attr","Package":"tailscale.com/cmd/testwrapper/flakytest","Test":"TestMarked_Root","Key":"flaky-test-issue-url","Value":"https://github.com/tailscale/tailscale/issues/0"} + // And then remove this Logf a month or so after that. t.Logf("flakytest: issue tracking this flaky test: %s", issue) + if boolEnv("TS_SKIP_FLAKY_TESTS") { + t.Skipf("skipping due to TS_SKIP_FLAKY_TESTS") + } + // Record the root test name as flakey. rootFlakesMu.Lock() defer rootFlakesMu.Unlock() @@ -71,3 +90,12 @@ func Marked(t testing.TB) bool { } return false } + +func boolEnv(k string) bool { + s := os.Getenv(k) + if s == "" { + return false + } + v, _ := strconv.ParseBool(s) + return v +} diff --git a/cmd/testwrapper/flakytest/flakytest_test.go b/cmd/testwrapper/flakytest/flakytest_test.go index 64cbfd9a3cd1f..54dd2121bd1f3 100644 --- a/cmd/testwrapper/flakytest/flakytest_test.go +++ b/cmd/testwrapper/flakytest/flakytest_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Tailscale Inc & AUTHORS +// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package flakytest @@ -14,7 +14,8 @@ func TestIssueFormat(t *testing.T) { want bool }{ {"https://github.com/tailscale/cOrp/issues/1234", true}, - {"https://github.com/otherproject/corp/issues/1234", false}, + {"https://github.com/otherproject/corp/issues/1234", true}, + {"https://not.huyb/tailscale/corp/issues/1234", false}, {"https://github.com/tailscale/corp/issues/", false}, } for _, testCase := range testCases { diff --git a/cmd/testwrapper/testdata/A_baseline/raw.txt b/cmd/testwrapper/testdata/A_baseline/raw.txt new file mode 100644 index 0000000000000..36bfad518344b --- /dev/null +++ b/cmd/testwrapper/testdata/A_baseline/raw.txt @@ -0,0 +1,6 @@ +=== RUN TestOne + a_test.go:6: hello from one +--- PASS: TestOne (0.00s) +=== RUN TestTwo +--- PASS: TestTwo (0.00s) +PASS diff --git a/cmd/testwrapper/testdata/A_baseline/src.go b/cmd/testwrapper/testdata/A_baseline/src.go new file mode 100644 index 0000000000000..5f90db1dda074 --- /dev/null +++ b/cmd/testwrapper/testdata/A_baseline/src.go @@ -0,0 +1,13 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package baseline + +import "testing" + +func TestOne(t *testing.T) { + t.Log("hello from one") +} + +func TestTwo(t *testing.T) { +} diff --git a/cmd/testwrapper/testdata/B_inbody/raw.txt b/cmd/testwrapper/testdata/B_inbody/raw.txt new file mode 100644 index 0000000000000..108a0ec59cf14 --- /dev/null +++ b/cmd/testwrapper/testdata/B_inbody/raw.txt @@ -0,0 +1,30 @@ +=== RUN TestRace +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 8: + racesurvey/B_inbody.TestRace.func1() + /tmp/racesurvey/B_inbody/b_test.go:13 +0x74 + +Previous write at 0x0000007f12a8 by goroutine 9: + racesurvey/B_inbody.TestRace.func2() + /tmp/racesurvey/B_inbody/b_test.go:14 +0x8c + +Goroutine 8 (running) created at: + racesurvey/B_inbody.TestRace() + /tmp/racesurvey/B_inbody/b_test.go:13 +0xbe + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 9 (finished) created at: + racesurvey/B_inbody.TestRace() + /tmp/racesurvey/B_inbody/b_test.go:14 +0x124 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== + testing.go:1712: race detected during execution of test +--- FAIL: TestRace (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/B_inbody/src.go b/cmd/testwrapper/testdata/B_inbody/src.go new file mode 100644 index 0000000000000..f0abebb993006 --- /dev/null +++ b/cmd/testwrapper/testdata/B_inbody/src.go @@ -0,0 +1,19 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package inbody + +import ( + "sync" + "testing" +) + +var counter int + +func TestRace(t *testing.T) { + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); counter++ }() + go func() { defer wg.Done(); counter++ }() + wg.Wait() +} diff --git a/cmd/testwrapper/testdata/C_spawnwait/caught.raw.txt b/cmd/testwrapper/testdata/C_spawnwait/caught.raw.txt new file mode 100644 index 0000000000000..c2dcf3655bc85 --- /dev/null +++ b/cmd/testwrapper/testdata/C_spawnwait/caught.raw.txt @@ -0,0 +1,32 @@ +=== RUN TestSpawn +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 9: + racesurvey/C_spawnwait.TestSpawn.func2() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x70 + +Previous write at 0x0000007f12a8 by goroutine 8: + racesurvey/C_spawnwait.TestSpawn.func1() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x88 + +Goroutine 9 (running) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x44 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 8 (finished) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x34 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== + testing.go:1712: race detected during execution of test +--- FAIL: TestSpawn (0.00s) +=== RUN TestWait +--- PASS: TestWait (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/C_spawnwait/pass.raw.txt b/cmd/testwrapper/testdata/C_spawnwait/pass.raw.txt new file mode 100644 index 0000000000000..c7ba6ebf37e92 --- /dev/null +++ b/cmd/testwrapper/testdata/C_spawnwait/pass.raw.txt @@ -0,0 +1,31 @@ +=== RUN TestSpawn +--- PASS: TestSpawn (0.00s) +=== RUN TestWait +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 8: + racesurvey/C_spawnwait.TestSpawn.func1() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x70 + +Previous write at 0x0000007f12a8 by goroutine 9: + racesurvey/C_spawnwait.TestSpawn.func2() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x88 + +Goroutine 8 (running) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:13 +0x34 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 9 (finished) created at: + racesurvey/C_spawnwait.TestSpawn() + /tmp/racesurvey/C_spawnwait/c_test.go:14 +0x44 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== +--- PASS: TestWait (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/C_spawnwait/src.go b/cmd/testwrapper/testdata/C_spawnwait/src.go new file mode 100644 index 0000000000000..42e13fef180b7 --- /dev/null +++ b/cmd/testwrapper/testdata/C_spawnwait/src.go @@ -0,0 +1,22 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package spawnwait + +import ( + "sync" + "testing" +) + +var counter int +var wg sync.WaitGroup + +func TestSpawn(t *testing.T) { + wg.Add(2) + go func() { defer wg.Done(); counter++ }() + go func() { defer wg.Done(); counter++ }() +} + +func TestWait(t *testing.T) { + wg.Wait() +} diff --git a/cmd/testwrapper/testdata/D_delayed/raw.txt b/cmd/testwrapper/testdata/D_delayed/raw.txt new file mode 100644 index 0000000000000..d7d22f21e7137 --- /dev/null +++ b/cmd/testwrapper/testdata/D_delayed/raw.txt @@ -0,0 +1,32 @@ +=== RUN TestA +--- PASS: TestA (0.00s) +=== RUN TestSleep +================== +WARNING: DATA RACE +Read at 0x0000007f12a8 by goroutine 9: + racesurvey/D_postterminal.TestA.func2() + /tmp/racesurvey/D_postterminal/d_test.go:16 +0x8a + +Previous write at 0x0000007f12a8 by goroutine 8: + racesurvey/D_postterminal.TestA.func1() + /tmp/racesurvey/D_postterminal/d_test.go:15 +0xa4 + +Goroutine 9 (running) created at: + racesurvey/D_postterminal.TestA() + /tmp/racesurvey/D_postterminal/d_test.go:16 +0x44 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 8 (finished) created at: + racesurvey/D_postterminal.TestA() + /tmp/racesurvey/D_postterminal/d_test.go:15 +0x34 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 +================== + testing.go:1712: race detected during execution of test +--- FAIL: TestSleep (0.05s) +FAIL diff --git a/cmd/testwrapper/testdata/D_delayed/src.go b/cmd/testwrapper/testdata/D_delayed/src.go new file mode 100644 index 0000000000000..f0fed3cfc6122 --- /dev/null +++ b/cmd/testwrapper/testdata/D_delayed/src.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package delayed + +import ( + "sync" + "testing" + "time" +) + +var counter int +var wg sync.WaitGroup +var trigger = make(chan struct{}) + +func TestA(t *testing.T) { + wg.Add(2) + go func() { defer wg.Done(); <-trigger; counter++ }() + go func() { defer wg.Done(); <-trigger; counter++ }() +} + +func TestSleep(t *testing.T) { + close(trigger) + // Sleep long enough that the goroutines race during this sleep, + // but TestSleep itself doesn't write to counter. + time.Sleep(50 * time.Millisecond) + wg.Wait() +} diff --git a/cmd/testwrapper/testdata/E_testmain/raw.txt b/cmd/testwrapper/testdata/E_testmain/raw.txt new file mode 100644 index 0000000000000..df96dbe1fdbd4 --- /dev/null +++ b/cmd/testwrapper/testdata/E_testmain/raw.txt @@ -0,0 +1,26 @@ +=== RUN TestPass +--- PASS: TestPass (0.00s) +PASS +================== +WARNING: DATA RACE +Read at 0x0000007f52a8 by goroutine 9: + racesurvey/E_testmain.TestMain.func2() + /tmp/racesurvey/E_testmain/e_test.go:18 +0x74 + +Previous write at 0x0000007f52a8 by goroutine 8: + racesurvey/E_testmain.TestMain.func1() + /tmp/racesurvey/E_testmain/e_test.go:17 +0x8c + +Goroutine 9 (running) created at: + racesurvey/E_testmain.TestMain() + /tmp/racesurvey/E_testmain/e_test.go:18 +0x139 + main.main() + _testmain.go:48 +0x171 + +Goroutine 8 (finished) created at: + racesurvey/E_testmain.TestMain() + /tmp/racesurvey/E_testmain/e_test.go:17 +0xcc + main.main() + _testmain.go:48 +0x171 +================== +Found 1 data race(s) diff --git a/cmd/testwrapper/testdata/E_testmain/src.go b/cmd/testwrapper/testdata/E_testmain/src.go new file mode 100644 index 0000000000000..ec15be22b3f28 --- /dev/null +++ b/cmd/testwrapper/testdata/E_testmain/src.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package testmain + +import ( + "sync" + "testing" +) + +var counter int + +func TestPass(t *testing.T) { +} + +func TestMain(m *testing.M) { + code := m.Run() + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); counter++ }() + go func() { defer wg.Done(); counter++ }() + wg.Wait() + _ = code +} diff --git a/cmd/testwrapper/testdata/F_parallel/split.raw.txt b/cmd/testwrapper/testdata/F_parallel/split.raw.txt new file mode 100644 index 0000000000000..6c8e1137d4dc5 --- /dev/null +++ b/cmd/testwrapper/testdata/F_parallel/split.raw.txt @@ -0,0 +1,57 @@ +=== RUN TestParA +=== PAUSE TestParA +=== RUN TestParB +=== PAUSE TestParB +=== CONT TestParA +=== CONT TestParB +--- PASS: TestParA (0.00s) +================== +WARNING: DATA RACE +Read at 0x0000007f02b0 by goroutine 8: + racesurvey/F_parallel.TestParB() + /tmp/racesurvey/F_parallel/f_test.go:17 +0x3b + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Previous write at 0x0000007f02b0 by goroutine 7: + racesurvey/F_parallel.TestParA() + /tmp/racesurvey/F_parallel/f_test.go:10 +0x53 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.(*T).Run.gowrap1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0x38 + +Goroutine 8 (running) created at: + testing.(*T).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0xb12 + testing.runTests.func1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2585 +0x84 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.runTests() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2583 +0x9e9 + testing.(*M).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2443 +0xf4b + main.main() + _testmain.go:48 +0x164 + +Goroutine 7 (running) created at: + testing.(*T).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2101 +0xb12 + testing.runTests.func1() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2585 +0x84 + testing.tRunner() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2036 +0x21c + testing.runTests() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2583 +0x9e9 + testing.(*M).Run() + /home/ubuntu/sdk/go1.26.3/src/testing/testing.go:2443 +0xf4b + main.main() + _testmain.go:48 +0x164 +================== +=== NAME TestParB + testing.go:1712: race detected during execution of test +--- FAIL: TestParB (0.00s) +FAIL diff --git a/cmd/testwrapper/testdata/F_parallel/src.go b/cmd/testwrapper/testdata/F_parallel/src.go new file mode 100644 index 0000000000000..18cd968ffe5a2 --- /dev/null +++ b/cmd/testwrapper/testdata/F_parallel/src.go @@ -0,0 +1,22 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package parallel + +import "testing" + +var counter int + +func TestParA(t *testing.T) { + t.Parallel() + for i := 0; i < 100; i++ { + counter++ + } +} + +func TestParB(t *testing.T) { + t.Parallel() + for i := 0; i < 100; i++ { + counter++ + } +} diff --git a/cmd/testwrapper/testdata/README.md b/cmd/testwrapper/testdata/README.md new file mode 100644 index 0000000000000..d15dc64c3f885 --- /dev/null +++ b/cmd/testwrapper/testdata/README.md @@ -0,0 +1,133 @@ +# Race-output test corpus + +This directory is a corpus of captured Go test binary outputs that +exercise the various ways the `-race` detector's `WARNING: DATA RACE` +text can land relative to `=== RUN` / `--- PASS:` / `--- FAIL:` / +`=== NAME` lines, and how `cmd/internal/test2json` attributes that +output to tests. + +Each scenario subdirectory contains: + +- `src.go` — the Go source code that was compiled and run to produce + the captured output. Reproduce via + `go test -race -c -o /tmp/scenario.test .//`. +- `raw.txt` (or scenario-specific name) — the raw stdout+stderr of + the resulting test binary when run as `./scenario.test -test.v`. + This is the byte stream that `go test -json` feeds to + `go tool test2json` in production. + +`go test -json` adds two things on top of what `test2json` sees, +which are NOT in these captures: a `FAIL\t\t