diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6300f45b7..3388f839d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,7 +16,10 @@ }, "features": { - "ghcr.io/devcontainers/features/github-cli:1": {} + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/dotnet:2": { + "version": "8.0" + } }, "remoteUser": "vscode" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 89041ba69..9a03a40d5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -148,12 +148,15 @@ jobs: dist/linux-x64/ado2gh-linux-amd64 dist/linux-x64/bbs2gh-linux-amd64 dist/linux-x64/gei-linux-amd64 + dist/linux-x64/gl2gh-linux-amd64 dist/osx-x64/ado2gh-darwin-amd64 dist/osx-x64/bbs2gh-darwin-amd64 dist/osx-x64/gei-darwin-amd64 + dist/osx-x64/gl2gh-darwin-amd64 dist/win-x64/ado2gh-windows-amd64.exe dist/win-x64/bbs2gh-windows-amd64.exe dist/win-x64/gei-windows-amd64.exe + dist/win-x64/gl2gh-windows-amd64.exe e2e-test: permissions: @@ -166,7 +169,7 @@ jobs: fail-fast: false matrix: runner-os: [windows-latest, ubuntu-latest, macos-latest] - source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github] + source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github, Gitlab] runs-on: ${{ matrix.runner-os }} concurrency: integration-test-${{ matrix.source-vcs }}-${{ matrix.runner-os }} steps: @@ -191,9 +194,11 @@ jobs: New-Item -Path "./" -Name "gh-gei" -ItemType "directory" New-Item -Path "./" -Name "gh-ado2gh" -ItemType "directory" New-Item -Path "./" -Name "gh-bbs2gh" -ItemType "directory" + New-Item -Path "./" -Name "gh-gl2gh" -ItemType "directory" Copy-Item ./dist/linux-x64/gei-linux-amd64 ./gh-gei/gh-gei Copy-Item ./dist/linux-x64/ado2gh-linux-amd64 ./gh-ado2gh/gh-ado2gh Copy-Item ./dist/linux-x64/bbs2gh-linux-amd64 ./gh-bbs2gh/gh-bbs2gh + Copy-Item ./dist/linux-x64/gl2gh-linux-amd64 ./gh-gl2gh/gh-gl2gh shell: pwsh - name: Copy binary to root (windows) @@ -202,9 +207,11 @@ jobs: New-Item -Path "./" -Name "gh-gei" -ItemType "directory" New-Item -Path "./" -Name "gh-ado2gh" -ItemType "directory" New-Item -Path "./" -Name "gh-bbs2gh" -ItemType "directory" + New-Item -Path "./" -Name "gh-gl2gh" -ItemType "directory" Copy-Item ./dist/win-x64/gei-windows-amd64.exe ./gh-gei/gh-gei.exe Copy-Item ./dist/win-x64/ado2gh-windows-amd64.exe ./gh-ado2gh/gh-ado2gh.exe Copy-Item ./dist/win-x64/bbs2gh-windows-amd64.exe ./gh-bbs2gh/gh-bbs2gh.exe + Copy-Item ./dist/win-x64/gl2gh-windows-amd64.exe ./gh-gl2gh/gh-gl2gh.exe shell: pwsh - name: Copy binary to root (macos) @@ -213,9 +220,11 @@ jobs: New-Item -Path "./" -Name "gh-gei" -ItemType "directory" New-Item -Path "./" -Name "gh-ado2gh" -ItemType "directory" New-Item -Path "./" -Name "gh-bbs2gh" -ItemType "directory" + New-Item -Path "./" -Name "gh-gl2gh" -ItemType "directory" Copy-Item ./dist/osx-x64/gei-darwin-amd64 ./gh-gei/gh-gei Copy-Item ./dist/osx-x64/ado2gh-darwin-amd64 ./gh-ado2gh/gh-ado2gh Copy-Item ./dist/osx-x64/bbs2gh-darwin-amd64 ./gh-bbs2gh/gh-bbs2gh + Copy-Item ./dist/osx-x64/gl2gh-darwin-amd64 ./gh-gl2gh/gh-gl2gh shell: pwsh - name: Set execute permissions @@ -223,6 +232,7 @@ jobs: chmod +x ./gh-gei/gh-gei chmod +x ./gh-ado2gh/gh-ado2gh chmod +x ./gh-bbs2gh/gh-bbs2gh + chmod +x ./gh-gl2gh/gh-gl2gh - name: Install gh-gei extension run: gh extension install . @@ -245,6 +255,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install gh-gl2gh extension + run: gh extension install . + shell: pwsh + working-directory: ./gh-gl2gh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Integration Test env: ADO_PAT: ${{ secrets.ADO_PAT }} @@ -266,6 +283,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} + GITLAB_PAT: ${{ secrets.GITLAB_PAT }} LD_LIBRARY_PATH: "$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native" run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9 @@ -386,6 +404,21 @@ jobs: ./dist/osx-x64/bbs2gh-darwin-amd64 ./dist/osx-arm64/bbs2gh-darwin-arm64 + - name: Create gh-gl2gh Release + # a06a81a03ee405af7f2048a818ed3f03bbf83c7b is tag v2, pinned to avoid supply chain attacks + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b + with: + body_path: ./RELEASENOTES.md + repository: github/gh-gl2gh + token: ${{ secrets.RELEASE_NOTES_PAT }} + files: | + ./dist/win-x86/gl2gh-windows-386.exe + ./dist/win-x64/gl2gh-windows-amd64.exe + ./dist/linux-x64/gl2gh-linux-amd64 + ./dist/linux-arm64/gl2gh-linux-arm64 + ./dist/osx-x64/gl2gh-darwin-amd64 + ./dist/osx-arm64/gl2gh-darwin-arm64 + - name: Archive Release Notes shell: pwsh run: | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 308a477f4..64308fda4 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -78,12 +78,15 @@ jobs: dist/linux-x64/ado2gh-linux-amd64 dist/linux-x64/bbs2gh-linux-amd64 dist/linux-x64/gei-linux-amd64 + dist/linux-x64/gl2gh-linux-amd64 dist/osx-x64/ado2gh-darwin-amd64 dist/osx-x64/bbs2gh-darwin-amd64 dist/osx-x64/gei-darwin-amd64 + dist/osx-x64/gl2gh-darwin-amd64 dist/win-x64/ado2gh-windows-amd64.exe dist/win-x64/bbs2gh-windows-amd64.exe dist/win-x64/gei-windows-amd64.exe + dist/win-x64/gl2gh-windows-amd64.exe e2e-test: needs: [ build-for-e2e-test ] @@ -91,7 +94,7 @@ jobs: fail-fast: false matrix: runner-os: [windows-latest, ubuntu-latest, macos-latest] - source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github] + source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github, Gitlab] runs-on: ${{ matrix.runner-os }} concurrency: integration-test-${{ matrix.source-vcs }}-${{ matrix.runner-os }} steps: @@ -124,9 +127,11 @@ jobs: New-Item -Path "./" -Name "gh-gei" -ItemType "directory" New-Item -Path "./" -Name "gh-ado2gh" -ItemType "directory" New-Item -Path "./" -Name "gh-bbs2gh" -ItemType "directory" + New-Item -Path "./" -Name "gh-gl2gh" -ItemType "directory" Copy-Item ./dist/linux-x64/gei-linux-amd64 ./gh-gei/gh-gei Copy-Item ./dist/linux-x64/ado2gh-linux-amd64 ./gh-ado2gh/gh-ado2gh Copy-Item ./dist/linux-x64/bbs2gh-linux-amd64 ./gh-bbs2gh/gh-bbs2gh + Copy-Item ./dist/linux-x64/gl2gh-linux-amd64 ./gh-gl2gh/gh-gl2gh shell: pwsh - name: Copy binary to root (windows) @@ -135,9 +140,11 @@ jobs: New-Item -Path "./" -Name "gh-gei" -ItemType "directory" New-Item -Path "./" -Name "gh-ado2gh" -ItemType "directory" New-Item -Path "./" -Name "gh-bbs2gh" -ItemType "directory" + New-Item -Path "./" -Name "gh-gl2gh" -ItemType "directory" Copy-Item ./dist/win-x64/gei-windows-amd64.exe ./gh-gei/gh-gei.exe Copy-Item ./dist/win-x64/ado2gh-windows-amd64.exe ./gh-ado2gh/gh-ado2gh.exe Copy-Item ./dist/win-x64/bbs2gh-windows-amd64.exe ./gh-bbs2gh/gh-bbs2gh.exe + Copy-Item ./dist/win-x64/gl2gh-windows-amd64.exe ./gh-gl2gh/gh-gl2gh.exe shell: pwsh - name: Copy binary to root (macos) @@ -146,9 +153,11 @@ jobs: New-Item -Path "./" -Name "gh-gei" -ItemType "directory" New-Item -Path "./" -Name "gh-ado2gh" -ItemType "directory" New-Item -Path "./" -Name "gh-bbs2gh" -ItemType "directory" + New-Item -Path "./" -Name "gh-gl2gh" -ItemType "directory" Copy-Item ./dist/osx-x64/gei-darwin-amd64 ./gh-gei/gh-gei Copy-Item ./dist/osx-x64/ado2gh-darwin-amd64 ./gh-ado2gh/gh-ado2gh Copy-Item ./dist/osx-x64/bbs2gh-darwin-amd64 ./gh-bbs2gh/gh-bbs2gh + Copy-Item ./dist/osx-x64/gl2gh-darwin-amd64 ./gh-gl2gh/gh-gl2gh shell: pwsh - name: Set execute permissions @@ -156,6 +165,7 @@ jobs: chmod +x ./gh-gei/gh-gei chmod +x ./gh-ado2gh/gh-ado2gh chmod +x ./gh-bbs2gh/gh-bbs2gh + chmod +x ./gh-gl2gh/gh-gl2gh - name: Install gh-gei extension run: gh extension install . @@ -178,6 +188,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install gh-gl2gh extension + run: gh extension install . + shell: pwsh + working-directory: ./gh-gl2gh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Integration Test env: ADO_PAT: ${{ secrets.ADO_PAT }} @@ -199,6 +216,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} + GITLAB_PAT: ${{ secrets.GITLAB_PAT }} GEI_DEBUG_MODE: 'true' LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:${{ github.workspace }}/src/OctoshiftCLI.IntegrationTests/bin/Debug/net8.0/runtimes/ubuntu.18.04-x64/native' run: dotnet test src/OctoshiftCLI.IntegrationTests/OctoshiftCLI.IntegrationTests.csproj --filter "${{ matrix.source-vcs }}ToGithub" --logger:"junit;LogFilePath=integration-tests.xml" --logger "console;verbosity=normal" /p:VersionPrefix=9.9 diff --git a/README.md b/README.md index b288d3b72..d86bb500d 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ The [GitHub Enterprise Importer](https://docs.github.com/en/migrations/using-git > GEI is generally available for repository migrations originating from Azure DevOps or GitHub that target GitHub Enterprise Cloud. It is in public beta for repository migrations from BitBucket Server and Data Center to GitHub Enterprise Cloud. ## Using the GEI CLI -There are 3 separate CLIs that we ship as extensions for the official [GitHub CLI](https://github.com/cli/cli#installation): +There are 4 separate CLIs that we ship as extensions for the official [GitHub CLI](https://github.com/cli/cli#installation): - `gh gei` - Run migrations between GitHub products - `gh ado2gh` - Run migrations from Azure DevOps to GitHub - `gh bbs2gh` - Run migrations from BitBucket Server or Data Center to GitHub +- `gh gl2gh` - Run migrations from GitLab to GitHub _(not yet generally available)_ To use `gh gei` first install the latest [GitHub CLI](https://github.com/cli/cli#installation), then run the command >`gh extension install github/gh-gei` @@ -22,6 +23,11 @@ To use `gh ado2gh` first install the latest [GitHub CLI](https://github.com/cli/ To use `gh bbs2gh` first install the latest [GitHub CLI](https://github.com/cli/cli#installation), then run the command >`gh extension install github/gh-bbs2gh` +To use `gh gl2gh` first install the latest [GitHub CLI](https://github.com/cli/cli#installation), then run the command +>`gh extension install github/gh-gl2gh` + +> **Note:** `gh gl2gh` is not yet generally available. The extension repo and releases may not be published yet. + We update the extensions frequently, so make sure you update them on a regular basis: >`gh extension upgrade github/gh-gei` @@ -33,6 +39,8 @@ To see the available commands and options run: >`gh bbs2gh --help` +>`gh gl2gh --help` + ### GitHub to GitHub Usage (GitHub.com -> GitHub.com) 1. Create Personal Access Tokens with access to the source GitHub org, and the target GitHub org (for more details on scopes needed refer to our [official documentation](https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer)). @@ -87,6 +95,24 @@ Refer to the [official documentation](https://docs.github.com/en/migrations/usin Refer to the [official documentation](https://docs.github.com/en/migrations/using-github-enterprise-importer/migrating-repositories-with-github-enterprise-importer/migrating-repositories-from-bitbucket-server-to-github-enterprise-cloud) for more details. +### GitLab to GitHub Usage +1. Create a Personal Access Token for the source GitLab instance (with `api` and `read_repository` scopes) and one for the target GitHub org (for more details on scopes needed refer to our [official documentation](https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer)). + +2. Set the `GITLAB_PAT` and `GH_PAT` environment variables. + +3. Run the `generate-script` command to generate a migration script. +``` +> gh gl2gh generate-script --gitlab-server-url GITLAB-SERVER-URL \ + --github-org DESTINATION \ + --output FILENAME +``` + +The `--gitlab-server-url` flag accepts both GitLab.com (`https://gitlab.com`) and self-hosted GitLab instances. + +4. The previous command will have created a `migrate.ps1` PowerShell script. Review the steps in the generated script and tweak if necessary. + +5. The `migrate.ps1` script requires PowerShell to run. If not already installed see the [install instructions](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell?view=powershell-7.2) to install PowerShell on Windows, Linux, or Mac. Then run the script. + ### Skipping version checks When the CLI is launched, it logs if a newer version of the CLI is available. You can skip this check by setting the `GEI_SKIP_VERSION_CHECK` environment variable to `true`. diff --git a/RELEASENOTES.md b/RELEASENOTES.md index e69de29bb..09c5d5724 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -0,0 +1 @@ +- Added new `gl2gh` CLI (not yet generally available) for migrating GitLab repositories (GitLab.com and self-hosted) to GitHub. Includes `migrate-repo`, `generate-script`, and `inventory-report` along with the standard supporting commands (e.g. `download-logs`, `wait-for-migration`, `abort-migration`, `grant-migrator-role`, `revoke-migrator-role`, `create-team`, `generate-mannequin-csv`, `reclaim-mannequin`). Archives can be uploaded via Azure Blob Storage, AWS S3, or GitHub-owned storage. diff --git a/global.json b/global.json index 34fe1191d..e3ef79017 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.410", + "version": "8.0.421", "rollForward": "minor" } } \ No newline at end of file diff --git a/justfile b/justfile index 4875c8708..6185e9287 100644 --- a/justfile +++ b/justfile @@ -60,6 +60,10 @@ run-ado2gh *args: build run-bbs2gh *args: build dotnet run --project src/bbs2gh/bbs2gh.csproj {{args}} +# Build and run the gl2gh CLI locally +run-gl2gh *args: build + dotnet run --project src/gl2gh/gl2gh.csproj {{args}} + # Watch and auto-rebuild gei on changes watch-gei: dotnet watch build --project src/gei/gei.csproj @@ -72,6 +76,10 @@ watch-ado2gh: watch-bbs2gh: dotnet watch build --project src/bbs2gh/bbs2gh.csproj +# Watch and auto-rebuild gl2gh on changes +watch-gl2gh: + dotnet watch build --project src/gl2gh/gl2gh.csproj + # Build self-contained binaries for all platforms (requires PowerShell) publish: pwsh ./publish.ps1 @@ -115,19 +123,21 @@ install-extensions: publish-linux set -euo pipefail # Create extension directories - mkdir -p gh-gei gh-ado2gh gh-bbs2gh + mkdir -p gh-gei gh-ado2gh gh-bbs2gh gh-gl2gh # Copy binaries cp ./dist/linux-x64/gei-linux-amd64 ./gh-gei/gh-gei cp ./dist/linux-x64/ado2gh-linux-amd64 ./gh-ado2gh/gh-ado2gh cp ./dist/linux-x64/bbs2gh-linux-amd64 ./gh-bbs2gh/gh-bbs2gh + cp ./dist/linux-x64/gl2gh-linux-amd64 ./gh-gl2gh/gh-gl2gh # Set execute permissions - chmod +x ./gh-gei/gh-gei ./gh-ado2gh/gh-ado2gh ./gh-bbs2gh/gh-bbs2gh + chmod +x ./gh-gei/gh-gei ./gh-ado2gh/gh-ado2gh ./gh-bbs2gh/gh-bbs2gh ./gh-gl2gh/gh-gl2gh # Install extensions cd gh-gei && gh extension install . && cd .. cd gh-ado2gh && gh extension install . && cd .. cd gh-bbs2gh && gh extension install . && cd .. + cd gh-gl2gh && gh extension install . && cd .. echo "Extensions installed successfully!" diff --git a/publish.ps1 b/publish.ps1 index ca2dfd65c..eb55bc777 100755 --- a/publish.ps1 +++ b/publish.ps1 @@ -289,4 +289,94 @@ else { } Rename-Item ./dist/osx-arm64/bbs2gh bbs2gh-darwin-arm64 +} + +### gl2gh ### +if ((Test-Path env:SKIP_WINDOWS) -And $env:SKIP_WINDOWS.ToUpper() -eq "TRUE") { + Write-Output "Skipping gl2gh Windows build because SKIP_WINDOWS is set" +} +else { + dotnet publish src/gl2gh/gl2gh.csproj -c Release -o dist/win-x64/ -r win-x64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion + + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + if (Test-Path -Path ./dist/win-x64/gl2gh-windows-amd64.exe) { + Remove-Item ./dist/win-x64/gl2gh-windows-amd64.exe + } + + Rename-Item ./dist/win-x64/gl2gh.exe gl2gh-windows-amd64.exe + + dotnet publish src/gl2gh/gl2gh.csproj -c Release -o dist/win-x86/ -r win-x86 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion + + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + if (Test-Path -Path ./dist/win-x86/gl2gh-windows-386.exe) { + Remove-Item ./dist/win-x86/gl2gh-windows-386.exe + } + + Rename-Item ./dist/win-x86/gl2gh.exe gl2gh-windows-386.exe +} + +if ((Test-Path env:SKIP_LINUX) -And $env:SKIP_LINUX.ToUpper() -eq "TRUE") { + Write-Output "Skipping gl2gh Linux build because SKIP_LINUX is set" +} +else { + dotnet publish src/gl2gh/gl2gh.csproj -c Release -o dist/linux-x64/ -r linux-x64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion + + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + if (Test-Path -Path ./dist/linux-x64/gl2gh-linux-amd64) { + Remove-Item ./dist/linux-x64/gl2gh-linux-amd64 + } + + Rename-Item ./dist/linux-x64/gl2gh gl2gh-linux-amd64 + + # linux-arm64 build + dotnet publish src/gl2gh/gl2gh.csproj -c Release -o dist/linux-arm64/ -r linux-arm64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion + + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + if (Test-Path -Path ./dist/linux-arm64/gl2gh-linux-arm64) { + Remove-Item ./dist/linux-arm64/gl2gh-linux-arm64 + } + + Rename-Item ./dist/linux-arm64/gl2gh gl2gh-linux-arm64 +} + +if ((Test-Path env:SKIP_MACOS) -And $env:SKIP_MACOS.ToUpper() -eq "TRUE") { + Write-Output "Skipping gl2gh MacOS build because SKIP_MACOS is set" +} +else { + dotnet publish src/gl2gh/gl2gh.csproj -c Release -o dist/osx-x64/ -r osx-x64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion + + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + if (Test-Path -Path ./dist/osx-x64/gl2gh-darwin-amd64) { + Remove-Item ./dist/osx-x64/gl2gh-darwin-amd64 + } + + Rename-Item ./dist/osx-x64/gl2gh gl2gh-darwin-amd64 + + # osx-arm64 build + dotnet publish src/gl2gh/gl2gh.csproj -c Release -o dist/osx-arm64/ -r osx-arm64 -p:PublishSingleFile=true -p:PublishTrimmed=true -p:TrimMode=partial --self-contained true /p:DebugType=None /p:IncludeNativeLibrariesForSelfExtract=true /p:VersionPrefix=$AssemblyVersion + + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + if (Test-Path -Path ./dist/osx-arm64/gl2gh-darwin-arm64) { + Remove-Item ./dist/osx-arm64/gl2gh-darwin-arm64 + } + + Rename-Item ./dist/osx-arm64/gl2gh gl2gh-darwin-arm64 } \ No newline at end of file diff --git a/src/Octoshift/Models/GitlabProject.cs b/src/Octoshift/Models/GitlabProject.cs new file mode 100644 index 000000000..bee22dd75 --- /dev/null +++ b/src/Octoshift/Models/GitlabProject.cs @@ -0,0 +1,9 @@ +namespace Octoshift.Models; + +public record GitlabProject +{ + public string Id { get; init; } + public string Name { get; init; } + public string Path { get; init; } + public bool Archived { get; init; } +} diff --git a/src/Octoshift/Services/EnvironmentVariableProvider.cs b/src/Octoshift/Services/EnvironmentVariableProvider.cs index e59b47109..878d92734 100644 --- a/src/Octoshift/Services/EnvironmentVariableProvider.cs +++ b/src/Octoshift/Services/EnvironmentVariableProvider.cs @@ -15,6 +15,7 @@ public class EnvironmentVariableProvider private const string AWS_REGION = "AWS_REGION"; private const string BBS_USERNAME = "BBS_USERNAME"; private const string BBS_PASSWORD = "BBS_PASSWORD"; + private const string GITLAB_PAT = "GITLAB_PAT"; private const string SMB_PASSWORD = "SMB_PASSWORD"; private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK"; private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK"; @@ -57,6 +58,9 @@ public virtual string BbsUsername(bool throwIfNotFound = true) => public virtual string BbsPassword(bool throwIfNotFound = true) => GetSecret(BBS_PASSWORD, throwIfNotFound); + public virtual string GitlabPat(bool throwIfNotFound = true) => + GetSecret(GITLAB_PAT, throwIfNotFound); + public virtual string SmbPassword(bool throwIfNotFound = true) => GetSecret(SMB_PASSWORD, throwIfNotFound); diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 00bf29e4e..7e42442cb 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -346,6 +346,30 @@ public virtual async Task CreateBbsMigrationSource(string orgId) return (string)data["data"]["createMigrationSource"]["migrationSource"]["id"]; } + public virtual async Task CreateGitlabMigrationSource(string orgId) + { + var url = $"{_apiUrl}/graphql"; + + var query = "mutation createMigrationSource($name: String!, $url: String!, $ownerId: ID!, $type: MigrationSourceType!)"; + var gql = "createMigrationSource(input: {name: $name, url: $url, ownerId: $ownerId, type: $type}) { migrationSource { id, name, url, type } }"; + + var payload = new + { + query = $"{query} {{ {gql} }}", + variables = new + { + name = "GitLab Source", + url = "https://not-used", + ownerId = orgId, + type = "GITLAB" + }, + operationName = "createMigrationSource" + }; + + var data = await _client.PostGraphQLAsync(url, payload); + return (string)data["data"]["createMigrationSource"]["migrationSource"]["id"]; + } + public virtual async Task CreateGhecMigrationSource(string orgId) { var url = $"{_apiUrl}/graphql"; @@ -534,6 +558,23 @@ public virtual async Task StartBbsMigration(string migrationSourceId, st ); } + public virtual async Task StartGitlabMigration(string migrationSourceId, string gitlabRepoUrl, string orgId, string repo, string targetToken, string archiveUrl, string targetRepoVisibility = null) + { + return await StartMigration( + migrationSourceId, + gitlabRepoUrl, // source repository URL + orgId, + repo, + "not-used", // source access token + targetToken, + archiveUrl, + null, // GitLab archive contains both git and metadata — GitHub falls back to gitArchiveUrl + false, // skip releases + targetRepoVisibility, + false // lock source + ); + } + public virtual async Task<(string State, string RepositoryName, int WarningsCount, string FailureReason, string MigrationLogUrl)> GetMigration(string migrationId) { var url = $"{_apiUrl}/graphql"; diff --git a/src/Octoshift/Services/GithubClient.cs b/src/Octoshift/Services/GithubClient.cs index 6d846bc8d..f31e2a9fc 100644 --- a/src/Octoshift/Services/GithubClient.cs +++ b/src/Octoshift/Services/GithubClient.cs @@ -38,7 +38,7 @@ public GithubClient(OctoLogger log, HttpClient httpClient, IVersionProvider vers if (_httpClient != null) { _httpClient.DefaultRequestHeaders.Add("Accept", "application/vnd.github.v3+json"); - _httpClient.DefaultRequestHeaders.Add("GraphQL-Features", "import_api,mannequin_claiming_emu,org_import_api"); + _httpClient.DefaultRequestHeaders.Add("GraphQL-Features", "import_api,mannequin_claiming_emu,org_import_api,octoshift_ll_gitlab_self_serve"); _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", personalAccessToken); _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OctoshiftCLI", versionProvider?.GetCurrentVersion())); if (versionProvider?.GetVersionComments() is { } comments) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs new file mode 100644 index 000000000..ef667625f --- /dev/null +++ b/src/Octoshift/Services/GitlabApi.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using OctoshiftCLI.Extensions; + +namespace OctoshiftCLI.Services; + +public class GitlabApi +{ + private readonly GitlabClient _client; + private readonly string _gitlabBaseUrl; + private readonly OctoLogger _log; + + public GitlabApi(GitlabClient client, string gitlabServerUrl, OctoLogger log) + { + _client = client; + _gitlabBaseUrl = gitlabServerUrl?.TrimEnd('/'); + _log = log; + } + + public virtual async Task<(string Version, bool Enterprise)> GetServerVersion() + { + var url = $"{_gitlabBaseUrl}/api/v4/version"; + + var content = await _client.GetAsync(url); + var data = JObject.Parse(content); + + return ((string)data["version"], (bool?)data["enterprise"] ?? false); + } + + public virtual async Task LogServerVersion() + { + var (version, enterprise) = await GetServerVersion(); + if (!string.IsNullOrWhiteSpace(version)) + { + var edition = enterprise ? "Enterprise" : "Community"; + _log?.LogInformation($"GitLab version: {version} ({edition} Edition)"); + } + } + + public virtual async Task StartExport(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; + + var exportResponse = await _client.PostAsync(url, new { }); + var exportData = JObject.Parse(exportResponse); + + return (string)exportData["message"]; + } + + public virtual async Task<(string ExportStatus, string DownloadUrl)> GetExport(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; + + var exportResponse = await _client.GetAsync(url); + var exportData = JObject.Parse(exportResponse); + + return ( + (string)exportData["export_status"], + (string)exportData["_links"]?["api_url"] + ); + } + + public virtual async Task DownloadExportArchive(string groupPath, string projectPath, string file) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export/download"; + + await _client.DownloadToFile(url, file); + } + + public virtual async Task> GetProjects(string groupPath) + { + var encodedGroupPath = Uri.EscapeDataString(groupPath); + var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}/projects?per_page=100"; + + return await _client.GetAllAsync(url) + .Select(x => ((long)x["id"], (string)x["path"], (string)x["name"], (bool)x["archived"])) + .ToListAsync(); + } + + public virtual async Task<(long Id, string Path, string Name)> GetGroup(string groupPath) + { + var encodedGroupPath = groupPath.EscapeDataString(); + var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}"; + + var groupResponse = await _client.GetAsync(url); + var groupData = JObject.Parse(groupResponse); + + return ( + (long)groupData["id"], + (string)groupData["full_path"], + (string)groupData["name"] + ); + } + + public virtual async Task> GetGroups() + { + var url = $"{_gitlabBaseUrl}/api/v4/groups?per_page=100"; + + return await _client.GetAllAsync(url) + .Select(x => ((long)x["id"], (string)x["full_path"], (string)x["name"])) + .ToListAsync(); + } + + public virtual async Task GetIsProjectArchived(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}"; + + var projectResponse = await _client.GetAsync(url); + var projectData = JObject.Parse(projectResponse); + + return (bool)projectData["archived"]; + } + + public virtual async Task GetRepositoryLatestCommitDate(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/repository/commits?per_page=1"; + + // Empty projects (no Git repo yet) return 404 here; treat as no commits. + var commitsResponse = await _client.GetOrNullForNotFoundAsync(url); + if (commitsResponse is null) + { + return null; + } + + var commitsData = JArray.Parse(commitsResponse); + var lastCommittedDate = (string)commitsData.First?["committed_date"]; + + return string.IsNullOrWhiteSpace(lastCommittedDate) ? null : DateTimeOffset.Parse(lastCommittedDate); + } + + public virtual async Task<(long RepositorySize, long AttachmentsSize)> GetRepositoryAndAttachmentsSize(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?statistics=true"; + + var projectResponse = await _client.GetAsync(url); + var projectData = JObject.Parse(projectResponse); + var projectStatistics = (JObject)projectData["statistics"]; + + var repositorySize = (long)projectStatistics["repository_size"]; + var attachmentsSize = (long)projectStatistics["uploads_size"]; + + return (repositorySize, attachmentsSize); + } + + public virtual async Task GetMergeRequestCount(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/merge_requests?state=all&per_page=1&page=1"; + + using var mrResponse = await _client.GetAsyncHttpResponseMessage(url); + var mrTotal = mrResponse.Headers.GetValues("X-Total").Single(); + + return int.Parse(mrTotal); + } + + private static string GetEncodedProjectPath(string groupPath, string projectPath) + { + var pathWithNamespace = $"{groupPath}/{projectPath}"; + return pathWithNamespace.EscapeDataString(); + } +} diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs new file mode 100644 index 000000000..41aff22d4 --- /dev/null +++ b/src/Octoshift/Services/GitlabClient.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading.Tasks; +using System.Web; +using Newtonsoft.Json.Linq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; + +namespace OctoshiftCLI.Services; + +public class GitlabClient +{ + private const int DEFAULT_PAGE_SIZE = 100; + private readonly HttpClient _httpClient; + private readonly OctoLogger _log; + private readonly RetryPolicy _retryPolicy; + private readonly FileSystemProvider _fileSystemProvider; + + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, string gitlabPat, FileSystemProvider fileSystemProvider) : + this(log, httpClient, versionProvider, retryPolicy, fileSystemProvider) + { + if (_httpClient != null) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", gitlabPat); + } + } + + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, FileSystemProvider fileSystemProvider) + { + _log = log; + _httpClient = httpClient; + _retryPolicy = retryPolicy; + _fileSystemProvider = fileSystemProvider; + + if (_httpClient != null) + { + _httpClient.DefaultRequestHeaders.Add("accept", "application/json"); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OctoshiftCLI", versionProvider?.GetCurrentVersion())); + if (versionProvider?.GetVersionComments() is { } comments) + { + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(comments)); + } + } + } + + public virtual async Task GetAsync(string url) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + return await response.Content.ReadAsStringAsync(); + } + + /// + /// Like , but returns null when the server responds with 404 Not Found + /// instead of throwing. Other HTTP errors are still retried and bubble up as exceptions. + /// + public virtual async Task GetOrNullForNotFoundAsync(string url) + { + try + { + using var response = await _retryPolicy.HttpRetry( + async () => await SendAsync(HttpMethod.Get, url), + ex => ex.StatusCode != HttpStatusCode.NotFound); + return await response.Content.ReadAsStringAsync(); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + public virtual async IAsyncEnumerable GetAllAsync(string url) + { + var nextPage = 1; + while (nextPage > 0) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, AddPageParam(url, nextPage))); + var content = await response.Content.ReadAsStringAsync(); + var jArray = JArray.Parse(content); + + foreach (var jToken in jArray) + { + yield return jToken; + } + + nextPage = response.Headers.TryGetValues("X-Next-Page", out var values) + && int.TryParse(values.FirstOrDefault(), out var parsed) + ? parsed + : 0; + } + } + + public virtual async Task GetAsyncHttpResponseMessage(string url) + { + return await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + } + + public virtual async Task PostAsync(string url, object body) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Post, url, body)); + return await response.Content.ReadAsStringAsync(); + } + + public virtual async Task DeleteAsync(string url) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Delete, url)); + return await response.Content.ReadAsStringAsync(); + } + + private async Task SendAsync(HttpMethod httpMethod, string url, object body = null) + { + _log.LogVerbose($"HTTP {httpMethod}: {url}"); + + if (body != null) + { + _log.LogVerbose($"HTTP BODY: {body.ToJson()}"); + } + + using var payload = body?.ToJson().ToStringContent(); + var response = httpMethod.ToString() switch + { + "GET" => await _httpClient.GetAsync(url), + "DELETE" => await _httpClient.DeleteAsync(url), + "POST" => await _httpClient.PostAsync(url, payload), + "PUT" => await _httpClient.PutAsync(url, payload), + "PATCH" => await _httpClient.PatchAsync(url, payload), + _ => throw new ArgumentOutOfRangeException($"{httpMethod} is not supported.") + }; + var content = await response.Content.ReadAsStringAsync(); + _log.LogVerbose($"RESPONSE ({response.StatusCode}): {content}"); + + response.EnsureSuccessStatusCode(); + + return response; + } + + private static string AddPageParam(string url, int page) + { + var uri = new Uri(url); + var path = uri.GetLeftPart(UriPartial.Path); + var queryParams = HttpUtility.ParseQueryString(uri.Query); + + queryParams["page"] = page.ToString(); + if (string.IsNullOrEmpty(queryParams["per_page"])) + { + queryParams["per_page"] = DEFAULT_PAGE_SIZE.ToString(); + } + + return $"{path}?{queryParams}"; + } + + public virtual async Task DownloadToFile(string url, string file) + { + _log.LogVerbose($"HTTP GET: {url}"); + + await _retryPolicy.Retry(async () => + { + using var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); + + _log.LogVerbose($"RESPONSE ({response.StatusCode}): "); + + if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + { + var retryAfter = response.Headers.RetryAfter?.Delta?.TotalSeconds + ?? (response.Headers.RetryAfter?.Date.HasValue == true + ? (response.Headers.RetryAfter.Date.Value - DateTimeOffset.UtcNow).TotalSeconds + : null); + var retryAfterMessage = retryAfter.HasValue ? $" GitLab requested a retry after {Math.Max(0, (int)retryAfter.Value)} seconds." : ""; + _log.LogWarning($"GitLab rate limit hit (HTTP 429) downloading the export archive.{retryAfterMessage} Retrying..."); + + if (retryAfter is > 0) + { + await Task.Delay(TimeSpan.FromSeconds(Math.Min(retryAfter.Value, 60))); + } + } + + response.EnsureSuccessStatusCode(); + + await using var streamToReadFrom = await response.Content.ReadAsStreamAsync(); + await using var streamToWriteTo = _fileSystemProvider.Open(file, FileMode.Create); + await _fileSystemProvider.CopySourceToTargetStreamAsync(streamToReadFrom, streamToWriteTo); + }); + } +} diff --git a/src/OctoshiftCLI.IntegrationTests/GitlabToGithub.cs b/src/OctoshiftCLI.IntegrationTests/GitlabToGithub.cs new file mode 100644 index 000000000..ebee55d2d --- /dev/null +++ b/src/OctoshiftCLI.IntegrationTests/GitlabToGithub.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using OctoshiftCLI.Services; +using Xunit; +using Xunit.Abstractions; + +namespace OctoshiftCLI.IntegrationTests; + +[Collection("Integration Tests")] +public sealed class GitlabToGithub : IDisposable +{ + private const string GitlabServerUrl = "https://gitlab.com"; + private const string GitlabGroup = "Mouse-Hack"; + private const string GitlabProject = "no-merge-requests"; + + private readonly ITestOutputHelper _output; + private readonly OctoLogger _logger; + private readonly TestHelper _targetHelper; + private readonly HttpClient _versionClient; + private readonly HttpClient _targetGithubHttpClient; + private readonly GithubClient _targetGithubClient; + private readonly GithubApi _targetGithubApi; + private readonly Dictionary _tokens; + private readonly DateTime _startTime; + + public GitlabToGithub(ITestOutputHelper output) + { + _startTime = DateTime.Now; + _output = output; + + TestHelper.AssertCredentialsPresent( + ("GITLAB_PAT", "GitLab.com personal access token"), + ("GHEC_PAT", "GitHub Enterprise Cloud personal access token")); + + _logger = new OctoLogger(_ => { }, x => _output.WriteLine(x), _ => { }, _ => { }); + + var sourceGitlabToken = Environment.GetEnvironmentVariable("GITLAB_PAT"); + var targetGithubToken = Environment.GetEnvironmentVariable("GHEC_PAT"); + + _tokens = new Dictionary + { + ["GITLAB_PAT"] = sourceGitlabToken, + ["GH_PAT"] = targetGithubToken + }; + + _versionClient = new HttpClient(); + + _targetGithubHttpClient = new HttpClient(); + _targetGithubClient = new GithubClient(_logger, _targetGithubHttpClient, new VersionChecker(_versionClient, _logger), new RetryPolicy(_logger, "GitHub (GHEC_PAT)"), new DateTimeProvider(), targetGithubToken); + _targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(_logger, "GitHub (GHEC_PAT)"), null); + + _targetHelper = new TestHelper(_output, _targetGithubApi, _targetGithubClient); + } + + [Fact] + public async Task Basic() + { + var githubTargetOrg = $"octoshift-e2e-gitlab-{TestHelper.GetOsName()}"; + // generate-script derives the GitHub repo name from "{group}-{project}". + var targetRepo = $"{GitlabGroup}-{GitlabProject}"; + + // Pre-clean: wipe the target org so a prior run doesn't influence this one. + // Matches the pattern used by every other adapter's integration test. + var retryPolicy = new RetryPolicy(null); + await retryPolicy.Retry(async () => + { + await _targetHelper.ResetGithubTestEnvironment(githubTargetOrg); + }); + + // Exercise inventory-report against the source group. + await _targetHelper.RunCliCommand( + $"gl2gh inventory-report --gitlab-server-url {GitlabServerUrl} --gitlab-group {GitlabGroup}", + "gh", + _tokens); + + // Exercise generate-script + run the generated migrate.ps1, scoped to a single project + // so we only ever migrate the repo we own for this test. + await _targetHelper.RunCliMigration( + $"gl2gh generate-script --gitlab-server-url {GitlabServerUrl} --gitlab-group {GitlabGroup} --gitlab-project {GitlabProject} --github-org {githubTargetOrg} --use-github-storage", + "gh", + _tokens); + + _targetHelper.AssertNoErrorInLogs(_startTime); + + await _targetHelper.AssertGithubRepoExists(githubTargetOrg, targetRepo); + await _targetHelper.AssertGithubRepoInitialized(githubTargetOrg, targetRepo); + } + + public void Dispose() + { + _targetGithubHttpClient?.Dispose(); + _versionClient?.Dispose(); + } +} diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs index 733621b49..e987886f0 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs @@ -1202,6 +1202,117 @@ mutation startRepositoryMigration( expectedRepositoryMigrationId.Should().Be(actualRepositoryMigrationId); } + [Fact] + public async Task StartGitlabMigration_Sends_Null_Metadata_Url() + { + // Arrange + const string migrationSourceId = "MIGRATION_SOURCE_ID"; + const string sourceRepoUrl = "https://gitlab.com/my-group/my-project"; + const string orgId = "ORG_ID"; + const string url = "https://api.github.com/graphql"; + const string gitArchiveUrl = "GIT_ARCHIVE_URL"; + const string targetToken = "TARGET_TOKEN"; + + const string unusedSourceToken = "not-used"; + string nullMetadataArchiveUrl = null; + string nullTargetRepoVisibility = null; + + const string query = @" + mutation startRepositoryMigration( + $sourceId: ID!, + $ownerId: ID!, + $sourceRepositoryUrl: URI!, + $repositoryName: String!, + $continueOnError: Boolean!, + $gitArchiveUrl: String, + $metadataArchiveUrl: String, + $accessToken: String!, + $githubPat: String, + $skipReleases: Boolean, + $targetRepoVisibility: String, + $lockSource: Boolean)"; + const string gql = @" + startRepositoryMigration( + input: { + sourceId: $sourceId, + ownerId: $ownerId, + sourceRepositoryUrl: $sourceRepositoryUrl, + repositoryName: $repositoryName, + continueOnError: $continueOnError, + gitArchiveUrl: $gitArchiveUrl, + metadataArchiveUrl: $metadataArchiveUrl, + accessToken: $accessToken, + githubPat: $githubPat, + skipReleases: $skipReleases, + targetRepoVisibility: $targetRepoVisibility, + lockSource: $lockSource + } + ) { + repositoryMigration { + id, + databaseId, + migrationSource { + id, + name, + type + }, + sourceUrl, + state, + failureReason + } + }"; + var payload = new + { + query = $"{query} {{ {gql} }}", + variables = new + { + sourceId = migrationSourceId, + ownerId = orgId, + sourceRepositoryUrl = sourceRepoUrl, + repositoryName = GITHUB_REPO, + continueOnError = true, + gitArchiveUrl, + metadataArchiveUrl = nullMetadataArchiveUrl, + accessToken = unusedSourceToken, + githubPat = targetToken, + skipReleases = false, + targetRepoVisibility = nullTargetRepoVisibility, + lockSource = false + }, + operationName = "startRepositoryMigration" + }; + const string actualRepositoryMigrationId = "RM_kgC4NjFhNmE2NGU2ZWE1YTQwMDA5ODliZjhi"; + var response = JObject.Parse($@" + {{ + ""data"": {{ + ""startRepositoryMigration"": {{ + ""repositoryMigration"": {{ + ""id"": ""{actualRepositoryMigrationId}"", + ""databaseId"": ""3ba25b34-b23d-43fb-a819-f44414be8dc0"", + ""migrationSource"": {{ + ""id"": ""MS_kgC4NjFhNmE2NDViNWZmOTEwMDA5MTZiMGQw"", + ""name"": ""GitLab Source"", + ""type"": ""GITLAB"" + }}, + ""sourceUrl"": ""{sourceRepoUrl}"", + ""state"": ""QUEUED"", + ""failureReason"": """" + }} + }} + }} + }}"); + + _githubClientMock + .Setup(m => m.PostGraphQLAsync(url, It.Is(x => x.ToJson() == payload.ToJson()), null)) + .ReturnsAsync(response); + + // Act + var expectedRepositoryMigrationId = await _githubApi.StartGitlabMigration(migrationSourceId, sourceRepoUrl, orgId, GITHUB_REPO, targetToken, gitArchiveUrl); + + // Assert + expectedRepositoryMigrationId.Should().Be(actualRepositoryMigrationId); + } + [Fact] public async Task StartMigration_Does_Not_Throw_When_Errors_Is_Empty() { diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs new file mode 100644 index 000000000..f03b8fba6 --- /dev/null +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs @@ -0,0 +1,84 @@ +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.Octoshift.Services; + +public class GitlabApiTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabClient = TestHelpers.CreateMock(); + + private readonly GitlabApi _sut; + + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + + public GitlabApiTests() + { + _sut = new GitlabApi(_mockGitlabClient.Object, GITLAB_SERVER_URL, _mockOctoLogger.Object); + } + + [Fact] + public async Task GetServerVersion_Returns_Server_Version() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0-ee"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = true + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + var (actualVersion, enterprise) = await _sut.GetServerVersion(); + + actualVersion.Should().Be(version); + enterprise.Should().BeTrue(); + } + + [Fact] + public async Task LogServerVersion_Logs_Version_With_Enterprise_Edition() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0-ee"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = true + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + await _sut.LogServerVersion(); + + _mockOctoLogger.Verify(m => m.LogInformation($"GitLab version: {version} (Enterprise Edition)"), Times.Once); + } + + [Fact] + public async Task LogServerVersion_Logs_Version_With_Community_Edition() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = false + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + await _sut.LogServerVersion(); + + _mockOctoLogger.Verify(m => m.LogInformation($"GitLab version: {version} (Community Edition)"), Times.Once); + } +} diff --git a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj index 833ba590f..835eeefa4 100644 --- a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj +++ b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj @@ -32,4 +32,8 @@ + + + + diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs new file mode 100644 index 000000000..1464e8562 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs @@ -0,0 +1,22 @@ +using FluentAssertions; +using OctoshiftCLI.GitlabToGithub.Commands.AbortMigration; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.AbortMigration; + +public class AbortMigrationCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new AbortMigrationCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("abort-migration"); + command.Options.Count.Should().Be(4); + + TestHelpers.VerifyCommandOption(command.Options, "migration-id", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs new file mode 100644 index 000000000..a9a32e501 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs @@ -0,0 +1,23 @@ +using OctoshiftCLI.GitlabToGithub.Commands.CreateTeam; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.CreateTeam; + +public class CreateTeamCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new CreateTeamCommand(); + Assert.NotNull(command); + Assert.Equal("create-team", command.Name); + Assert.Equal(6, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "team-name", true); + TestHelpers.VerifyCommandOption(command.Options, "idp-group", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs new file mode 100644 index 000000000..6c2a3073a --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs @@ -0,0 +1,38 @@ +using System.Linq; +using OctoshiftCLI.GitlabToGithub.Commands.DownloadLogs; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.DownloadLogs; + +public class DownloadLogsCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new DownloadLogsCommand(); + Assert.NotNull(command); + Assert.Equal("download-logs", command.Name); + Assert.Equal(8, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", false); + TestHelpers.VerifyCommandOption(command.Options, "github-repo", false); + TestHelpers.VerifyCommandOption(command.Options, "migration-id", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "migration-log-file", false); + TestHelpers.VerifyCommandOption(command.Options, "overwrite", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + } + + [Fact] + public void Should_Support_Github_Api_Url_Alias_For_Backward_Compatibility() + { + // Test that --github-api-url still works as an alias for --target-api-url + var command = new DownloadLogsCommand(); + var option = command.Options.FirstOrDefault(o => o.Name == "target-api-url"); + + Assert.NotNull(option); + Assert.Contains("--target-api-url", option.Aliases); + Assert.Contains("--github-api-url", option.Aliases); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs new file mode 100644 index 000000000..c7b663412 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs @@ -0,0 +1,23 @@ +using OctoshiftCLI.GitlabToGithub.Commands.GenerateMannequinCsv; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateMannequinCsv; + +public class GenerateMannequinCsvCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new GenerateMannequinCsvCommand(); + Assert.NotNull(command); + Assert.Equal("generate-mannequin-csv", command.Name); + Assert.Equal(6, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "output", false); + TestHelpers.VerifyCommandOption(command.Options, "include-reclaimed", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs new file mode 100644 index 000000000..9e617347e --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs @@ -0,0 +1,41 @@ +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandArgsTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly GenerateScriptCommandArgs _args = new(); + + [Fact] + public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsBucketName = "my-bucket"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + [Fact] + public void It_Throws_If_Both_AwsRegion_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsRegion = "aws-region"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs new file mode 100644 index 000000000..adc2e9790 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -0,0 +1,205 @@ +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandHandlerTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockVersionProvider = new(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + + private readonly GenerateScriptCommandHandler _handler; + + private const string GITHUB_ORG = "GITHUB-ORG"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string OUTPUT = "unit-test-output"; + private const string GROUP_PATH_FOO = "group-foo"; + private const string GROUP_NAME_FOO = "Group Foo"; + private const string GROUP_PATH_BAR = "group-bar"; + private const string GROUP_NAME_BAR = "Group Bar"; + private const string PROJECT_PATH_1 = "project-1"; + private const string PROJECT_NAME_1 = "Project 1"; + private const string PROJECT_PATH_2 = "project-2"; + private const string PROJECT_NAME_2 = "Project 2"; + private const string AWS_BUCKET_NAME = "AWS-BUCKET-NAME"; + private const string AWS_REGION = "us-east-1"; + + public GenerateScriptCommandHandlerTests() + { + _handler = new GenerateScriptCommandHandler( + _mockOctoLogger.Object, + _mockVersionProvider.Object, + _mockFileSystemProvider.Object, + _mockGitlabApi.Object); + } + + [Fact] + public async Task No_Output_Path_Does_Not_Write_File() + { + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task No_Groups_Generates_Header_Only() + { + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT) + }; + + await _handler.Handle(args); + + capturedScript.Should().NotBeNullOrEmpty(); + capturedScript.Should().Contain("VALIDATE_GH_PAT".Replace("VALIDATE_GH_PAT", "GH_PAT")); + capturedScript.Should().Contain("GITLAB_PAT"); + capturedScript.Should().NotContain("# =========== Group:"); + } + + [Fact] + public async Task Default_Generates_Migrate_Repo_Command_For_Each_Project() + { + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] + { + (Id: 1L, Path: GROUP_PATH_FOO, Name: GROUP_NAME_FOO), + (Id: 2L, Path: GROUP_PATH_BAR, Name: GROUP_NAME_BAR) + }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_FOO)) + .ReturnsAsync(new[] + { + (Id: 1L, Path: PROJECT_PATH_1, Name: PROJECT_NAME_1, Archived: false), + (Id: 2L, Path: PROJECT_PATH_2, Name: PROJECT_NAME_2, Archived: false) + }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_BAR)) + .ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name, bool Archived)>()); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT) + }; + + await _handler.Handle(args); + + capturedScript.Should().NotBeNullOrEmpty(); + capturedScript.Should().Contain($"# =========== Group: {GROUP_PATH_FOO} ==========="); + capturedScript.Should().Contain($"# =========== Group: {GROUP_PATH_BAR} ==========="); + capturedScript.Should().Contain($"--gitlab-server-url \"{GITLAB_SERVER_URL}\""); + capturedScript.Should().Contain($"--gitlab-group \"{GROUP_PATH_FOO}\""); + capturedScript.Should().Contain($"--gitlab-project \"{PROJECT_PATH_1}\""); + capturedScript.Should().Contain($"--gitlab-project \"{PROJECT_PATH_2}\""); + capturedScript.Should().Contain($"--github-org \"{GITHUB_ORG}\""); + capturedScript.Should().Contain($"--github-repo \"{GROUP_PATH_FOO}-{PROJECT_PATH_1}\""); + capturedScript.Should().Contain("--target-repo-visibility private"); + capturedScript.Should().Contain("Skipping this group because it has no projects."); + capturedScript.Should().NotContain("--queue-only"); + capturedScript.Should().NotContain("--verbose"); + capturedScript.Should().NotContain("--aws-bucket-name"); + capturedScript.Should().NotContain("--aws-region"); + capturedScript.Should().NotContain("--keep-archive"); + capturedScript.Should().NotContain("--use-github-storage"); + capturedScript.Should().NotContain("--no-ssl-verify"); + capturedScript.Should().NotContain("--kerberos"); + } + + [Fact] + public async Task Includes_Optional_Flags_When_Set() + { + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] { (Id: 1L, Path: GROUP_PATH_FOO, Name: GROUP_NAME_FOO) }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_FOO)) + .ReturnsAsync(new[] { (Id: 1L, Path: PROJECT_PATH_1, Name: PROJECT_NAME_1, Archived: false) }); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT), + Verbose = true, + AwsBucketName = AWS_BUCKET_NAME, + AwsRegion = AWS_REGION, + KeepArchive = true + }; + + await _handler.Handle(args); + + capturedScript.Should().Contain("--verbose"); + capturedScript.Should().Contain($"--aws-bucket-name \"{AWS_BUCKET_NAME}\""); + capturedScript.Should().Contain($"--aws-region \"{AWS_REGION}\""); + capturedScript.Should().Contain("--keep-archive"); + capturedScript.Should().Contain("VALIDATE_AWS_ACCESS_KEY_ID".Replace("VALIDATE_", "").Replace("ID", "ID")); + capturedScript.Should().Contain("AWS_SECRET_ACCESS_KEY"); + capturedScript.Should().NotContain("AZURE_STORAGE_CONNECTION_STRING"); + } + + [Fact] + public async Task UseGithubStorage_Skips_Azure_And_Aws_Validation() + { + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT), + UseGithubStorage = true + }; + + await _handler.Handle(args); + + capturedScript.Should().NotContain("AZURE_STORAGE_CONNECTION_STRING"); + capturedScript.Should().NotContain("AWS_ACCESS_KEY_ID"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs new file mode 100644 index 000000000..7bc9901b3 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -0,0 +1,71 @@ +using System; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + + private readonly Mock _mockServiceProvider = new(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockVersionProvider = new(); + + private readonly GenerateScriptCommand _command = []; + + public GenerateScriptCommandTests() + { + _mockServiceProvider.Setup(m => m.GetService(typeof(OctoLogger))).Returns(_mockOctoLogger.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(EnvironmentVariableProvider))).Returns(_mockEnvironmentVariableProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(IVersionProvider))).Returns(_mockVersionProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object); + } + + [Fact] + public void Should_Have_Options() + { + _command.Should().NotBeNull(); + _command.Name.Should().Be("generate-script"); + _command.Options.Count.Should().Be(14); + + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(_command.Options, "target-uploads-url", false, true); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-group", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-project", false); + TestHelpers.VerifyCommandOption(_command.Options, "output", false); + TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(_command.Options, "aws-bucket-name", false); + TestHelpers.VerifyCommandOption(_command.Options, "aws-region", false); + TestHelpers.VerifyCommandOption(_command.Options, "keep-archive", false); + TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(_command.Options, "use-github-storage", false, true); + } + + [Fact] + public void It_Creates_The_GitlabApi_With_The_Provided_Server_Url_And_Pat() + { + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, false)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs new file mode 100644 index 000000000..f61e6e528 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs @@ -0,0 +1,24 @@ +using OctoshiftCLI.GitlabToGithub.Commands.GrantMigratorRole; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GrantMigratorRole; + +public class GrantMigratorRoleCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new GrantMigratorRoleCommand(); + Assert.NotNull(command); + Assert.Equal("grant-migrator-role", command.Name); + Assert.Equal(7, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "actor", true); + TestHelpers.VerifyCommandOption(command.Options, "actor-type", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs new file mode 100644 index 000000000..ab29c15d5 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs @@ -0,0 +1,130 @@ +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport; + +public class InventoryReportCommandHandlerTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_GROUP = "foo-group"; + private const string GITLAB_PAT = "gitlab-pat"; + private const bool NO_SSL_VERIFY = true; + + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGroupsCsvGenerator = TestHelpers.CreateMock(); + private readonly Mock _mockProjectsCsvGenerator = TestHelpers.CreateMock(); + + private string _groupsCsvOutput = ""; + private string _projectsCsvOutput = ""; + + private readonly InventoryReportCommandHandler _handler; + + public InventoryReportCommandHandlerTests() + { + _handler = new InventoryReportCommandHandler( + TestHelpers.CreateMock().Object, + _mockGitlabApi.Object, + _mockGitlabInspectorService.Object, + _mockGroupsCsvGenerator.Object, + _mockProjectsCsvGenerator.Object) + { + WriteToFile = (path, contents) => + { + if (path == "groups.csv") + { + _groupsCsvOutput = contents; + } + + if (path == "projects.csv") + { + _projectsCsvOutput = contents; + } + + return Task.CompletedTask; + } + }; + } + + [Fact] + public async Task Happy_Path() + { + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; + + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(new[] { (Id: 1L, Path: GITLAB_GROUP, Name: "Foo Group") }); + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(It.IsAny())).ReturnsAsync(1); + + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedProjectsCsv); + + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + NoSslVerify = NO_SSL_VERIFY + }; + await _handler.Handle(args); + + _groupsCsvOutput.Should().Be(expectedGroupsCsv); + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + } + + [Fact] + public async Task Scoped_To_Single_Group() + { + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; + + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(GITLAB_GROUP)).ReturnsAsync(1); + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GITLAB_GROUP, false)).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GITLAB_GROUP, false)).ReturnsAsync(expectedProjectsCsv); + + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabPat = GITLAB_PAT, + NoSslVerify = NO_SSL_VERIFY + }; + await _handler.Handle(args); + + _groupsCsvOutput.Should().Be(expectedGroupsCsv); + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + + _mockGitlabApi.Verify(m => m.GetGroups(), Times.Never); + } + + [Fact] + public async Task It_Generates_Minimal_Csvs_When_Requested() + { + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; + + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(new[] { (Id: 1L, Path: GITLAB_GROUP, Name: "Foo Group") }); + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(It.IsAny())).ReturnsAsync(1); + + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedProjectsCsv); + + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + NoSslVerify = NO_SSL_VERIFY, + Minimal = true + }; + await _handler.Handle(args); + + _groupsCsvOutput.Should().Be(expectedGroupsCsv); + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + + _mockGroupsCsvGenerator.Verify(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, true)); + _mockProjectsCsvGenerator.Verify(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, true)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs new file mode 100644 index 000000000..12e738abf --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs @@ -0,0 +1,67 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommandTests + { + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockGroupsCsvGeneratorService = TestHelpers.CreateMock(); + private readonly Mock _mockProjectsCsvGeneratorService = TestHelpers.CreateMock(); + + private readonly ServiceProvider _serviceProvider; + private readonly InventoryReportCommand _command = []; + + public InventoryReportCommandTests() + { + var serviceCollection = new ServiceCollection(); + serviceCollection + .AddSingleton(_mockOctoLogger.Object) + .AddSingleton(_mockGitlabApiFactory.Object) + .AddSingleton(_mockGitlabInspectorServiceFactory.Object) + .AddSingleton(_mockGroupsCsvGeneratorService.Object) + .AddSingleton(_mockProjectsCsvGeneratorService.Object); + + _serviceProvider = serviceCollection.BuildServiceProvider(); + } + + [Fact] + public void Should_Have_Options() + { + _command.Should().NotBeNull(); + _command.Name.Should().Be("inventory-report"); + _command.Options.Count.Should().Be(6); + + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-group", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(_command.Options, "minimal", false); + TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); + } + + [Fact] + public void BuildHandler_Creates_The_Handler() + { + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = "https://gitlab.contoso.com", + GitlabPat = "gitlab-pat" + }; + + var handler = _command.BuildHandler(args, _serviceProvider); + + handler.Should().NotBeNull(); + _mockGitlabApiFactory.Verify(m => m.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify)); + _mockGitlabInspectorServiceFactory.Verify(m => m.Create(It.IsAny())); + } + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs new file mode 100644 index 000000000..ce94458a6 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs @@ -0,0 +1,269 @@ +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandArgsTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + + private const string ARCHIVE_PATH = "path/to/archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/gitlab-archive.tar"; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_SESSION_TOKEN = "aws-session-token"; + private const string AWS_REGION = "aws-region"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_GROUP = "gitlab-group"; + private const string GITLAB_PROJECT = "gitlab-project"; + private const string GITLAB_PAT = "gitlab-pat"; + + [Fact] + public void It_Throws_When_Neither_Gitlab_Server_Url_Nor_Archive_Source_Is_Provided() + { + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-server-url*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_Both_Archive_Path_And_Archive_Url_Are_Provided() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_Gitlab_Group_Or_Project_Is_Missing_When_Generating_Archive() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-group*--gitlab-project*"); + } + + [Fact] + public void It_Throws_When_Gitlab_Pat_Is_Provided_With_Archive_Path() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GitlabPat = GITLAB_PAT + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-pat*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_No_Ssl_Verify_Is_Provided_With_Archive_Url() + { + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + NoSslVerify = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--no-ssl-verify*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_Aws_Access_Key_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsAccessKey = AWS_ACCESS_KEY_ID + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Secret_Key_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSecretKey = AWS_SECRET_ACCESS_KEY + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Session_Token_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSessionToken = AWS_SESSION_TOKEN + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Region_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsRegion = AWS_REGION + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Use_Github_Storage_Provided_With_Aws_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AwsBucketName = AWS_BUCKET_NAME, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--use-github-storage flag was provided with an AWS S3 Bucket name*"); + } + + [Fact] + public void It_Throws_When_Use_Github_Storage_Provided_With_Azure_Storage_Connection_String() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--use-github-storage flag was provided with a connection string*"); + } + + [Fact] + public void It_Throws_When_Github_Org_Is_Missing_For_Import() + { + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubRepo = GITHUB_REPO + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-org*GitLab archive*"); + } + + [Fact] + public void It_Throws_When_Github_Repo_Is_Missing_For_Import() + { + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-repo*GitLab archive*"); + } + + [Fact] + public void Valid_Generate_And_Upload_Args_Do_Not_Throw() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + GitlabPat = GITLAB_PAT, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().NotThrow(); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs new file mode 100644 index 000000000..994f77a2f --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs @@ -0,0 +1,312 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandHandlerTests +{ + private readonly Mock _mockGithubApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockAzureApi = TestHelpers.CreateMock(); + private readonly Mock _mockAwsApi = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + + private readonly WarningsCountLogger _warningsCountLogger; + private readonly MigrateRepoCommandHandler _handler; + + private const string ARCHIVE_PATH = "/tmp/gitlab-archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/gitlab-archive.tar"; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string GITHUB_PAT = "github-pat"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_REGION = "eu-west-1"; + + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const string GITLAB_GROUP = "gitlab-group"; + private const string GITLAB_PROJECT = "gitlab-project"; + private const string GITLAB_PROJECT_URL = $"{GITLAB_SERVER_URL}/{GITLAB_GROUP}/{GITLAB_PROJECT}"; + private const string UNUSED_REPO_URL = "https://not-used"; + + private const string GITHUB_ORG_ID = "github-org-id"; + private const string MIGRATION_SOURCE_ID = "migration-source-id"; + private const string MIGRATION_ID = "migration-id"; + + public MigrateRepoCommandHandlerTests() + { + _warningsCountLogger = new WarningsCountLogger(_mockOctoLogger.Object); + _handler = new MigrateRepoCommandHandler( + _mockOctoLogger.Object, + _mockGithubApi.Object, + _mockGitlabApi.Object, + _mockEnvironmentVariableProvider.Object, + _mockAzureApi.Object, + _mockAwsApi.Object, + _mockFileSystemProvider.Object, + _warningsCountLogger + ); + + _mockFileSystemProvider.Setup(m => m.FileExists(It.IsAny())).Returns(true); + _mockFileSystemProvider.Setup(m => m.GetTempFileName()).Returns(ARCHIVE_PATH); + } + + [Fact] + public async Task Throws_If_Args_Is_Null() + { + await FluentActions + .Invoking(() => _handler.Handle(null)) + .Should() + .ThrowExactlyAsync(); + } + + [Fact] + public async Task Throws_If_Target_Repo_Already_Exists() + { + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(true); + + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + }; + + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage($"A repository called {GITHUB_ORG}/{GITHUB_REPO} already exists"); + } + + [Fact] + public async Task Generate_Only_Calls_Start_Export_And_Downloads() + { + _mockGitlabApi.Setup(x => x.GetExport(GITLAB_GROUP, GITLAB_PROJECT)) + .ReturnsAsync(("finished", ARCHIVE_URL)); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + }; + + await _handler.Handle(args); + + _mockGitlabApi.Verify(m => m.StartExport(GITLAB_GROUP, GITLAB_PROJECT)); + _mockGitlabApi.Verify(m => m.DownloadExportArchive(GITLAB_GROUP, GITLAB_PROJECT, It.IsAny())); + _mockGithubApi.Verify(m => m.DoesRepoExist(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Ingest_Only_Starts_Gitlab_Migration_With_Unused_Source_Url() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null)) + .ReturnsAsync(MIGRATION_ID); + + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + UNUSED_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null)); + } + + [Fact] + public async Task Passes_Gitlab_Project_Url_When_All_Gitlab_Args_Provided() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, GITLAB_PROJECT_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null)) + .ReturnsAsync(MIGRATION_ID); + _mockGitlabApi.Setup(x => x.GetExport(GITLAB_GROUP, GITLAB_PROJECT)).ReturnsAsync(("finished", ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + using var archiveStream = new MemoryStream(new byte[] { 1, 2, 3 }); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(archiveStream); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + GITLAB_PROJECT_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null)); + } + + [Fact] + public async Task Uploads_To_Aws_When_Aws_Bucket_Name_Provided() + { + _mockEnvironmentVariableProvider.Setup(m => m.AwsAccessKeyId(It.IsAny())).Returns(AWS_ACCESS_KEY_ID); + _mockEnvironmentVariableProvider.Setup(m => m.AwsSecretAccessKey(It.IsAny())).Returns(AWS_SECRET_ACCESS_KEY); + _mockEnvironmentVariableProvider.Setup(m => m.AwsRegion(It.IsAny())).Returns(AWS_REGION); + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())).ReturnsAsync(ARCHIVE_URL); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AwsBucketName = AWS_BUCKET_NAME, + AwsAccessKey = AWS_ACCESS_KEY_ID, + AwsSecretKey = AWS_SECRET_ACCESS_KEY, + AwsRegion = AWS_REGION, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockAwsApi.Verify(m => m.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())); + } + + [Fact] + public async Task Deletes_Archive_By_Default() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + using var archiveStream = new MemoryStream(new byte[] { 1, 2, 3 }); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(archiveStream); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(ARCHIVE_PATH)); + } + + [Fact] + public async Task Keeps_Archive_When_KeepArchive_Set() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + using var archiveStream = new MemoryStream(new byte[] { 1, 2, 3 }); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(archiveStream); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + KeepArchive = true, + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Throws_When_Gitlab_Pat_Not_Provided_For_Generate() + { + string nullGitlabPat = null; + _mockEnvironmentVariableProvider.Setup(m => m.GitlabPat(It.IsAny())).Returns(nullGitlabPat); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + }; + + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*GitLab PAT*GITLAB_PAT*--gitlab-pat*"); + } + + [Fact] + public async Task Throws_When_Archive_Path_Does_Not_Exist() + { + _mockFileSystemProvider.Setup(m => m.FileExists(ARCHIVE_PATH)).Returns(false); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + }; + + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*archive*--archive-path*"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs new file mode 100644 index 000000000..ee8856a7e --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs @@ -0,0 +1,172 @@ +using System; +using System.Net.Http; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Factories; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const string GITHUB_ORG = "github-org"; + private const string GITHUB_PAT = "github-pat"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + + private readonly Mock _mockServiceProvider = new(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockGithubApiFactory = new(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockAzureApiFactory = new(); + private readonly Mock _warningsCountLogger = TestHelpers.CreateMock(); + + private readonly MigrateRepoCommand _command = []; + + public MigrateRepoCommandTests() + { + _mockServiceProvider.Setup(m => m.GetService(typeof(OctoLogger))).Returns(_mockOctoLogger.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(EnvironmentVariableProvider))).Returns(_mockEnvironmentVariableProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(ITargetGithubApiFactory))).Returns(_mockGithubApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(IAzureApiFactory))).Returns(_mockAzureApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(HttpDownloadServiceFactory))) + .Returns(new HttpDownloadServiceFactory( + _mockOctoLogger.Object, + new Mock().Object, + _mockFileSystemProvider.Object, + new Mock().Object)); + _mockServiceProvider.Setup(m => m.GetService(typeof(WarningsCountLogger))).Returns(_warningsCountLogger.Object); + } + + [Fact] + public void Should_Have_Options() + { + var command = new MigrateRepoCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("migrate-repo"); + command.Options.Count.Should().Be(23); + + TestHelpers.VerifyCommandOption(command.Options, "gitlab-server-url", false); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-group", false); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-project", false); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "archive-url", false); + TestHelpers.VerifyCommandOption(command.Options, "archive-path", false); + TestHelpers.VerifyCommandOption(command.Options, "azure-storage-connection-string", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-bucket-name", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-access-key", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-session-token", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-region", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-secret-key", false); + TestHelpers.VerifyCommandOption(command.Options, "github-org", false); + TestHelpers.VerifyCommandOption(command.Options, "github-repo", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "queue-only", false); + TestHelpers.VerifyCommandOption(command.Options, "target-repo-visibility", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "keep-archive", false); + TestHelpers.VerifyCommandOption(command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-uploads-url", false, true); + TestHelpers.VerifyCommandOption(command.Options, "use-github-storage", false, true); + } + + [Fact] + public void BuildHandler_Creates_The_Handler() + { + var args = new MigrateRepoCommandArgs(); + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGithubApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockGitlabApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockAzureApiFactory.Verify(m => m.Create(It.IsAny()), Times.Never); + } + + [Fact] + public void BuildHandler_Creates_GitHub_Api_When_Github_Org_Is_Provided() + { + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubPat = GITHUB_PAT + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGithubApiFactory.Verify(m => m.Create(null, null, GITHUB_PAT)); + } + + [Fact] + public void BuildHandler_Uses_Target_Api_Url_When_Provided() + { + var targetApiUrl = "https://api.github.com"; + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubPat = GITHUB_PAT, + TargetApiUrl = targetApiUrl + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, GITHUB_PAT)); + } + + [Fact] + public void BuildHandler_Creates_Gitlab_Api_When_Gitlab_Server_Url_Is_Provided() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, false)); + } + + [Fact] + public void BuildHandler_Forwards_NoSslVerify_To_Gitlab_Api_Factory() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + NoSslVerify = true + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, true)); + } + + [Fact] + public void BuildHandler_Creates_Azure_Api_When_Connection_String_Is_Provided_Via_Args() + { + var args = new MigrateRepoCommandArgs + { + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockAzureApiFactory.Verify(m => m.Create(AZURE_STORAGE_CONNECTION_STRING)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs new file mode 100644 index 000000000..bdc05d23b --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs @@ -0,0 +1,28 @@ +using OctoshiftCLI.GitlabToGithub.Commands.ReclaimMannequin; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.ReclaimMannequin; + +public class ReclaimMannequinCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new ReclaimMannequinCommand(); + Assert.NotNull(command); + Assert.Equal("reclaim-mannequin", command.Name); + Assert.Equal(11, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "csv", false); + TestHelpers.VerifyCommandOption(command.Options, "mannequin-user", false); + TestHelpers.VerifyCommandOption(command.Options, "mannequin-id", false); + TestHelpers.VerifyCommandOption(command.Options, "target-user", false); + TestHelpers.VerifyCommandOption(command.Options, "force", false); + TestHelpers.VerifyCommandOption(command.Options, "no-prompt", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "skip-invitation", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs new file mode 100644 index 000000000..de7298f2c --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs @@ -0,0 +1,24 @@ +using OctoshiftCLI.GitlabToGithub.Commands.RevokeMigratorRole; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.RevokeMigratorRole; + +public class RevokeMigratorRoleCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new RevokeMigratorRoleCommand(); + Assert.NotNull(command); + Assert.Equal("revoke-migrator-role", command.Name); + Assert.Equal(7, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "actor", true); + TestHelpers.VerifyCommandOption(command.Options, "actor-type", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs new file mode 100644 index 000000000..1c6238b62 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs @@ -0,0 +1,22 @@ +using FluentAssertions; +using OctoshiftCLI.GitlabToGithub.Commands.WaitForMigration; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.WaitForMigration; + +public class WaitForMigrationCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new WaitForMigrationCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("wait-for-migration"); + command.Options.Count.Should().Be(4); + + TestHelpers.VerifyCommandOption(command.Options, "migration-id", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs new file mode 100644 index 000000000..6c8b728c1 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs @@ -0,0 +1,59 @@ +using System.Linq; +using System.Net.Http; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Factories; + +public class GitlabApiFactoryTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockHttpClientFactory = new(); + + private readonly GitlabApiFactory _gitlabApiFactory; + + public GitlabApiFactoryTests() + { + _gitlabApiFactory = new GitlabApiFactory(_mockOctoLogger.Object, _mockHttpClientFactory.Object, _mockEnvironmentVariableProvider.Object, null, null, null); + } + + [Fact] + public void Should_Create_GitlabApi_With_Default() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("Default")) + .Returns(httpClient); + + // Act + var gitlabApi = _gitlabApiFactory.Create(GITLAB_SERVER_URL, "pat"); + + // Assert + gitlabApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } + + [Fact] + public void Should_Create_GitlabApi_With_No_Ssl_Verify() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("NoSSL")) + .Returns(httpClient); + + // Act + var gitlabApi = _gitlabApiFactory.Create(GITLAB_SERVER_URL, "pat", true); + + // Assert + gitlabApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs new file mode 100644 index 000000000..6df183f3a --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs @@ -0,0 +1,106 @@ +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Services; + +public class GitlabInspectorServiceTests +{ + private readonly OctoLogger _logger = TestHelpers.CreateMock().Object; + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly GitlabInspectorService _service; + + private const string GROUP_PATH_1 = "group-1"; + private const string GROUP_NAME_1 = "Group 1"; + private const string GROUP_PATH_2 = "group-2"; + private const string GROUP_NAME_2 = "Group 2"; + + public GitlabInspectorServiceTests() => _service = new(_logger, _mockGitlabApi.Object); + + [Fact] + public async Task GetGroups_Returns_Path_And_Name() + { + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] + { + (Id: 1L, Path: GROUP_PATH_1, Name: GROUP_NAME_1), + (Id: 2L, Path: GROUP_PATH_2, Name: GROUP_NAME_2) + }); + + var result = await _service.GetGroups(); + + result.Should().BeEquivalentTo([(GROUP_PATH_1, GROUP_NAME_1), (GROUP_PATH_2, GROUP_NAME_2)]); + } + + [Fact] + public async Task GetProjects_Returns_Projects_For_Group() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] + { + (Id: 1L, Path: "project-1", Name: "Project 1", Archived: false), + (Id: 2L, Path: "project-2", Name: "Project 2", Archived: true) + }); + + var result = (await _service.GetProjects(GROUP_PATH_1)).ToList(); + + result.Should().HaveCount(2); + result[0].Path.Should().Be("project-1"); + result[0].Name.Should().Be("Project 1"); + result[1].Path.Should().Be("project-2"); + result[1].Name.Should().Be("Project 2"); + } + + [Fact] + public async Task GetProjectCount_For_Group_Returns_Count() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] + { + (Id: 1L, Path: "p1", Name: "p1", Archived: false), + (Id: 2L, Path: "p2", Name: "p2", Archived: false) + }); + + var result = await _service.GetProjectCount(GROUP_PATH_1); + + result.Should().Be(2); + } + + [Fact] + public async Task GetProjectCount_For_Multiple_Groups_Returns_Sum() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] { (Id: 1L, Path: "p1", Name: "p1", Archived: false) }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_2)) + .ReturnsAsync(new[] + { + (Id: 2L, Path: "p2", Name: "p2", Archived: false), + (Id: 3L, Path: "p3", Name: "p3", Archived: false) + }); + + var result = await _service.GetProjectCount(new[] { GROUP_PATH_1, GROUP_PATH_2 }); + + result.Should().Be(3); + } + + [Fact] + public async Task GetProjectMergeRequestCount_Returns_Count_From_Api() + { + _mockGitlabApi + .Setup(m => m.GetMergeRequestCount(GROUP_PATH_1, "project-1")) + .ReturnsAsync(7); + + var result = await _service.GetProjectMergeRequestCount(GROUP_PATH_1, "project-1"); + + result.Should().Be(7); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs new file mode 100644 index 000000000..232d7ef79 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs @@ -0,0 +1,89 @@ +using System; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using Octoshift.Models; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Services; + +public class ProjectsCsvGeneratorServiceTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const bool NO_SSL_VERIFY = true; + + private const string GROUP_PATH = "group-1"; + private const string GROUP_NAME = "Group 1"; + private const string PROJECT_PATH = "project-1"; + private const string PROJECT_NAME = "Project 1"; + + private const string FULL_CSV_HEADER = "group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived,mr-count"; + private const string MINIMAL_CSV_HEADER = "group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived"; + + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + + private readonly ProjectsCsvGeneratorService _service; + + public ProjectsCsvGeneratorServiceTests() + { + _mockGitlabApiFactory.Setup(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + _mockGitlabInspectorServiceFactory.Setup(m => m.Create(_mockGitlabApi.Object)).Returns(_mockGitlabInspectorService.Object); + _service = new ProjectsCsvGeneratorService(_mockGitlabInspectorServiceFactory.Object, _mockGitlabApiFactory.Object); + } + + [Fact] + public async Task Generate_Returns_Csv_For_Single_Group() + { + var lastCommitDate = new DateTimeOffset(2024, 1, 2, 3, 4, 5, TimeSpan.Zero); + const long repoSize = 1234; + const long attachmentsSize = 5678; + const int mrCount = 7; + + _mockGitlabInspectorService.Setup(m => m.GetGroup(GROUP_PATH)).ReturnsAsync((GROUP_PATH, GROUP_NAME)); + _mockGitlabInspectorService + .Setup(m => m.GetProjects(GROUP_PATH)) + .ReturnsAsync(new[] { new GitlabProject { Name = PROJECT_NAME, Path = PROJECT_PATH } }); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(GROUP_PATH, PROJECT_PATH)).ReturnsAsync((repoSize, attachmentsSize)); + _mockGitlabInspectorService.Setup(m => m.GetProjectMergeRequestCount(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(mrCount); + + var result = await _service.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GROUP_PATH); + + var expected = + $"{FULL_CSV_HEADER}{Environment.NewLine}" + + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"False\",{mrCount}{Environment.NewLine}"; + + result.Should().Be(expected); + } + + [Fact] + public async Task Generate_Returns_Minimal_Csv_When_Requested() + { + const long repoSize = 1234; + const long attachmentsSize = 5678; + var lastCommitDate = new DateTimeOffset(2024, 1, 2, 3, 4, 5, TimeSpan.Zero); + + _mockGitlabInspectorService.Setup(m => m.GetGroup(GROUP_PATH)).ReturnsAsync((GROUP_PATH, GROUP_NAME)); + _mockGitlabInspectorService + .Setup(m => m.GetProjects(GROUP_PATH)) + .ReturnsAsync(new[] { new GitlabProject { Name = PROJECT_NAME, Path = PROJECT_PATH } }); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(GROUP_PATH, PROJECT_PATH)).ReturnsAsync((repoSize, attachmentsSize)); + + var result = await _service.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GROUP_PATH, minimal: true); + + var expected = + $"{MINIMAL_CSV_HEADER}{Environment.NewLine}" + + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"False\"{Environment.NewLine}"; + + result.Should().Be(expected); + _mockGitlabInspectorService.Verify(m => m.GetProjectMergeRequestCount(It.IsAny(), It.IsAny()), Times.Never); + } +} diff --git a/src/OctoshiftCLI.sln b/src/OctoshiftCLI.sln index 72aee8119..17c0dcb4b 100644 --- a/src/OctoshiftCLI.sln +++ b/src/OctoshiftCLI.sln @@ -21,6 +21,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OctoshiftCLI.IntegrationTes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "bbs2gh", "bbs2gh\bbs2gh.csproj", "{39EE734C-AC4A-42AC-801A-ECCA661C23A1}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gl2gh", "gl2gh\gl2gh.csproj", "{180D1E3B-CA23-4603-BE00-9C856E0665CF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +53,10 @@ Global {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Debug|Any CPU.Build.0 = Debug|Any CPU {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Release|Any CPU.ActiveCfg = Release|Any CPU {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Release|Any CPU.Build.0 = Release|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs b/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs new file mode 100644 index 000000000..06fc255ec --- /dev/null +++ b/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.AbortMigration; + +namespace OctoshiftCLI.GitlabToGithub.Commands.AbortMigration; + +public sealed class AbortMigrationCommand : AbortMigrationCommandBase +{ + public AbortMigrationCommand() : base() => AddOptions(); +} diff --git a/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs b/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs new file mode 100644 index 000000000..56911ff57 --- /dev/null +++ b/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.CreateTeam; + +namespace OctoshiftCLI.GitlabToGithub.Commands.CreateTeam; + +public sealed class CreateTeamCommand : CreateTeamCommandBase +{ + public CreateTeamCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs b/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs new file mode 100644 index 000000000..52adb4ce4 --- /dev/null +++ b/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs @@ -0,0 +1,17 @@ +using System.Runtime.CompilerServices; +using OctoshiftCLI.Commands.DownloadLogs; + +[assembly: InternalsVisibleTo("OctoshiftCLI.Tests")] + +namespace OctoshiftCLI.GitlabToGithub.Commands.DownloadLogs; + +public sealed class DownloadLogsCommand : DownloadLogsCommandBase +{ + public DownloadLogsCommand() + { + // Add backward compatibility alias for --github-api-url + GithubApiUrl.AddAlias("--github-api-url"); + + AddOptions(); + } +} diff --git a/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs b/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs new file mode 100644 index 000000000..7632ec845 --- /dev/null +++ b/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.GenerateMannequinCsv; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateMannequinCsv; + +public sealed class GenerateMannequinCsvCommand : GenerateMannequinCsvCommandBase +{ + public GenerateMannequinCsvCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs new file mode 100644 index 000000000..e18afacae --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -0,0 +1,118 @@ +using System; +using System.CommandLine; +using System.IO; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommand : CommandBase +{ + public GenerateScriptCommand() : base( + name: "generate-script", + description: "Generates a migration script. This provides you the ability to review the steps that this tool will take, and optionally modify the script if desired before running it.") + { + AddOption(GitlabServerUrl); + AddOption(GithubOrg); + AddOption(TargetApiUrl); + AddOption(TargetUploadsUrl); + AddOption(GitlabPat); + AddOption(GitlabGroup); + AddOption(GitlabProject); + AddOption(Output); + AddOption(Verbose); + AddOption(AwsBucketName); + AddOption(AwsRegion); + AddOption(KeepArchive); + AddOption(NoSslVerify); + AddOption(UseGithubStorage); + } + + public Option GitlabServerUrl { get; } = new( + name: "--gitlab-server-url", + description: "The full URL of the GitLab server to migrate from, e.g. https://gitlab.mycompany.com") + { IsRequired = true }; + + public Option GitlabPat { get; } = new( + name: "--gitlab-pat", + description: "The GitLab PAT of a user with admin privileges to get the list of all groups and their projects. If not set will be read from GITLAB_PAT environment variable." + + $"{Environment.NewLine}" + + "Note: The PAT will not get included in the generated script and it has to be set as an env variable before running the script."); + + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group to migrate. If not set will migrate all groups the user has access to."); + + public Option GitlabProject { get; } = new( + name: "--gitlab-project", + description: "The GitLab project to migrate. Requires --gitlab-group. If not set will migrate all projects in the group."); + + public Option GithubOrg { get; } = new("--github-org") + { IsRequired = true }; + + public Option Output { get; } = new( + name: "--output", + getDefaultValue: () => new FileInfo("./migrate.ps1")); + + public Option AwsBucketName { get; } = new( + name: "--aws-bucket-name", + description: "If using AWS, the name of the S3 bucket to upload the GitLab archive to."); + + public Option AwsRegion { get; } = new( + name: "--aws-region", + description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " + + "Required if using AWS."); + + public Option Verbose { get; } = new("--verbose"); + + public Option KeepArchive { get; } = new( + name: "--keep-archive", + description: "Keeps the downloaded export archive after successfully uploading it. By default, it will be automatically deleted."); + + public Option TargetApiUrl { get; } = new("--target-api-url") + { + Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" + }; + + public Option TargetUploadsUrl { get; } = new( + name: "--target-uploads-url", + description: "The URL of the target uploads API, if not migrating to github.com. Defaults to https://uploads.github.com") + { IsHidden = true }; + + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your GitLab instance. All other migration steps will continue to verify SSL. " + + "If your GitLab instance has a self-signed SSL certificate, this flag will allow the migration archive to be exported."); + + public Option UseGithubStorage { get; } = new("--use-github-storage") + { + IsHidden = true, + Description = "Enables multipart uploads to a GitHub owned storage for use during migration. " + + "Configure chunk size with the GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES environment variable (default: 100 MiB, minimum: 5 MiB).", + }; + + public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var versionProvider = sp.GetRequiredService(); + var fileSystemProvider = sp.GetRequiredService(); + + var gitlabApiFactory = sp.GetRequiredService(); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + + return new GenerateScriptCommandHandler(log, versionProvider, fileSystemProvider, gitlabApi); + } +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs new file mode 100644 index 000000000..c03cbdb6f --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -0,0 +1,42 @@ +using System.IO; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandArgs : CommandArgs +{ + public string GitlabServerUrl { get; set; } + public string GithubOrg { get; set; } + [Secret] + public string GitlabPat { get; set; } + public string GitlabGroup { get; set; } + public string GitlabProject { get; set; } + public bool NoSslVerify { get; set; } + public FileInfo Output { get; set; } + public string AwsBucketName { get; set; } + public string AwsRegion { get; set; } + public bool KeepArchive { get; set; } + public string TargetApiUrl { get; set; } + public string TargetUploadsUrl { get; set; } + public bool UseGithubStorage { get; set; } + + public override void Validate(OctoLogger log) + { + if (GitlabProject.HasValue() && GitlabGroup.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--gitlab-group must be provided when --gitlab-project is specified."); + } + + if (AwsBucketName.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + if (AwsRegion.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } + } +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs new file mode 100644 index 000000000..67265a069 --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -0,0 +1,184 @@ +using System; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandHandler : ICommandHandler +{ + private readonly OctoLogger _log; + private readonly IVersionProvider _versionProvider; + private readonly FileSystemProvider _fileSystemProvider; + private readonly GitlabApi _gitlabApi; + + public GenerateScriptCommandHandler( + OctoLogger log, + IVersionProvider versionProvider, + FileSystemProvider fileSystemProvider, + GitlabApi gitlabApi) + { + _log = log; + _versionProvider = versionProvider; + _fileSystemProvider = fileSystemProvider; + _gitlabApi = gitlabApi; + } + + public async Task Handle(GenerateScriptCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + _log.LogInformation("Generating Script..."); + + await _gitlabApi.LogServerVersion(); + + var script = await GenerateScript(args); + + if (script.HasValue() && args.Output.HasValue()) + { + await _fileSystemProvider.WriteAllTextAsync(args.Output.FullName, script); + } + } + + private async Task GenerateScript(GenerateScriptCommandArgs args) + { + var content = new StringBuilder(); + content.AppendLine(PWSH_SHEBANG); + content.AppendLine(); + content.AppendLine(VersionComment); + content.AppendLine(EXEC_FUNCTION_BLOCK); + + content.AppendLine(VALIDATE_GH_PAT); + content.AppendLine(VALIDATE_GITLAB_PAT); + if (args.AwsBucketName.HasValue() || args.AwsRegion.HasValue()) + { + content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID); + content.AppendLine(VALIDATE_AWS_SECRET_ACCESS_KEY); + } + else if (!args.UseGithubStorage) + { + content.AppendLine(VALIDATE_AZURE_STORAGE_CONNECTION_STRING); + } + + var groups = args.GitlabGroup.HasValue() + ? new[] { args.GitlabGroup } + : (await _gitlabApi.GetGroups()).Select(x => x.Path); + + foreach (var groupPath in groups) + { + _log.LogInformation($"Group: {groupPath}"); + + content.AppendLine(); + content.AppendLine($"# =========== Group: {groupPath} ==========="); + + var projects = await _gitlabApi.GetProjects(groupPath); + + if (args.GitlabProject.HasValue()) + { + projects = projects.Where(p => p.Path == args.GitlabProject).ToArray(); + } + + if (!projects.Any()) + { + content.AppendLine("# Skipping this group because it has no projects."); + continue; + } + + content.AppendLine(); + + foreach (var (_, projectPath, projectName, _) in projects) + { + _log.LogInformation($" Project: {projectName}"); + + content.AppendLine(Exec(MigrateGithubRepoScript(args, groupPath, projectPath, true))); + } + } + + return content.ToString(); + } + + private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string gitlabGroup, string gitlabProject, bool wait) + { + var gitlabServerUrlOption = $" --gitlab-server-url \"{args.GitlabServerUrl}\""; + var gitlabGroupOption = $" --gitlab-group \"{gitlabGroup}\""; + var gitlabProjectOption = $" --gitlab-project \"{gitlabProject}\""; + var githubOrgOption = $" --github-org \"{args.GithubOrg}\""; + var githubRepoOption = $" --github-repo \"{GetGithubRepoName(gitlabGroup, gitlabProject)}\""; + var waitOption = wait ? "" : " --queue-only"; + var verboseOption = args.Verbose ? " --verbose" : ""; + var awsBucketNameOption = args.AwsBucketName.HasValue() ? $" --aws-bucket-name \"{args.AwsBucketName}\"" : ""; + var awsRegionOption = args.AwsRegion.HasValue() ? $" --aws-region \"{args.AwsRegion}\"" : ""; + var keepArchive = args.KeepArchive ? " --keep-archive" : ""; + var noSslVerifyOption = args.NoSslVerify ? " --no-ssl-verify" : ""; + var targetRepoVisibility = " --target-repo-visibility private"; + var targetApiUrlOption = args.TargetApiUrl.HasValue() ? $" --target-api-url \"{args.TargetApiUrl}\"" : ""; + var targetUploadsUrlOption = args.TargetUploadsUrl.HasValue() ? $" --target-uploads-url \"{args.TargetUploadsUrl}\"" : ""; + var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; + + return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{gitlabServerUrlOption}{gitlabGroupOption}{gitlabProjectOption}" + + $"{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerifyOption}{targetRepoVisibility}{githubStorageOption}"; + } + + private string Exec(string script) => Wrap(script, "Exec"); + + private string Wrap(string script, string outerCommand = "") => script.IsNullOrWhiteSpace() ? string.Empty : $"{outerCommand} {{ {script} }}".Trim(); + + private string GetGithubRepoName(string gitlabGroup, string gitlabProject) => $"{gitlabGroup}-{gitlabProject}".ReplaceInvalidCharactersWithDash(); + + private string VersionComment => $"# =========== Created with CLI version {_versionProvider.GetCurrentVersion()} ==========="; + + private const string PWSH_SHEBANG = "#!/usr/bin/env pwsh"; + + private const string EXEC_FUNCTION_BLOCK = @" +function Exec { + param ( + [scriptblock]$ScriptBlock + ) + & @ScriptBlock + if ($lastexitcode -ne 0) { + exit $lastexitcode + } +}"; + private const string VALIDATE_GH_PAT = @" +if (-not $env:GH_PAT) { + Write-Error ""GH_PAT environment variable must be set to a valid GitHub Personal Access Token with the appropriate scopes. For more information see https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer#creating-a-personal-access-token-for-github-enterprise-importer"" + exit 1 +} else { + Write-Host ""GH_PAT environment variable is set and will be used to authenticate to GitHub."" +}"; + private const string VALIDATE_GITLAB_PAT = @" +if (-not $env:GITLAB_PAT) { + Write-Error ""GITLAB_PAT environment variable must be set to a valid PAT that will be used to call the GitLab API to generate a migration archive."" + exit 1 +} else { + Write-Host ""GITLAB_PAT environment variable is set and will be used to authenticate to the GitLab API."" +}"; + private const string VALIDATE_AZURE_STORAGE_CONNECTION_STRING = @" +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + private const string VALIDATE_AWS_ACCESS_KEY_ID = @" +if (-not $env:AWS_ACCESS_KEY_ID) { + Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + private const string VALIDATE_AWS_SECRET_ACCESS_KEY = @" +if (-not $env:AWS_SECRET_ACCESS_KEY) { + Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; +} diff --git a/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs b/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs new file mode 100644 index 000000000..fdd3ec62e --- /dev/null +++ b/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.GrantMigratorRole; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GrantMigratorRole; + +public sealed class GrantMigratorRoleCommand : GrantMigratorRoleCommandBase +{ + public GrantMigratorRoleCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs new file mode 100644 index 000000000..6fac798f6 --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -0,0 +1,78 @@ +using System; +using System.CommandLine; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.Commands; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommand : CommandBase + { + public InventoryReportCommand() : base( + name: "inventory-report", + description: "Generates several CSV files containing lists of GitLab groups and projects. Useful for planning large migrations. Personal projects owned by individual users will not be included." + + Environment.NewLine + + "Note: Expects GITLAB_PAT env variable or --gitlab-pat options to be set.") + { + AddOption(GitlabServerUrl); + AddOption(GitlabGroup); + AddOption(GitlabPat); + AddOption(NoSslVerify); + AddOption(Minimal); + AddOption(Verbose); + } + + public Option GitlabServerUrl { get; } = new( + name: "--gitlab-server-url", + description: "The full URL of the GitLab server, e.g. https://gitlab.mycompany.com") + { IsRequired = true }; + + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group. Iterates over all projects that the user has access to if not provided."); + + public Option GitlabPat { get; } = new( + name: "--gitlab-pat", + description: "The GitLab PAT. If not passed, it will read the PAT from the GITLAB_PAT environment variable."); + + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your GitLab instance. " + + "If your GitLab instance has a self-signed SSL certificate then setting this flag will allow data to be extracted."); + + public Option Minimal { get; } = new( + name: "--minimal", + description: "Omit the MR count from group and project reports for quicker report generation."); + + public Option Verbose { get; } = new("--verbose"); + + public override InventoryReportCommandHandler BuildHandler(InventoryReportCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var gitlabApiFactory = sp.GetRequiredService(); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + var gitlabInspectorServiceFactory = sp.GetRequiredService(); + var gitlabInspectorService = gitlabInspectorServiceFactory.Create(gitlabApi); + var groupsCsvGeneratorService = sp.GetRequiredService(); + var projectsCsvGeneratorService = sp.GetRequiredService(); + + return new InventoryReportCommandHandler( + log, + gitlabApi, + gitlabInspectorService, + groupsCsvGeneratorService, + projectsCsvGeneratorService); + } + } +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs new file mode 100644 index 000000000..e7ba94096 --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs @@ -0,0 +1,14 @@ +using OctoshiftCLI.Commands; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommandArgs : CommandArgs + { + public string GitlabServerUrl { get; set; } + public string GitlabGroup { get; set; } + [Secret] + public string GitlabPat { get; set; } + public bool NoSslVerify { get; set; } + public bool Minimal { get; set; } + } +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs new file mode 100644 index 000000000..c984cc533 --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -0,0 +1,68 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; + +public class InventoryReportCommandHandler : ICommandHandler +{ + internal Func WriteToFile = async (path, contents) => await File.WriteAllTextAsync(path, contents); + + private readonly OctoLogger _log; + private readonly GitlabApi _gitlabApi; + private readonly GitlabInspectorService _gitlabInspectorService; + private readonly GroupsCsvGeneratorService _groupsCsvGenerator; + private readonly ProjectsCsvGeneratorService _projectsCsvGenerator; + + public InventoryReportCommandHandler( + OctoLogger log, + GitlabApi gitlabApi, + GitlabInspectorService gitlabInspectorService, + GroupsCsvGeneratorService groupsCsvGeneratorService, + ProjectsCsvGeneratorService projectsCsvGeneratorService) + { + _log = log; + _gitlabApi = gitlabApi; + _gitlabInspectorService = gitlabInspectorService; + _groupsCsvGenerator = groupsCsvGeneratorService; + _projectsCsvGenerator = projectsCsvGeneratorService; + } + + public async Task Handle(InventoryReportCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + _log.LogInformation("Creating inventory report..."); + + await _gitlabApi.LogServerVersion(); + + var groupPaths = Array.Empty(); + if (string.IsNullOrWhiteSpace(args.GitlabGroup)) + { + _log.LogInformation("Finding Groups..."); + var groups = await _gitlabApi.GetGroups(); + groupPaths = groups.Select(x => x.Path).ToArray(); + _log.LogInformation($"Found {groups.Count()} Groups"); + } + + _log.LogInformation("Finding Projects..."); + var projectCount = string.IsNullOrWhiteSpace(args.GitlabGroup) ? await _gitlabInspectorService.GetProjectCount(groupPaths) : await _gitlabInspectorService.GetProjectCount(args.GitlabGroup); + _log.LogInformation($"Found {projectCount} Projects"); + + _log.LogInformation("Generating data for groups.csv..."); + var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify, args.GitlabGroup, args.Minimal); + await WriteToFile("groups.csv", groupsCsvText); + _log.LogSuccess("groups.csv generated"); + + _log.LogInformation("Generating projects.csv..."); + var projectsCsvText = await _projectsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify, args.GitlabGroup, args.Minimal); + await WriteToFile("projects.csv", projectsCsvText); + _log.LogSuccess("projects.csv generated"); + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs new file mode 100644 index 000000000..db6d93b66 --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -0,0 +1,186 @@ +using System; +using System.CommandLine; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Factories; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommand : CommandBase +{ + public MigrateRepoCommand() : base( + name: "migrate-repo", + description: "Import a GitLab archive to GitHub." + + Environment.NewLine + + "Note: Expects GH_PAT env variable or --github-pat option to be set.") + { + AddOption(ArchiveUrl); + AddOption(GithubOrg); + AddOption(GithubRepo); + AddOption(GithubPat); + AddOption(GitlabServerUrl); + AddOption(GitlabGroup); + AddOption(GitlabProject); + AddOption(GitlabPat); + AddOption(ArchivePath); + AddOption(AzureStorageConnectionString); + AddOption(AwsBucketName); + AddOption(AwsAccessKey); + AddOption(AwsSecretKey); + AddOption(AwsSessionToken); + AddOption(AwsRegion); + AddOption(QueueOnly); + AddOption(TargetRepoVisibility.FromAmong("public", "private", "internal")); + AddOption(Verbose); + AddOption(KeepArchive); + AddOption(NoSslVerify); + AddOption(TargetApiUrl); + AddOption(TargetUploadsUrl); + AddOption(UseGithubStorage); + } + + public Option GitlabServerUrl { get; } = new( + name: "--gitlab-server-url", + description: "The full URL of the GitLab server, e.g. https://gitlab.mycompany.com"); + + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group (full namespace path) that contains the project to migrate. For nested subgroups, use the full path, e.g. parent-group/subgroup."); + + public Option GitlabProject { get; } = new( + name: "--gitlab-project", + description: "The GitLab project to migrate."); + + public Option GitlabPat { get; } = new( + name: "--gitlab-pat", + description: "The GitLab PAT. If not passed, it will read the PAT from the GITLAB_PAT environment variable."); + + public Option ArchiveUrl { get; } = new( + name: "--archive-url", + description: + "URL used to download the GitLab migration archive. Only needed if you want to manually retrieve the archive from GitLab instead of letting this CLI do that for you."); + + public Option ArchivePath { get; } = new( + name: "--archive-path", + description: "Path to the GitLab migration archive on disk. When --gitlab-server-url is provided, the generated archive will be written to this path (overwriting any existing file)."); + + public Option AzureStorageConnectionString { get; } = new( + name: "--azure-storage-connection-string", + description: "A connection string for an Azure Storage account, used to upload the GitLab archive. If not passed, it will read the AZURE_STORAGE_CONNECTION_STRING environment variable."); + + public Option AwsBucketName { get; } = new( + name: "--aws-bucket-name", + description: "If using AWS, the name of the S3 bucket to upload the GitLab archive to."); + + public Option AwsAccessKey { get; } = new( + name: "--aws-access-key", + description: "If uploading to S3, the AWS access key. If not provided, it will be read from AWS_ACCESS_KEY_ID environment variable."); + + public Option AwsSecretKey { get; } = new( + name: "--aws-secret-key", + description: "If uploading to S3, the AWS secret key. If not provided, it will be read from AWS_SECRET_ACCESS_KEY environment variable."); + + public Option AwsSessionToken { get; } = new( + name: "--aws-session-token", + description: "If using AWS, the AWS session token. If not provided, it will be read from AWS_SESSION_TOKEN environment variable."); + + public Option AwsRegion { get; } = new( + name: "--aws-region", + description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " + + "Required if using AWS."); + + public Option GithubOrg { get; } = new("--github-org"); + + public Option GithubRepo { get; } = new("--github-repo"); + + public Option GithubPat { get; } = new( + name: "--github-pat", + description: "The GitHub personal access token to be used for the migration. If not set will be read from GH_PAT environment variable."); + + public Option QueueOnly { get; } = new( + name: "--queue-only", + description: "Only queues the migration, does not wait for it to finish. Use the wait-for-migration command to subsequently wait for it to finish and view the status."); + + public Option TargetRepoVisibility { get; } = new( + name: "--target-repo-visibility", + description: "The visibility of the target repo. Defaults to private. Valid values are public, private, or internal."); + + public Option Verbose { get; } = new("--verbose"); + + public Option KeepArchive { get; } = new( + name: "--keep-archive", + description: "Keeps the downloaded export archive after successfully uploading it. By default, it will be automatically deleted."); + public Option TargetApiUrl { get; } = new("--target-api-url") + { + Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" + }; + public Option TargetUploadsUrl { get; } = new( + name: "--target-uploads-url", + description: "The URL of the target uploads API, if not migrating to github.com. Defaults to https://uploads.github.com") + { IsHidden = true }; + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your GitLab instance. All other migration steps will continue to verify SSL. " + + "If your GitLab instance has a self-signed SSL certificate, this flag will allow the migration archive to be exported."); + public Option UseGithubStorage { get; } = new( + name: "--use-github-storage", + description: "Enables multipart uploads to a GitHub owned storage for use during migration. " + + "Configure chunk size with the GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES environment variable (default: 100 MiB, minimum: 5 MiB).") + { IsHidden = true }; + + public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var environmentVariableProvider = sp.GetRequiredService(); + var fileSystemProvider = sp.GetRequiredService(); + + GithubApi githubApi = null; + GitlabApi gitlabApi = null; + AzureApi azureApi = null; + AwsApi awsApi = null; + + if (args.GithubOrg.HasValue()) + { + var githubApiFactory = sp.GetRequiredService(); + githubApi = githubApiFactory.Create(args.TargetApiUrl, args.TargetUploadsUrl, args.GithubPat); + } + + if (args.GitlabServerUrl.HasValue()) + { + var gitlabApiFactory = sp.GetRequiredService(); + + gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + } + + var azureStorageConnectionString = args.AzureStorageConnectionString ?? environmentVariableProvider.AzureStorageConnectionString(false); + if (azureStorageConnectionString.HasValue()) + { + var azureApiFactory = sp.GetRequiredService(); + azureApi = azureApiFactory.Create(azureStorageConnectionString); + } + + if (args.AwsBucketName.HasValue()) + { + var awsApiFactory = sp.GetRequiredService(); + awsApi = awsApiFactory.Create(args.AwsRegion, args.AwsAccessKey, args.AwsSecretKey, args.AwsSessionToken); + } + + var warningsCountLogger = sp.GetRequiredService(); + + return new MigrateRepoCommandHandler(log, githubApi, gitlabApi, environmentVariableProvider, azureApi, awsApi, fileSystemProvider, warningsCountLogger); + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs new file mode 100644 index 000000000..603dfb36e --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -0,0 +1,132 @@ +using System.Linq; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandArgs : CommandArgs +{ + public string ArchiveUrl { get; set; } + public string ArchivePath { get; set; } + + [Secret] + public string AzureStorageConnectionString { get; set; } + + public string AwsBucketName { get; set; } + [Secret] + public string AwsAccessKey { get; set; } + [Secret] + public string AwsSecretKey { get; set; } + [Secret] + public string AwsSessionToken { get; set; } + public string AwsRegion { get; set; } + + public string GithubOrg { get; set; } + public string GithubRepo { get; set; } + [Secret] + public string GithubPat { get; set; } + public bool QueueOnly { get; set; } + public string TargetRepoVisibility { get; set; } + public string TargetApiUrl { get; set; } + public string TargetUploadsUrl { get; set; } + + public string GitlabServerUrl { get; set; } + public string GitlabGroup { get; set; } + public string GitlabProject { get; set; } + [Secret] + public string GitlabPat { get; set; } + public bool NoSslVerify { get; set; } + + public bool KeepArchive { get; set; } + public bool UseGithubStorage { get; set; } + + public override void Validate(OctoLogger log) + { + if (!GitlabServerUrl.HasValue() && !ArchiveUrl.HasValue() && !ArchivePath.HasValue()) + { + throw new OctoshiftCliException("Either --gitlab-server-url, --archive-path, or --archive-url must be specified."); + } + + if (ArchivePath.HasValue() && ArchiveUrl.HasValue()) + { + throw new OctoshiftCliException("Only one of --archive-path or --archive-url can be specified."); + } + + if (ShouldGenerateArchive()) + { + ValidateGenerateOptions(); + } + else + { + ValidateNoGenerateOptions(); + } + + if (ShouldUploadArchive()) + { + ValidateUploadOptions(); + } + + if (ShouldImportArchive()) + { + ValidateImportOptions(); + } + } + + private void ValidateNoGenerateOptions() + { + if (GitlabPat.HasValue()) + { + throw new OctoshiftCliException("--gitlab-pat cannot be provided with --archive-path or --archive-url."); + } + + if (NoSslVerify) + { + throw new OctoshiftCliException("--no-ssl-verify cannot be provided with --archive-path or --archive-url."); + } + } + + public bool ShouldGenerateArchive() => GitlabServerUrl.HasValue() && !ArchiveUrl.HasValue(); + + public bool ShouldUploadArchive() => ArchiveUrl.IsNullOrWhiteSpace() && GithubOrg.HasValue(); + + // NOTE: ArchiveUrl doesn't necessarily refer to the value passed in by the user to the CLI - it is set during CLI runtime when an archive is uploaded to blob storage + public bool ShouldImportArchive() => ArchiveUrl.HasValue() || GithubOrg.HasValue(); + + private void ValidateGenerateOptions() + { + if (GitlabGroup.IsNullOrWhiteSpace() || GitlabProject.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("Both --gitlab-group and --gitlab-project must be provided."); + } + } + + private void ValidateUploadOptions() + { + if (AwsBucketName.IsNullOrWhiteSpace() && new[] { AwsAccessKey, AwsSecretKey, AwsSessionToken, AwsRegion }.Any(x => x.HasValue())) + { + throw new OctoshiftCliException("The AWS S3 bucket name must be provided with --aws-bucket-name if other AWS S3 upload options are set."); + } + if (UseGithubStorage && AwsBucketName.HasValue()) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + if (AzureStorageConnectionString.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with a connection string for an Azure storage account. Archive cannot be uploaded to both locations."); + } + } + + private void ValidateImportOptions() + { + if (GithubOrg.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--github-org must be provided in order to import the GitLab archive."); + } + + if (GithubRepo.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--github-repo must be provided in order to import the GitLab archive."); + } + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs new file mode 100644 index 000000000..da7fb951b --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -0,0 +1,343 @@ +using System; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandHandler : ICommandHandler +{ + private readonly OctoLogger _log; + private readonly GithubApi _githubApi; + private readonly GitlabApi _gitlabApi; + private readonly AzureApi _azureApi; + private readonly AwsApi _awsApi; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly FileSystemProvider _fileSystemProvider; + private readonly WarningsCountLogger _warningsCountLogger; + private const int CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS = 10000; + private const int CHECK_MIGRATION_STATUS_DELAY_IN_MILLISECONDS = 60000; + + public MigrateRepoCommandHandler( + OctoLogger log, + GithubApi githubApi, + GitlabApi gitlabApi, + EnvironmentVariableProvider environmentVariableProvider, + AzureApi azureApi, + AwsApi awsApi, + FileSystemProvider fileSystemProvider, + WarningsCountLogger warningsCountLogger) + { + _log = log; + _githubApi = githubApi; + _gitlabApi = gitlabApi; + _azureApi = azureApi; + _awsApi = awsApi; + _environmentVariableProvider = environmentVariableProvider; + _fileSystemProvider = fileSystemProvider; + _warningsCountLogger = warningsCountLogger; + } + + public async Task Handle(MigrateRepoCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + ValidateOptions(args); + + await _gitlabApi.LogServerVersion(); + + var migrationSourceId = ""; + + if (args.ShouldImportArchive()) + { + var targetRepoExists = await _githubApi.DoesRepoExist(args.GithubOrg, args.GithubRepo); + + if (targetRepoExists) + { + throw new OctoshiftCliException($"A repository called {args.GithubOrg}/{args.GithubRepo} already exists"); + } + + migrationSourceId = await CreateMigrationSource(args); + } + + if (args.ShouldGenerateArchive()) + { + await GenerateArchive(args); + + _log.LogInformation($"Downloading GitLab archive..."); + + args.ArchivePath ??= _fileSystemProvider.GetTempFileName(); + await _gitlabApi.DownloadExportArchive(args.GitlabGroup, args.GitlabProject, args.ArchivePath); + + _log.LogInformation(args.KeepArchive ? $"Archive downloaded to \"{args.ArchivePath}\"" : "Archive download complete"); + } + + if (args.ShouldUploadArchive()) + { + _log.LogInformation($"Archive path: {args.ArchivePath}"); + + try + { + if (args.UseGithubStorage) + { + args.ArchiveUrl = await UploadArchiveToGithub(args.GithubOrg, args.ArchivePath); + } +#pragma warning disable IDE0045 + else if (args.AwsBucketName.HasValue()) +#pragma warning restore IDE0045 + { + args.ArchiveUrl = await UploadArchiveToAws(args.AwsBucketName, args.ArchivePath); + } + else + { + args.ArchiveUrl = await UploadArchiveToAzure(args.ArchivePath); + } + + } + finally + { + if (!args.KeepArchive) + { + DeleteArchive(args.ArchivePath); + } + } + } + + if (args.ShouldImportArchive()) + { + await ImportArchive(args, migrationSourceId, args.ArchiveUrl); + } + } + + private void DeleteArchive(string path) + { + try + { + _fileSystemProvider.DeleteIfExists(path); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + _log.LogWarning($"Couldn't delete the downloaded archive. Error message: \"{ex.Message}\""); + _log.LogVerbose(ex.ToString()); + } + } + + private async Task GenerateArchive(MigrateRepoCommandArgs args) + { + await _gitlabApi.StartExport(args.GitlabGroup, args.GitlabProject); + + _log.LogInformation($"Export started."); + + var (exportState, archiveUrl) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); + + while (ExportState.IsInProgress(exportState)) + { + _log.LogInformation($"Export status: {exportState}."); + await Task.Delay(CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS); + (exportState, archiveUrl) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); + } + + if (ExportState.IsError(exportState)) + { + throw new OctoshiftCliException($"GitLab archive export failed!"); + } + + _log.LogInformation($"Archive export completed."); + + return archiveUrl; + } + + private async Task UploadArchiveToAzure(string archivePath) + { + _log.LogInformation("Uploading Archive to Azure..."); + +#pragma warning disable IDE0063 + await using (var archiveData = _fileSystemProvider.OpenRead(archivePath)) +#pragma warning restore IDE0063 + { + var archiveName = GenerateArchiveName(); + var archiveBlobUrl = await _azureApi.UploadToBlob(archiveName, archiveData); + return archiveBlobUrl.ToString(); + } + } + + private string GenerateArchiveName() => $"{Guid.NewGuid()}.tar"; + + private async Task UploadArchiveToAws(string bucketName, string archivePath) + { + _log.LogInformation("Uploading Archive to AWS..."); + + var keyName = GenerateArchiveName(); + var archiveBlobUrl = await _awsApi.UploadToBucket(bucketName, archivePath, keyName); + + return archiveBlobUrl; + } + + private async Task UploadArchiveToGithub(string org, string archivePath) + { + await using var archiveData = _fileSystemProvider.OpenRead(archivePath); + var githubOrgDatabaseId = await _githubApi.GetOrganizationDatabaseId(org); + + _log.LogInformation("Uploading archive to GitHub Storage"); + var keyName = GenerateArchiveName(); + var authenticatedGitArchiveUri = await _githubApi.UploadArchiveToGithubStorage(githubOrgDatabaseId, keyName, archiveData); + + return authenticatedGitArchiveUri; + } + + private async Task CreateMigrationSource(MigrateRepoCommandArgs args) + { + _log.LogInformation("Creating Migration Source..."); + + args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); + var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); + + try + { + return await _githubApi.CreateGitlabMigrationSource(githubOrgId); + } + catch (OctoshiftCliException ex) when (ex.Message.Contains("not have the correct permissions to execute")) + { + var insufficientPermissionsMessage = InsufficientPermissionsMessageGenerator.Generate(args.GithubOrg); + var message = $"{ex.Message}{insufficientPermissionsMessage}"; + throw new OctoshiftCliException(message, ex); + } + } + + private async Task ImportArchive(MigrateRepoCommandArgs args, string migrationSourceId, string archiveUrl = null) + { + _log.LogInformation("Importing Archive..."); + + archiveUrl ??= args.ArchiveUrl; + + var gitlabRepoUrl = GetGitlabProjectUrl(args); + + args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); + var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); + + string migrationId; + + try + { + migrationId = await _githubApi.StartGitlabMigration(migrationSourceId, gitlabRepoUrl, githubOrgId, args.GithubRepo, args.GithubPat, archiveUrl, args.TargetRepoVisibility); + } + catch (OctoshiftCliException ex) when (ex.Message == $"A repository called {args.GithubOrg}/{args.GithubRepo} already exists") + { + _log.LogWarning($"The Org '{args.GithubOrg}' already contains a repository with the name '{args.GithubRepo}'. No operation will be performed"); + return; + } + + if (args.QueueOnly) + { + _log.LogInformation($"A repository migration (ID: {migrationId}) was successfully queued."); + return; + } + + var (migrationState, _, warningsCount, failureReason, migrationLogUrl) = await _githubApi.GetMigration(migrationId); + + while (RepositoryMigrationStatus.IsPending(migrationState)) + { + _log.LogInformation($"Migration in progress (ID: {migrationId}). State: {migrationState}. Waiting 60 seconds..."); + await Task.Delay(CHECK_MIGRATION_STATUS_DELAY_IN_MILLISECONDS); + (migrationState, _, warningsCount, failureReason, migrationLogUrl) = await _githubApi.GetMigration(migrationId); + } + + var migrationLogAvailableMessage = $"Migration log available at {migrationLogUrl} or by running `gh {CliContext.RootCommand} download-logs --github-org {args.GithubOrg} --github-repo {args.GithubRepo}`"; + + if (RepositoryMigrationStatus.IsFailed(migrationState)) + { + _log.LogError($"Migration Failed. Migration ID: {migrationId}"); + _warningsCountLogger.LogWarningsCount(warningsCount); + _log.LogInformation(migrationLogAvailableMessage); + throw new OctoshiftCliException(failureReason); + } + + _log.LogSuccess($"Migration completed (ID: {migrationId})! State: {migrationState}"); + _warningsCountLogger.LogWarningsCount(warningsCount); + _log.LogInformation(migrationLogAvailableMessage); + } + + private string GetAwsAccessKey(MigrateRepoCommandArgs args) => args.AwsAccessKey.HasValue() ? args.AwsAccessKey : _environmentVariableProvider.AwsAccessKeyId(false); + + private string GetAwsSecretKey(MigrateRepoCommandArgs args) => args.AwsSecretKey.HasValue() ? args.AwsSecretKey : _environmentVariableProvider.AwsSecretAccessKey(false); + + private string GetAwsRegion(MigrateRepoCommandArgs args) => args.AwsRegion.HasValue() ? args.AwsRegion : _environmentVariableProvider.AwsRegion(false); + + private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => args.AzureStorageConnectionString.HasValue() + ? args.AzureStorageConnectionString + : _environmentVariableProvider.AzureStorageConnectionString(false); + + private string GetGitlabPat(MigrateRepoCommandArgs args) => args.GitlabPat.HasValue() ? args.GitlabPat : _environmentVariableProvider.GitlabPat(false); + + private string GetGitlabProjectUrl(MigrateRepoCommandArgs args) + { + return args.GitlabServerUrl.HasValue() && args.GitlabGroup.HasValue() && args.GitlabProject.HasValue() + ? $"{args.GitlabServerUrl.TrimEnd('/')}/{args.GitlabGroup}/{args.GitlabProject}" + : "https://not-used"; + } + + private void ValidateOptions(MigrateRepoCommandArgs args) + { + if (args.ShouldGenerateArchive() && GetGitlabPat(args).IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("GitLab PAT must be either set as GITLAB_PAT environment variable or passed as --gitlab-pat."); + } + + // Validate --archive-path if provided as an input (i.e. not generating a new archive) + if (!args.ShouldGenerateArchive() && args.ArchivePath.HasValue() && !_fileSystemProvider.FileExists(args.ArchivePath)) + { + throw new OctoshiftCliException($"The archive file provided with --archive-path does not exist or is not accessible: {args.ArchivePath}"); + } + + if (args.ShouldUploadArchive()) + { + ValidateUploadOptions(args); + } + } + + private void ValidateUploadOptions(MigrateRepoCommandArgs args) + { + var shouldUseAzureStorage = GetAzureStorageConnectionString(args).HasValue(); + var shouldUseAwsS3 = args.AwsBucketName.HasValue(); + if (!shouldUseAzureStorage && !shouldUseAwsS3 && !args.UseGithubStorage) + { + throw new OctoshiftCliException( + "Either Azure storage connection (--azure-storage-connection-string or AZURE_STORAGE_CONNECTION_STRING env. variable) or " + + "AWS S3 connection (--aws-bucket-name, --aws-access-key (or AWS_ACCESS_KEY_ID env. variable), --aws-secret-key (or AWS_SECRET_ACCESS_KEY env.variable)) or " + + "GitHub Storage Option (--use-github-storage) " + + "must be provided."); + } + + if (shouldUseAzureStorage && shouldUseAwsS3) + { + throw new OctoshiftCliException( + "Azure storage connection (--azure-storage-connection-string or AZURE_STORAGE_CONNECTION_STRING env. variable) and " + + "AWS S3 connection (--aws-bucket-name, --aws-access-key (or AWS_ACCESS_KEY_ID env. variable), --aws-secret-key (or AWS_SECRET_ACCESS_KEY env.variable)) cannot be " + + "specified together."); + } + + if (shouldUseAwsS3) + { + if (!GetAwsAccessKey(args).HasValue()) + { + throw new OctoshiftCliException("Either --aws-access-key or AWS_ACCESS_KEY_ID environment variable must be set."); + } + + if (!GetAwsSecretKey(args).HasValue()) + { + throw new OctoshiftCliException("Either --aws-secret-key or AWS_SECRET_ACCESS_KEY environment variable must be set."); + } + + if (GetAwsRegion(args).IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("Either --aws-region or AWS_REGION environment variable must be set."); + } + } + } +} diff --git a/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs b/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs new file mode 100644 index 000000000..587ed556c --- /dev/null +++ b/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.ReclaimMannequin; + +namespace OctoshiftCLI.GitlabToGithub.Commands.ReclaimMannequin; + +public sealed class ReclaimMannequinCommand : ReclaimMannequinCommandBase +{ + public ReclaimMannequinCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs b/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs new file mode 100644 index 000000000..b94e13975 --- /dev/null +++ b/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.RevokeMigratorRole; + +namespace OctoshiftCLI.GitlabToGithub.Commands.RevokeMigratorRole; + +public sealed class RevokeMigratorRoleCommand : RevokeMigratorRoleCommandBase +{ + public RevokeMigratorRoleCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs b/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs new file mode 100644 index 000000000..0a03bd1a9 --- /dev/null +++ b/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs @@ -0,0 +1,7 @@ +using OctoshiftCLI.Commands.WaitForMigration; +namespace OctoshiftCLI.GitlabToGithub.Commands.WaitForMigration; + +public sealed class WaitForMigrationCommand : WaitForMigrationCommandBase +{ + public WaitForMigrationCommand() => AddOptions(); +} diff --git a/src/gl2gh/ExportState.cs b/src/gl2gh/ExportState.cs new file mode 100644 index 000000000..c403e3b56 --- /dev/null +++ b/src/gl2gh/ExportState.cs @@ -0,0 +1,11 @@ +namespace OctoshiftCLI.GitlabToGithub; + +public static class ExportState +{ + public const string FINISHED = "finished"; + public const string FAILED = "failed"; + + public static bool IsInProgress(string state) => state is not FINISHED && !IsError(state); + + public static bool IsError(string state) => state is FAILED; +} diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs new file mode 100644 index 000000000..b5c4ad3af --- /dev/null +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -0,0 +1,42 @@ +using System.Net.Http; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Factories; + +public class GitlabApiFactory +{ + private readonly OctoLogger _octoLogger; + private readonly IHttpClientFactory _clientFactory; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly IVersionProvider _versionProvider; + private readonly RetryPolicy _retryPolicy; + private readonly FileSystemProvider _fileSystemProvider; + + public GitlabApiFactory( + OctoLogger octoLogger, + IHttpClientFactory clientFactory, + EnvironmentVariableProvider environmentVariableProvider, + IVersionProvider versionProvider, + RetryPolicy retryPolicy, + FileSystemProvider fileSystemProvider) + { + _octoLogger = octoLogger; + _clientFactory = clientFactory; + _environmentVariableProvider = environmentVariableProvider; + _versionProvider = versionProvider; + _retryPolicy = retryPolicy; + _fileSystemProvider = fileSystemProvider; + } + + public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPat, bool noSsl = false) + { + gitlabPat ??= _environmentVariableProvider.GitlabPat(); + + var httpClient = noSsl ? _clientFactory.CreateClient("NoSSL") : _clientFactory.CreateClient("Default"); + + var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); + var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPat, _fileSystemProvider); + return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); + } +} diff --git a/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs b/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs new file mode 100644 index 000000000..131aef62b --- /dev/null +++ b/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs @@ -0,0 +1,18 @@ +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Factories; + +public class GitlabInspectorServiceFactory +{ + private readonly OctoLogger _octoLogger; + private GitlabInspectorService _instance; + + public GitlabInspectorServiceFactory(OctoLogger octoLogger) => _octoLogger = octoLogger; + + public virtual GitlabInspectorService Create(GitlabApi gitlabApi) + { + _instance ??= new(_octoLogger, gitlabApi); + + return _instance; + } +} diff --git a/src/gl2gh/Program.cs b/src/gl2gh/Program.cs new file mode 100644 index 000000000..1fa4fadbb --- /dev/null +++ b/src/gl2gh/Program.cs @@ -0,0 +1,149 @@ +using System; +using System.CommandLine; +using System.CommandLine.Builder; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Factories; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; + +[assembly: InternalsVisibleTo("OctoshiftCLI.Tests")] +namespace OctoshiftCLI.GitlabToGithub +{ + public static class Program + { + private static readonly OctoLogger Logger = new(); + + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "If the version check fails for any reason, we want the CLI to carry on with the current command")] + public static async Task Main(string[] args) + { + Logger.LogDebug("Execution Started"); + + var serviceCollection = new ServiceCollection(); + serviceCollection + .AddSingleton(Logger) + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton() + .AddHttpClient("NoSSL", noSsl: true) + .AddHttpClient("Default"); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + var rootCommand = new RootCommand("Automate end-to-end GitLab to GitHub migrations.") + .AddCommands(serviceProvider); + + var commandLineBuilder = new CommandLineBuilder(rootCommand); + var parser = commandLineBuilder + .UseDefaults() + .UseExceptionHandler((ex, _) => + { + Logger.LogError(ex); + Environment.ExitCode = 1; + }, 1) + .Build(); + + SetContext(new InvocationContext(parser.Parse(args))); + + try + { + await GithubStatusCheck(serviceProvider); + } + catch (Exception ex) + { + Logger.LogWarning("Could not check GitHub availability from githubstatus.com. See https://www.githubstatus.com for details."); + Logger.LogVerbose(ex.ToString()); + } + + try + { + await LatestVersionCheck(serviceProvider); + } + catch (Exception ex) + { + Logger.LogWarning("Could not retrieve latest gl2gh extension version from github.com, please ensure you are using the latest version by running: gh extension upgrade gl2gh"); + Logger.LogVerbose(ex.ToString()); + } + + await parser.InvokeAsync(args); + } + + private static void SetContext(InvocationContext context) + { + CliContext.RootCommand = "gl2gh"; + CliContext.ExecutingCommand = context.ParseResult.CommandResult.Command.Name; + } + + private static async Task GithubStatusCheck(ServiceProvider sp) + { + var envProvider = sp.GetRequiredService(); + + if (envProvider.SkipStatusCheck()?.ToUpperInvariant() is "TRUE" or "1") + { + Logger.LogInformation("Skipped GitHub status check due to GEI_SKIP_STATUS_CHECK environment variable"); + return; + } + + var githubStatusApi = sp.GetRequiredService(); + + if (await githubStatusApi.GetUnresolvedIncidentsCount() > 0) + { + Logger.LogWarning("GitHub is currently experiencing availability issues. See https://www.githubstatus.com for details."); + } + } + + private static async Task LatestVersionCheck(ServiceProvider sp) + { + var envProvider = sp.GetRequiredService(); + + if (envProvider.SkipVersionCheck()?.ToUpperInvariant() is "TRUE" or "1") + { + Logger.LogInformation("Skipped latest version check due to GEI_SKIP_VERSION_CHECK environment variable"); + return; + } + + var versionChecker = sp.GetRequiredService(); + + if (await versionChecker.IsLatest()) + { + Logger.LogInformation($"You are running an up-to-date version of the gl2gh extension [v{versionChecker.GetCurrentVersion()}]"); + } + else + { + Logger.LogWarning($"You are running an old version of the gl2gh extension [v{versionChecker.GetCurrentVersion()}]. The latest version is v{await versionChecker.GetLatestVersion()}."); + Logger.LogWarning($"Please update by running: gh extension upgrade gl2gh"); + } + } + + private static IServiceCollection AddHttpClient(this IServiceCollection serviceCollection, string name, bool noSsl = false) => serviceCollection + .AddHttpClient(name, _ => { }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + ServerCertificateCustomValidationCallback = noSsl ? delegate { return true; } : null + }) + .Services; + } +} diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs new file mode 100644 index 000000000..b7100510f --- /dev/null +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Octoshift.Models; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class GitlabInspectorService + { + private readonly OctoLogger _log; + private readonly GitlabApi _gitlabApi; + + private IList<(string, string)> _groups; + private readonly IDictionary> _repos = new Dictionary>(); + private readonly IDictionary> _mrCounts = new Dictionary>(); + + public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) + { + _log = log; + _gitlabApi = gitlabApi; + } + + public virtual async Task> GetGroups() + { + if (_groups is null) + { + _log.LogInformation($"Retrieving list of all Groups the user has access to..."); + _groups = (await _gitlabApi.GetGroups()) + .Select(group => (group.Path, group.Name)) + .ToList(); + } + + return _groups; + } + + public virtual async Task<(string Key, string Name)> GetGroup(string groupPath) + { + _log.LogInformation($"Retrieving Group..."); + var (_, Key, Name) = await _gitlabApi.GetGroup(groupPath); + + return (Key, Name); + } + + public virtual async Task> GetProjects(string groupPath) + { + if (!_repos.TryGetValue(groupPath, out var repos)) + { + repos = (await _gitlabApi.GetProjects(groupPath)) + .Select(repo => new GitlabProject() { Name = repo.Name, Path = repo.Path, Archived = repo.Archived }) + .ToList(); + _repos.Add(groupPath, repos); + } + + return repos; + } + + public virtual async Task GetProjectCount(string[] groups) + { + return await groups.Sum(async key => await GetProjectCount(key)); + } + + public virtual async Task GetProjectCount() + { + var groups = await GetGroups(); + return await groups.Sum(async group => await GetProjectCount(group.Path)); + } + + public virtual async Task GetProjectCount(string groupPath) + { + return (await GetProjects(groupPath)).Count(); + } + + public virtual async Task GetMergeRequestCount(string groupPath) + { + var repos = await GetProjects(groupPath); + return await repos.Sum(async repo => await GetProjectMergeRequestCount(groupPath, repo.Path)); + } + + public virtual async Task GetProjectMergeRequestCount(string groupPath, string repo) + { + if (!_mrCounts.ContainsKey(groupPath)) + { + _mrCounts.Add(groupPath, new Dictionary()); + } + + if (!_mrCounts[groupPath].TryGetValue(repo, out var mrCount)) + { + mrCount = await _gitlabApi.GetMergeRequestCount(groupPath, repo); + _mrCounts[groupPath][repo] = mrCount; + } + + return mrCount; + } + } +} diff --git a/src/gl2gh/Services/GroupsCsvGeneratorService.cs b/src/gl2gh/Services/GroupsCsvGeneratorService.cs new file mode 100644 index 000000000..ceaf1b352 --- /dev/null +++ b/src/gl2gh/Services/GroupsCsvGeneratorService.cs @@ -0,0 +1,47 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Factories; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class GroupsCsvGeneratorService + { + private readonly GitlabInspectorServiceFactory _gitlabInspectorServiceFactory; + private readonly GitlabApiFactory _gitlabApiFactory; + + public GroupsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + { + _gitlabInspectorServiceFactory = gitlabInspectorServiceFactory; + _gitlabApiFactory = gitlabApiFactory; + } + + public virtual async Task Generate(string gitlabServerUrl, string gitlabPat, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + { + gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); + + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPat, noSslVerify); + var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); + var result = new StringBuilder(); + + result.Append("group-path,group-name,url,project-count"); + result.AppendLine(!minimal ? ",mr-count" : null); + + var groups = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; + + foreach (var (groupPath, groupName) in groups) + { + var url = $"{gitlabServerUrl.TrimEnd('/')}/{groupPath}"; + var projectCount = await inspector.GetProjectCount(groupPath); + var mrCount = !minimal ? await inspector.GetMergeRequestCount(groupPath) : 0; + + var name = groupName.Replace(",", Uri.EscapeDataString(",")); + + result.Append($"\"{groupPath}\",\"{name}\",\"{url}\",{projectCount}"); + result.AppendLine(!minimal ? $",{mrCount}" : null); + } + + return result.ToString(); + } + } +} diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs new file mode 100644 index 000000000..4e4f3a93b --- /dev/null +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -0,0 +1,60 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Factories; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class ProjectsCsvGeneratorService + { + private readonly GitlabInspectorServiceFactory _gitlabInspectorServiceFactory; + private readonly GitlabApiFactory _gitlabApiFactory; + + public ProjectsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + { + _gitlabInspectorServiceFactory = gitlabInspectorServiceFactory; + _gitlabApiFactory = gitlabApiFactory; + } + + public virtual async Task Generate(string gitlabServerUrl, string gitlabPat, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + { + gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); + + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPat, noSslVerify); + var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); + var result = new StringBuilder(); + + result.Append("group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived"); + result.AppendLine(!minimal ? ",mr-count" : null); + + var groups = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; + + foreach (var (groupPath, groupName) in groups) + { + foreach (var project in await inspector.GetProjects(groupPath)) + { + var url = $"{gitlabServerUrl.TrimEnd('/')}/{groupPath}/{project.Path}"; + var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(groupPath, project.Path); + var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(groupPath, project.Path); + var mrCount = !minimal ? await inspector.GetProjectMergeRequestCount(groupPath, project.Path) : 0; + + var group = groupName.Replace(",", Uri.EscapeDataString(",")); + var projectName = project.Name.Replace(",", Uri.EscapeDataString(",")); + + if (lastCommitDate == null) + { + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",,\"{repoSize:D}\",\"{attachmentsSize:D}\",\"{project.Archived}\""); + } + else + { + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"{project.Archived}\""); + } + + result.AppendLine(!minimal ? $",{mrCount}" : null); + } + } + + return result.ToString(); + } + } +} diff --git a/src/gl2gh/gl2gh.csproj b/src/gl2gh/gl2gh.csproj new file mode 100644 index 000000000..2088f3769 --- /dev/null +++ b/src/gl2gh/gl2gh.csproj @@ -0,0 +1,23 @@ + + + + Exe + net8.0 + gl2gh + 12 + OctoshiftCLI.GitlabToGithub + + + + + + + + + + + + + + +