From 457f6ba8a581b3da21848e7f913ca9c1a0cb74fa Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Wed, 20 May 2026 22:07:11 +0000 Subject: [PATCH 1/6] chore: add disk space cleanup steps to CI pipeline (Phase 3) Add a reusable cleanup-disk-space.yml step template and insert it at strategic points in the Linux CI jobs to address 71 'Free disk space on / is lower than 5%' warnings. Cleanup points in build-cli.yml: - After test run: clean Go cache, NuGet, and .NET temp artifacts - After release build: clean Go build cache - After Linux package build: clean Docker images/containers Cleanup points in cross-build-cli.yml: - After Linux ARM64 package build: clean Docker and Go cache All cleanup steps are Linux-only, use continueOnError to avoid breaking builds, and log disk usage before/after for observability. Resolves #7783 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/build-cli.yml | 14 +++++ .../templates/jobs/cross-build-cli.yml | 5 ++ .../templates/steps/cleanup-disk-space.yml | 55 +++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 eng/pipelines/templates/steps/cleanup-disk-space.yml diff --git a/eng/pipelines/templates/jobs/build-cli.yml b/eng/pipelines/templates/jobs/build-cli.yml index 9a379bb4ba6..48360afe787 100644 --- a/eng/pipelines/templates/jobs/build-cli.yml +++ b/eng/pipelines/templates/jobs/build-cli.yml @@ -171,6 +171,12 @@ jobs: displayName: Publish test results condition: succeededOrFailed() + - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml + parameters: + CleanGoCache: true + CleanNuGet: true + CleanDotNet: true + - task: PowerShell@2 inputs: pwsh: true @@ -200,10 +206,18 @@ jobs: workingDirectory: cli/azd displayName: Copy binary to artifact staging directory + - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml + parameters: + CleanGoCache: true + - template: /eng/pipelines/templates/steps/build-linux-packages.yml parameters: Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) + - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml + parameters: + CleanDocker: true + - pwsh: New-Item -ItemType Directory -Path $(Build.ArtifactStagingDirectory)/shield -Force condition: and(succeeded(), eq(variables['SetShieldInfo'], 'true')) displayName: Create shield artifact directory diff --git a/eng/pipelines/templates/jobs/cross-build-cli.yml b/eng/pipelines/templates/jobs/cross-build-cli.yml index 1e9897d93aa..364bd65f8f3 100644 --- a/eng/pipelines/templates/jobs/cross-build-cli.yml +++ b/eng/pipelines/templates/jobs/cross-build-cli.yml @@ -97,6 +97,11 @@ jobs: parameters: Architecture: arm64 Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) + + - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml + parameters: + CleanDocker: true + CleanGoCache: true templateContext: outputs: diff --git a/eng/pipelines/templates/steps/cleanup-disk-space.yml b/eng/pipelines/templates/steps/cleanup-disk-space.yml new file mode 100644 index 00000000000..2df3f531cfc --- /dev/null +++ b/eng/pipelines/templates/steps/cleanup-disk-space.yml @@ -0,0 +1,55 @@ +parameters: + - name: CleanDocker + type: boolean + default: false + - name: CleanGoCache + type: boolean + default: false + - name: CleanNuGet + type: boolean + default: false + - name: CleanDotNet + type: boolean + default: false + - name: Condition + type: string + default: succeeded() + +steps: + - bash: df -h / + displayName: Disk usage (before cleanup) + condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) + continueOnError: true + + - ${{ if eq(parameters.CleanDocker, true) }}: + - bash: docker system prune -af --volumes 2>/dev/null || true + displayName: Cleanup Docker + condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) + continueOnError: true + + - ${{ if eq(parameters.CleanGoCache, true) }}: + - bash: | + go clean -cache 2>/dev/null || true + go clean -testcache 2>/dev/null || true + displayName: Cleanup Go cache + condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) + continueOnError: true + + - ${{ if eq(parameters.CleanNuGet, true) }}: + - bash: dotnet nuget locals all --clear 2>/dev/null || true + displayName: Cleanup NuGet cache + condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) + continueOnError: true + + - ${{ if eq(parameters.CleanDotNet, true) }}: + - bash: | + rm -rf /tmp/NuGetScratch 2>/dev/null || true + rm -rf /tmp/.dotnet 2>/dev/null || true + displayName: Cleanup .NET temp artifacts + condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) + continueOnError: true + + - bash: df -h / + displayName: Disk usage (after cleanup) + condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) + continueOnError: true From 2cfc99a5f10dc45824801362c633e8ce15f840e9 Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Wed, 20 May 2026 22:21:05 +0000 Subject: [PATCH 2/6] fix: address Copilot review feedback (iteration 1) - Remove CleanGoCache from post-test cleanup to preserve cache for release build - Add BuildLinuxPackages condition to Docker cleanup in build-cli.yml - Add BuildLinuxPackages condition to Docker+Go cleanup in cross-build-cli.yml Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/build-cli.yml | 2 +- eng/pipelines/templates/jobs/cross-build-cli.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/build-cli.yml b/eng/pipelines/templates/jobs/build-cli.yml index 48360afe787..7ad7b61665b 100644 --- a/eng/pipelines/templates/jobs/build-cli.yml +++ b/eng/pipelines/templates/jobs/build-cli.yml @@ -173,7 +173,6 @@ jobs: - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml parameters: - CleanGoCache: true CleanNuGet: true CleanDotNet: true @@ -217,6 +216,7 @@ jobs: - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml parameters: CleanDocker: true + Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) - pwsh: New-Item -ItemType Directory -Path $(Build.ArtifactStagingDirectory)/shield -Force condition: and(succeeded(), eq(variables['SetShieldInfo'], 'true')) diff --git a/eng/pipelines/templates/jobs/cross-build-cli.yml b/eng/pipelines/templates/jobs/cross-build-cli.yml index 364bd65f8f3..4ab1753d220 100644 --- a/eng/pipelines/templates/jobs/cross-build-cli.yml +++ b/eng/pipelines/templates/jobs/cross-build-cli.yml @@ -102,6 +102,7 @@ jobs: parameters: CleanDocker: true CleanGoCache: true + Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) templateContext: outputs: From c5c1af03fbe684d603be676c2347d6874ad3ab22 Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Wed, 20 May 2026 22:28:52 +0000 Subject: [PATCH 3/6] fix: address Copilot review feedback (iteration 2) - Remove redundant 2>/dev/null || true from cleanup commands; rely on continueOnError - Update PR description to reflect post-test cleanup no longer includes Go cache Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/steps/cleanup-disk-space.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/steps/cleanup-disk-space.yml b/eng/pipelines/templates/steps/cleanup-disk-space.yml index 2df3f531cfc..7389b8c874c 100644 --- a/eng/pipelines/templates/steps/cleanup-disk-space.yml +++ b/eng/pipelines/templates/steps/cleanup-disk-space.yml @@ -22,29 +22,29 @@ steps: continueOnError: true - ${{ if eq(parameters.CleanDocker, true) }}: - - bash: docker system prune -af --volumes 2>/dev/null || true + - bash: docker system prune -af --volumes displayName: Cleanup Docker condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) continueOnError: true - ${{ if eq(parameters.CleanGoCache, true) }}: - bash: | - go clean -cache 2>/dev/null || true - go clean -testcache 2>/dev/null || true + go clean -cache + go clean -testcache displayName: Cleanup Go cache condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) continueOnError: true - ${{ if eq(parameters.CleanNuGet, true) }}: - - bash: dotnet nuget locals all --clear 2>/dev/null || true + - bash: dotnet nuget locals all --clear displayName: Cleanup NuGet cache condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) continueOnError: true - ${{ if eq(parameters.CleanDotNet, true) }}: - bash: | - rm -rf /tmp/NuGetScratch 2>/dev/null || true - rm -rf /tmp/.dotnet 2>/dev/null || true + rm -rf /tmp/NuGetScratch + rm -rf /tmp/.dotnet displayName: Cleanup .NET temp artifacts condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) continueOnError: true From d9624562c5f6fa66574bafe806c18c90aca9be78 Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Tue, 26 May 2026 23:32:41 +0000 Subject: [PATCH 4/6] fix: address remaining disk space warnings in CrossBuildCLI LinuxARM64 - Add Go cache cleanup BEFORE Linux package build (not just after) - Split cleanup: Go cache before Docker build, Docker after - Add temp/cache directory overrides (DOCKER_TMPDIR, TMPDIR, GOCACHE) to CrossBuildCLI LinuxARM64 matrix entry, matching BuildCLI Linux Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/cross-build-cli.yml | 6 +++++- eng/pipelines/templates/stages/build-and-test.yml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/cross-build-cli.yml b/eng/pipelines/templates/jobs/cross-build-cli.yml index 4ab1753d220..35233fc0322 100644 --- a/eng/pipelines/templates/jobs/cross-build-cli.yml +++ b/eng/pipelines/templates/jobs/cross-build-cli.yml @@ -93,6 +93,11 @@ jobs: workingDirectory: cli/azd displayName: Copy binary to artifact staging directory + - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml + parameters: + CleanGoCache: true + Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) + - template: /eng/pipelines/templates/steps/build-linux-packages.yml parameters: Architecture: arm64 @@ -101,7 +106,6 @@ jobs: - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml parameters: CleanDocker: true - CleanGoCache: true Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) templateContext: diff --git a/eng/pipelines/templates/stages/build-and-test.yml b/eng/pipelines/templates/stages/build-and-test.yml index 78a300cb4a6..69c777f7fab 100644 --- a/eng/pipelines/templates/stages/build-and-test.yml +++ b/eng/pipelines/templates/stages/build-and-test.yml @@ -27,6 +27,11 @@ parameters: GOOS: linux GOARCH: arm64 BuildLinuxPackages: true + # Set special temp/cache directory overrides to avoid out of space + # issues on hosted agents + DOCKER_TMPDIR: $(Agent.TempDirectory) + TMPDIR: $(Agent.TempDirectory) + GOCACHE: $(Agent.TempDirectory)/go-cache MacARM64: Pool: Azure Pipelines OSVmImage: $(MACVMIMAGE) From 9a8aaa6ac858fe88d59d8d2a06ee7a229791edf2 Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Thu, 28 May 2026 05:53:46 +0000 Subject: [PATCH 5/6] fix: add pre-Docker-build cleanup to prevent disk space warnings Add Docker prune and system cache cleanup (apt, temp files) BEFORE the build-linux-packages step. The Docker build + Copacetic security scan consume ~4GB, pushing disk from 92% to 98% and generating 54+ disk space warnings. By cleaning Docker images and system caches before the build, we free enough space to stay under the 95% warning threshold. Changes: - cleanup-disk-space.yml: Add CleanSystemCaches parameter (apt-get clean, remove apt lists/archives, temp Go build files) - build-cli.yml: Add Docker + system cache cleanup before Linux packages - cross-build-cli.yml: Add Docker + system cache cleanup before Linux packages Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/build-cli.yml | 3 +++ eng/pipelines/templates/jobs/cross-build-cli.yml | 2 ++ .../templates/steps/cleanup-disk-space.yml | 13 +++++++++++++ 3 files changed, 18 insertions(+) diff --git a/eng/pipelines/templates/jobs/build-cli.yml b/eng/pipelines/templates/jobs/build-cli.yml index 7ad7b61665b..2479a053024 100644 --- a/eng/pipelines/templates/jobs/build-cli.yml +++ b/eng/pipelines/templates/jobs/build-cli.yml @@ -208,6 +208,9 @@ jobs: - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml parameters: CleanGoCache: true + CleanDocker: true + CleanSystemCaches: true + Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) - template: /eng/pipelines/templates/steps/build-linux-packages.yml parameters: diff --git a/eng/pipelines/templates/jobs/cross-build-cli.yml b/eng/pipelines/templates/jobs/cross-build-cli.yml index 35233fc0322..5aaae72c414 100644 --- a/eng/pipelines/templates/jobs/cross-build-cli.yml +++ b/eng/pipelines/templates/jobs/cross-build-cli.yml @@ -96,6 +96,8 @@ jobs: - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml parameters: CleanGoCache: true + CleanDocker: true + CleanSystemCaches: true Condition: and(succeeded(), eq(variables['BuildLinuxPackages'], 'true')) - template: /eng/pipelines/templates/steps/build-linux-packages.yml diff --git a/eng/pipelines/templates/steps/cleanup-disk-space.yml b/eng/pipelines/templates/steps/cleanup-disk-space.yml index 7389b8c874c..796f0a586ca 100644 --- a/eng/pipelines/templates/steps/cleanup-disk-space.yml +++ b/eng/pipelines/templates/steps/cleanup-disk-space.yml @@ -11,6 +11,9 @@ parameters: - name: CleanDotNet type: boolean default: false + - name: CleanSystemCaches + type: boolean + default: false - name: Condition type: string default: succeeded() @@ -49,6 +52,16 @@ steps: condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) continueOnError: true + - ${{ if eq(parameters.CleanSystemCaches, true) }}: + - bash: | + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + sudo rm -rf /var/cache/apt/archives/* + rm -rf /tmp/go-build* + displayName: Cleanup system caches + condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) + continueOnError: true + - bash: df -h / displayName: Disk usage (after cleanup) condition: and(${{ parameters.Condition }}, eq(variables['Agent.OS'], 'Linux')) From 81ca6aeedc57eb308c57b38b322dda9aa51a4d28 Mon Sep 17 00:00:00 2001 From: Victor Vazquez Date: Fri, 29 May 2026 20:56:29 +0000 Subject: [PATCH 6/6] chore: add succeededOrFailed() condition to post-test cleanup Ensures disk space cleanup runs even when tests fail, freeing space for subsequent steps and jobs on the build agent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/build-cli.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/jobs/build-cli.yml b/eng/pipelines/templates/jobs/build-cli.yml index 2479a053024..8eb371409f3 100644 --- a/eng/pipelines/templates/jobs/build-cli.yml +++ b/eng/pipelines/templates/jobs/build-cli.yml @@ -173,6 +173,7 @@ jobs: - template: /eng/pipelines/templates/steps/cleanup-disk-space.yml parameters: + Condition: succeededOrFailed() CleanNuGet: true CleanDotNet: true