From e21d8f93281601085e8ed89ece66b28d1ce30eee Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 16 Jan 2023 19:38:14 -0400 Subject: [PATCH] feat(ci): delete unused artifacts in registries (#5873) * feat(ci): delete unused artifacts in registries Previous behavior: Docker artifacts are costing us a good part of our infrastructure budget, and we needed a way to remove unused artifacts. Expected behavior: Delete unused (not just old) docker artifacts in GAR (Google Artifact Registry), preferably using a generic solution is this needs to be expanded into other Docker registries. Solution: Implement GCR Cleaner https://github.com/GoogleCloudPlatform/gcr-cleaner, as this tools provided integration with `docker/login-action` to interact with multiple Docker v2 registries. * fix(action): use hours instead of days * chore: add TODO * Update .github/workflows/delete-gcp-resources.yml Co-authored-by: teor * fix: allow the action to fail if some images can't be deleted Co-authored-by: teor --- .github/workflows/delete-gcp-resources.yml | 46 ++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/delete-gcp-resources.yml b/.github/workflows/delete-gcp-resources.yml index 2781a5d4..4d6ad31e 100644 --- a/.github/workflows/delete-gcp-resources.yml +++ b/.github/workflows/delete-gcp-resources.yml @@ -1,3 +1,4 @@ +# TODO: rename this action name and filename to Delete infra resources name: Delete GCP resources on: @@ -17,6 +18,9 @@ env: # But keep the latest $KEEP_LATEST_IMAGE_COUNT images of each type. # We keep this small to reduce storage costs. KEEP_LATEST_IMAGE_COUNT: 2 + # Delete all artifacts in registry created before $DELETE_IMAGE_HOURS hours ago. + # We keep this long enough for PRs that are still on the same commit can re-run with the same image. + DELETE_IMAGE_HOURS: 504h # 21 days jobs: delete-resources: @@ -168,3 +172,45 @@ jobs: gcloud compute images delete "${IMAGE}" || continue done + + # We're using a generic approach here, which allows multiple registries to be included, + # even those not related to GCP. Enough reason to create a separate job. + clean-registries: + name: Delete unused artifacts in registry + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.2.0 + with: + persist-credentials: false + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v1.0.0 + with: + retries: '3' + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + - name: Login to Google Artifact Registry + uses: docker/login-action@v2.1.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + # Deletes all images older than $DELETE_IMAGE_HOURS days. + - uses: 'docker://us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner-cli' + continue-on-error: true # TODO: remove after fixig https://github.com/ZcashFoundation/zebra/issues/5933 + # Refer to the official documentation to understand available arguments: + # https://github.com/GoogleCloudPlatform/gcr-cleaner + with: + args: >- + -repo=us-docker.pkg.dev/zealous-zebra/zebra/zebrad-test + -repo=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd + -grace=${{ env.DELETE_IMAGE_HOURS }} + -keep=${{ env.KEEP_LATEST_IMAGE_COUNT }}