From 2cadb7304bfa852fae9a39563a53b5a25f69d692 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 21 Oct 2022 02:01:06 -0400 Subject: [PATCH] ci(sync): increase the height of blocks for some full sync jobs (#5391) * adds extra step to CI docker before logs-checkpoint * replaces logs-1790k with logs-1800k * Update .github/workflows/deploy-gcp-tests.yml Co-authored-by: teor * Update .github/workflows/deploy-gcp-tests.yml Co-authored-by: teor Co-authored-by: Arya Co-authored-by: teor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/deploy-gcp-tests.yml | 79 +++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index d30b69b9..73aaa502 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -899,13 +899,13 @@ jobs: -e 'test result:.*finished in' \ " - # follow the logs of the test we just launched, up to block 1,790,000 or later + # follow the logs of the test we just launched, up to block 1,800,000 or later # (or the test finishing) # - # We chose this height because it was about 16 hours into the NU5 sync, in September 2022. + # We chose this height because it was about 20 hours into the NU5 sync, in October 2022. # (These blocks seem to be larger than the previous ones.) - logs-1790k: - name: Log ${{ inputs.test_id }} test (1790k) + logs-1800k: + name: Log ${{ inputs.test_id }} test (1800k) needs: [ logs-1780k ] # If the previous job fails, we still want to show the logs. if: ${{ !cancelled() }} @@ -939,8 +939,8 @@ jobs: service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' token_format: 'access_token' - # Show recent logs, following until block 1,790,000 (or the test finishes) - - name: Show logs for ${{ inputs.test_id }} test (1790k) + # Show recent logs, following until block 1,800,000 (or the test finishes) + - name: Show logs for ${{ inputs.test_id }} test (1800k) run: | gcloud compute ssh \ github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ @@ -957,16 +957,79 @@ jobs: ${{ inputs.test_id }} | \ tee --output-error=exit /dev/stderr | \ grep --max-count=1 --extended-regexp --color=always \ - -e 'estimated progress.*current_height.*=.*179[0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ -e 'estimated progress.*current_height.*=.*1[8-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ -e 'test result:.*finished in' \ " + # follow the logs of the test we just launched, up to block 1,820,000 or later + # (or the test finishing) + # + # We chose this height because it was about 24 hours into the NU5 sync, in October 2022. + # (These blocks seem to be larger than the previous ones.) + logs-1820k: + name: Log ${{ inputs.test_id }} test (1820k) + needs: [ logs-1800k ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.1.0 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.1 + with: + retries: '3' + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Show recent logs, following until block 1,820,000 (or the test finishes) + - name: Show logs for ${{ inputs.test_id }} test (1820k) + run: | + gcloud compute ssh \ + github-service-account@${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --ssh-flag="-o ConnectionAttempts=20" \ + --ssh-flag="-o ConnectTimeout=5" \ + --command \ + "\ + sudo docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + -e 'estimated progress.*current_height.*=.*18[2-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*19[0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'estimated progress.*current_height.*=.*2[0-9][0-9][0-9][0-9][0-9][0-9].*remaining_sync_blocks' \ + -e 'test result:.*finished in' \ + " + # follow the logs of the test we just launched, up to the last checkpoint (or the test finishing) logs-checkpoint: name: Log ${{ inputs.test_id }} test (checkpoint) - needs: [ logs-1790k ] + needs: [ logs-1820k ] # If the previous job fails, we still want to show the logs. if: ${{ !cancelled() }} runs-on: ubuntu-latest