diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 7a2cffdb..d36271f9 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -169,6 +169,8 @@ jobs: zebra_skip_ipv6_tests: '1' rust_log: info + # zebrad tests without cached state + # Run all the zebra tests, including tests that are ignored by default. # Skips tests that need a cached state disk or a lightwalletd binary. # @@ -256,6 +258,8 @@ jobs: env: ZEBRA_TEST_LIGHTWALLETD: '1' + # zebrad cached checkpoint state tests + # Regenerate mandatory checkpoint Zebra cached state disks. # # Runs: @@ -275,7 +279,7 @@ jobs: needs_zebra_state: false saves_to_disk: true disk_suffix: checkpoint - height_grep_text: 'flushing database to disk .*height.*=.*Height' + height_grep_text: 'flushing database to disk .*height.*=.*Height.*\(' # Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk # @@ -294,6 +298,8 @@ jobs: saves_to_disk: false disk_suffix: checkpoint + # zebrad cached tip state tests + # Test that Zebra can run a full mainnet sync, # and regenerate chain tip Zebra cached state disks. # @@ -318,7 +324,7 @@ jobs: needs_zebra_state: false saves_to_disk: true disk_suffix: tip - height_grep_text: 'current_height.*=.*Height' + height_grep_text: 'current_height.*=.*Height.*\(' # We don't want to cancel running full syncs on `main` if a new PR gets merged, # because we might never finish a full sync during busy weeks. Instead, we let the # first sync complete, then queue the latest pending sync, cancelling any syncs in between. @@ -357,6 +363,68 @@ jobs: disk_suffix: tip root_state_path: '/var/cache' zebra_state_dir: 'zebrad-cache' + height_grep_text: 'current_height.*=.*Height.*\(' + + # lightwalletd cached tip state tests + + # Test full sync of lightwalletd with a Zebra tip state + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + lightwalletd-full-sync: + name: lightwalletd tip + needs: [ test-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + # to also run on Mergify head branches, + # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`: + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1 + if: ${{ (!cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true') || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) }} + with: + app_name: lightwalletd + test_id: lwd-full-sync + test_description: Test lightwalletd full sync + test_variables: '-e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' + needs_zebra_state: true + needs_lwd_state: false + saves_to_disk: true + disk_prefix: lwd-cache + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + lwd_state_dir: 'lwd-cache' + height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )' + + # Test update sync of lightwalletd with a lightwalletd and Zebra tip state + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + lightwalletd-update-sync: + name: lightwalletd tip update + needs: lightwalletd-full-sync + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' }} + with: + app_name: lightwalletd + test_id: lwd-update-sync + test_description: Test lightwalletd update sync with both states + test_variables: '-e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' + needs_zebra_state: true + needs_lwd_state: true + # update the disk on every PR, to increase CI speed + saves_to_disk: true + disk_prefix: lwd-cache + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + lwd_state_dir: 'lwd-cache' + height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )' # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state # @@ -409,63 +477,6 @@ jobs: zebra_state_dir: 'zebrad-cache' lwd_state_dir: 'lwd-cache' - # Test full sync of lightwalletd with a Zebra tip state - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached state to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - lightwalletd-full-sync: - name: lightwalletd tip - needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - # to also run on Mergify head branches, - # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`: - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1 - if: ${{ (!cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true') || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) }} - with: - app_name: lightwalletd - test_id: lwd-full-sync - test_description: Test lightwalletd full sync - test_variables: '-e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' - needs_zebra_state: true - needs_lwd_state: false - saves_to_disk: true - disk_prefix: lwd-cache - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - lwd_state_dir: 'lwd-cache' - - # Test update sync of lightwalletd with a lightwalletd and Zebra tip state - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached states to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - lightwalletd-update-sync: - name: lightwalletd tip update - needs: lightwalletd-full-sync - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' }} - with: - app_name: lightwalletd - test_id: lwd-update-sync - test_description: Test lightwalletd update sync with both states - test_variables: '-e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' - needs_zebra_state: true - needs_lwd_state: true - # update the disk on every PR, to increase CI speed - saves_to_disk: true - disk_prefix: lwd-cache - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - lwd_state_dir: 'lwd-cache' - # Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state # # Runs: diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index 7567e4a0..ff7805f1 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -1139,9 +1139,9 @@ jobs: --zone ${{ env.ZONE }} \ --quiet \ --ssh-flag="-o ServerAliveInterval=5" \ - --command="docker logs ${{ inputs.test_id }} --tail 20") + --command="docker logs ${{ inputs.test_id }} --tail 200") - SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE '${{ inputs.height_grep_text }}\([0-9]+\)' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]]) + SYNC_HEIGHT=$(echo $DOCKER_LOGS | grep -oE '${{ inputs.height_grep_text }}[0-9]+' | grep -oE '[0-9]+' | tail -1 || [[ $? == 1 ]]) echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV # Sets the $UPDATE_SUFFIX env var to "-u" if using cached state, @@ -1195,7 +1195,8 @@ jobs: --source-disk=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ --source-disk-zone=${{ env.ZONE }} \ --storage-location=us \ - --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}" + --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}" \ + --labels="height=${{ env.SYNC_HEIGHT }},purpose=${{ inputs.disk_prefix }},commit=${{ env.GITHUB_SHA_SHORT }},state-version=${{ env.STATE_VERSION }},network=${{ env.NETWORK }},target-height=${{ inputs.disk_suffix }},update-flag=${UPDATE_SUFFIX},test-id=${{ inputs.test_id }},app-name=${{ inputs.app_name }}" # delete the Google Cloud instance for this test delete-instance: