diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 47a21c4f..008a2009 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -53,6 +53,7 @@ on: - '.github/workflows/continous-integration-docker.yml' - '.github/workflows/deploy-gcp-tests.yml' - '.github/workflows/build-docker-image.yml' + - '.github/workflows/find-cached-disks.yml' push: branches: @@ -75,87 +76,37 @@ on: - '.github/workflows/continous-integration-docker.yml' - '.github/workflows/deploy-gcp-tests.yml' - '.github/workflows/build-docker-image.yml' + - '.github/workflows/find-cached-disks.yml' jobs: + + # Check if the cached state disks used by the tests are available for the default network. + # + # The default network is Mainnet unless a manually triggered workflow or repository variable + # is configured differently. + # + # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml get-available-disks: - runs-on: ubuntu-latest - name: Find available cached state disks - outputs: - lwd_tip_disk: ${{ steps.get-available-disks.outputs.lwd_tip_disk }} - zebra_tip_disk: ${{ steps.get-available-disks.outputs.zebra_tip_disk }} - zebra_checkpoint_disk: ${{ steps.get-available-disks.outputs.zebra_checkpoint_disk }} - permissions: - contents: 'read' - id-token: 'write' - steps: - - uses: actions/checkout@v3.5.2 - with: - persist-credentials: false - fetch-depth: 0 + name: Check if cached state disks exist for ${{ inputs.network || vars.ZCASH_NETWORK }} + uses: ./.github/workflows/find-cached-disks.yml + with: + network: ${{ inputs.network || vars.ZCASH_NETWORK }} - # Setup gcloud CLI - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v1.1.0 - with: - retries: '3' - workload_identity_provider: '${{ vars.GCP_WIF }}' - service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1.1.0 - - # Disk images in GCP are required to be in lowercase, but the blockchain network - # uses sentence case, so we need to downcase ${{ vars.ZCASH_NETWORK or github.event.inputs.network }} - # - # Passes a lowercase Network name to subsequent steps using $NETWORK env variable - - name: Downcase network name for disks - run: | - NETWORK_CAPS=${{ github.event.inputs.network || vars.ZCASH_NETWORK }} - echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV - - # Find a cached state disk for subsequent jobs needing a cached state without - # restricting the result from any branch. - # - # This search is executed considering the actual version from constants.rs - # - # Generate one of the following outputs with a boolean to pass to subsequent jobs: - # - lwd_tip_disk - # - zebra_tip_disk - # - zebra_checkpoint_disk - - name: Find cached state disks - id: get-available-disks - run: | - LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1) - echo "STATE_VERSION: $LOCAL_STATE_VERSION" - - LWD_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~lwd-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) - if [[ -z "$LWD_TIP_DISK" ]]; then - echo "No TIP disk found for LWD" - echo "lwd_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT" - else - echo "Disk: $LWD_TIP_DISK" - echo "lwd_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT" - fi - - ZEBRA_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) - if [[ -z "$ZEBRA_TIP_DISK" ]]; then - echo "No TIP disk found for ZEBRA" - echo "zebra_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT" - else - echo "Disk: $ZEBRA_TIP_DISK" - echo "zebra_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT" - fi - - ZEBRA_CHECKPOINT_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) - if [[ -z "$ZEBRA_CHECKPOINT_DISK" ]]; then - echo "No CHECKPOINT found for ZEBRA" - echo "zebra_checkpoint_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT" - else - echo "Disk: $ZEBRA_CHECKPOINT_DISK" - echo "zebra_checkpoint_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT" - fi + # Check if the cached state disks used by the tests are available for Testnet. + # + # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml + # Some outputs are ignored, because we don't run those jobs on Testnet. + get-available-disks-testnet: + name: Check if cached state disks exist for Testnet + uses: ./.github/workflows/find-cached-disks.yml + with: + network: 'Testnet' + # Build the docker image used by the tests. + # + # The default network in the Zebra config in the image is Mainnet, unless a manually triggered + # workflow or repository variable is configured differently. Testnet jobs change that config to + # testnet when running the image. build: name: Build CI Docker uses: ./.github/workflows/build-docker-image.yml @@ -163,8 +114,7 @@ jobs: dockerfile_path: ./docker/Dockerfile dockerfile_target: tests image_name: ${{ vars.CI_IMAGE_NAME }} - # The default network in the image is based on the default network variable in GitHub - network: ${{ vars.ZCASH_NETWORK }} + network: ${{ inputs.network || vars.ZCASH_NETWORK }} checkpoint_sync: true rust_backtrace: full rust_lib_backtrace: full diff --git a/.github/workflows/find-cached-disks.yml b/.github/workflows/find-cached-disks.yml new file mode 100644 index 00000000..6aeed6b9 --- /dev/null +++ b/.github/workflows/find-cached-disks.yml @@ -0,0 +1,99 @@ +name: Check if cached state disks exist + +on: + workflow_call: + inputs: + network: + description: 'The Zcash network used to look up the disks' + required: true + type: string + outputs: + lwd_tip_disk: + description: 'true if there is a lightwalletd and Zebra cached state disk, synced near the chain tip' + value: ${{ jobs.get-available-disks.outputs.lwd_tip_disk }} + zebra_tip_disk: + description: 'true if there is a Zebra cached state disk synced near the chain tip' + value: ${{ jobs.get-available-disks.outputs.zebra_tip_disk }} + zebra_checkpoint_disk: + description: 'true if there is a Zebra cached state disk synced to the mandatory Zebra checkpoint' + value: ${{ jobs.get-available-disks.outputs.zebra_checkpoint_disk }} + +jobs: + get-available-disks: + name: Check if cached state disks exist + runs-on: ubuntu-latest + outputs: + lwd_tip_disk: ${{ steps.get-available-disks.outputs.lwd_tip_disk }} + zebra_tip_disk: ${{ steps.get-available-disks.outputs.zebra_tip_disk }} + zebra_checkpoint_disk: ${{ steps.get-available-disks.outputs.zebra_checkpoint_disk }} + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.5.2 + with: + persist-credentials: false + fetch-depth: 0 + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v1.1.0 + with: + retries: '3' + workload_identity_provider: '${{ vars.GCP_WIF }}' + service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1.1.0 + + # Disk images in GCP are required to be in lowercase, but the blockchain network + # uses sentence case, so we need to downcase ${{ inputs.network }} + # + # Passes a lowercase Network name to subsequent steps using $NETWORK env variable + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Check if there are cached state disks available for subsequent jobs to use. + # + # This lookup uses the state version from constants.rs. + # It accepts disks generated by any branch, including draft and unmerged PRs. + # + # If the disk exists, sets the corresponding output to "true": + # - lwd_tip_disk + # - zebra_tip_disk + # - zebra_checkpoint_disk + - name: Check if cached state disks exist + id: get-available-disks + run: | + LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1) + echo "STATE_VERSION: $LOCAL_STATE_VERSION" + + LWD_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~lwd-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) + if [[ -z "$LWD_TIP_DISK" ]]; then + echo "No TIP disk found for lightwalletd on network: ${NETWORK}" + echo "lwd_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT" + else + echo "Disk: $LWD_TIP_DISK" + echo "lwd_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT" + fi + + ZEBRA_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) + if [[ -z "$ZEBRA_TIP_DISK" ]]; then + echo "No TIP disk found for Zebra on network: ${NETWORK}" + echo "zebra_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT" + else + echo "Disk: $ZEBRA_TIP_DISK" + echo "zebra_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT" + fi + + ZEBRA_CHECKPOINT_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) + if [[ -z "$ZEBRA_CHECKPOINT_DISK" ]]; then + echo "No CHECKPOINT disk found for Zebra on network: ${NETWORK}" + echo "zebra_checkpoint_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT" + else + echo "Disk: $ZEBRA_CHECKPOINT_DISK" + echo "zebra_checkpoint_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT" + fi