fix(docker): Stop resetting the `cargo-chef` cache in the Dockerfile (#6934)

* Fix Dockerfile cache use

* Remove cache-breaking COPY commands

* Use git to only reset files modified by cargo-chef

* Copy .git and sources before cargo chef cook

* Update .dockerignore to include .git

* Don't use .git

* Use rsync instead of git

* Maybe COPY is needed

* Actually copy changed files using rsync

* Actually copy the files using the correct rsync syntax

* Remove ls commands from Dockerfile

docker/Dockerfile

@@ -29,7 +29,7 @@ FROM chef AS deps
 SHELL ["/bin/bash", "-xo", "pipefail", "-c"]
 COPY --from=planner /opt/zebrad/recipe.json recipe.json
 
-# Install zebra build deps
+# Install zebra build deps and Dockerfile deps
 RUN apt-get -qq update && \
     apt-get -qq install -y --no-install-recommends \
     llvm \
@@ -37,6 +37,7 @@ RUN apt-get -qq update && \
     clang \
     ca-certificates \
     protobuf-compiler \
+    rsync \
     ; \
     rm -rf /var/lib/apt/lists/* /tmp/*
 
@@ -102,16 +103,28 @@ FROM deps AS tests
 COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params
 COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /opt/lightwalletd /usr/local/bin
 
+# cargo uses timestamps for its cache, so they need to be in this order:
+# unmodified source files < previous build cache < modified source files
+COPY . .
+
 # Re-hydrate the minimum project skeleton identified by `cargo chef prepare` in the planner stage,
+# over the top of the original source files,
 # and build it to cache all possible sentry and test dependencies.
 #
-# This is the caching Docker layer for Rust!
+# This is the caching Docker layer for Rust tests!
+# It creates fake empty test binaries so dependencies are built, but Zebra is not fully built.
 #
 # TODO: add --locked when cargo-chef supports it
 RUN cargo chef cook --tests --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json
 
-COPY . .
-# Test Zebra
+# Undo the source file changes made by cargo-chef.
+# rsync invalidates the cargo cache for the changed files only, by updating their timestamps.
+# This makes sure the fake empty binaries created by cargo-chef are rebuilt.
+COPY --from=planner /opt/zebrad zebra-original
+RUN rsync --recursive --checksum --itemize-changes --verbose zebra-original/ .
+RUN rm -r zebra-original
+
+# Build Zebra test binaries, but don't run them
 RUN cargo test --locked --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --no-run
 RUN cp /opt/zebrad/target/release/zebrad /usr/local/bin
 RUN cp /opt/zebrad/target/release/zebra-checkpoints /usr/local/bin
@@ -129,10 +142,19 @@ ENTRYPOINT [ "/entrypoint.sh" ]
 # zebrad binary from this step.
 FROM deps AS release
 
+COPY . .
+
+# This is the caching layer for Rust zebrad builds.
+# It creates a fake empty zebrad binary, see above for details.
+#
 # TODO: add --locked when cargo-chef supports it
 RUN cargo chef cook --release --features "${FEATURES}" --package zebrad --bin zebrad --recipe-path recipe.json
 
-COPY . .
+# Undo the source file changes made by cargo-chef, so the fake empty zebrad binary is rebuilt.
+COPY --from=planner /opt/zebrad zebra-original
+RUN rsync --recursive --checksum --itemize-changes --verbose zebra-original/ .
+RUN rm -r zebra-original
+
 # Build zebrad
 RUN cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad