diff --git a/.github/scripts/find_cache_paths.py b/.github/scripts/find_cache_paths.py index 38b8103c4c3..d6a5b7a4d51 100755 --- a/.github/scripts/find_cache_paths.py +++ b/.github/scripts/find_cache_paths.py @@ -48,6 +48,15 @@ def find_fingerprints_and_check_dirs(base_dir): return sorted(valid_paths), missing_content, fingerprint_contents +def parse_fingerprint_contents(fingerprint_content): + """Parse the fingerprint content and return a dict of digest-path tuples.""" + input_map = {} + for line in fingerprint_content.splitlines(): + digest, path = line.split() + input_map[path] = digest + return input_map + + def calculate_sha256(fingerprint_contents): sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0]) @@ -89,9 +98,16 @@ def main(file_path: str | None): fingerprint_file = f"{path}.fingerprint" if os.path.exists(fingerprint_file): file_digest = calculate_file_sha256(fingerprint_file) + + # Parse the fingerprint file to get the digest/path tuples + with open(fingerprint_file, 'r') as f: + fingerprint_content = f.read().strip() + input_map = parse_fingerprint_contents(fingerprint_content) + paths_with_digests.append({ "path": path, - "digest": file_digest + "digest": file_digest, + "input": input_map }) output = { diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 67fc5cd1878..9bfd7040fd1 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -41,6 +41,20 @@ jobs: - name: Run unit tests run: make unit + # save .tmp/cache_paths.json and .tmp/last_cache_paths.json as workflow artifacts + - name: Upload cache_paths.json as an artifact + uses: actions/upload-artifact@v3 + with: + name: cache_paths + path: .tmp/cache_paths.json + + - name: Upload last_cache_paths.json as an artifact + uses: actions/upload-artifact@v3 + with: + name: last_cache_paths + path: .tmp/last_cache_paths.json + + Integration-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline diff --git a/Taskfile.yaml b/Taskfile.yaml index b3bacd4a564..4239e2f20f3 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -11,7 +11,7 @@ vars: TMP_DIR: .tmp ORAS_CACHE: "{{ .TMP_DIR }}/oras-cache" CACHE_PATHS_FILE: "{{ .TMP_DIR }}/cache_paths.json" - LAST_CACHE_PULL_FILE: "{{ .TMP_DIR }}/last_cache_pull" + LAST_CACHE_PULL_FILE: "{{ .TMP_DIR }}/last_cache_paths.json" # TOOLS ORAS: "{{ .TOOL_DIR }}/oras" @@ -304,8 +304,10 @@ tasks: #echo -e "${YELLOW}• calculating fingerprint for '$dir/image-*'... ${RESET}" # find all image-* directories in this test-fixtures directory and generate sha256sum - find "$dir" -type d -name 'image-*' -exec bash -c 'find "$0" -type f | sort | xargs sha256sum' {} \; > "$dir/cache.fingerprint" - + pushd "$dir" > /dev/null + find . -type d -name 'image-*' -exec bash -c 'find "$0" -type f | sort | xargs sha256sum' {} \; > "cache.fingerprint" + popd > /dev/null + # for debugging... #cat "$dir/cache.fingerprint" | sha256sum | awk '{print $1}' done @@ -335,7 +337,7 @@ tasks: echo "desired cache: $WANT_FINGERPRINT" if [ -f {{ .LAST_CACHE_PULL_FILE }} ]; then - LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }}) + LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }} | {{ .YQ }} -r '.digest') else echo -e "${BOLD}${PURPLE}empty cache, refreshing cache...${RESET}" {{ .TASK }} clean-cache download-test-fixture-cache @@ -360,6 +362,13 @@ tasks: {{ .TASK }} clean-cache download-test-fixture-cache else echo -e "${BOLD}${PURPLE}found newer cache, but does not match what we want (skipping cache refresh)${RESET}" + + git status + + {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .LAST_CACHE_PULL_FILE }} > .tmp/last_cache_lines + {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .CACHE_PATHS_FILE }} > .tmp/cache_lines + diff .tmp/last_cache_lines .tmp/cache_lines || true + fi build-fixtures: @@ -389,7 +398,7 @@ tasks: sh: docker manifest inspect {{ .CACHE_IMAGE }} | jq -r '.annotations.fingerprint' cmds: - "ORAS_CACHE={{ .ORAS_CACHE }} {{ .ORAS }} pull {{ .CACHE_IMAGE }}" - - "echo '{{ .CACHE_DIGEST }}' > {{ .LAST_CACHE_PULL_FILE }}" + - "cp {{ .CACHE_PATHS_FILE }} {{ .LAST_CACHE_PULL_FILE }}" upload-test-fixture-cache: desc: Upload the test fixture cache to ghcr.io