show cache miss di

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

.github/scripts/find_cache_paths.py

@@ -48,6 +48,15 @@ def find_fingerprints_and_check_dirs(base_dir):
     return sorted(valid_paths), missing_content, fingerprint_contents
 
 
+def parse_fingerprint_contents(fingerprint_content):
+    """Parse the fingerprint content and return a dict of digest-path tuples."""
+    input_map = {}
+    for line in fingerprint_content.splitlines():
+        digest, path = line.split()
+        input_map[path] = digest
+    return input_map
+
+
 def calculate_sha256(fingerprint_contents):
     sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0])
 
@@ -89,9 +98,16 @@ def main(file_path: str | None):
         fingerprint_file = f"{path}.fingerprint"
         if os.path.exists(fingerprint_file):
             file_digest = calculate_file_sha256(fingerprint_file)
+
+            # Parse the fingerprint file to get the digest/path tuples
+            with open(fingerprint_file, 'r') as f:
+                fingerprint_content = f.read().strip()
+                input_map = parse_fingerprint_contents(fingerprint_content)
+
             paths_with_digests.append({
                 "path": path,
-                "digest": file_digest
+                "digest": file_digest,
+                "input": input_map
             })
 
     output = {

.github/workflows/validations.yaml

@@ -41,6 +41,20 @@ jobs:
       - name: Run unit tests
         run: make unit
 
+      # save .tmp/cache_paths.json and .tmp/last_cache_paths.json as workflow artifacts
+      - name: Upload cache_paths.json as an artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: cache_paths
+          path: .tmp/cache_paths.json
+
+      - name: Upload last_cache_paths.json as an artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: last_cache_paths
+          path: .tmp/last_cache_paths.json
+
+
 
   Integration-Test:
     # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline

Taskfile.yaml

@@ -11,7 +11,7 @@ vars:
   TMP_DIR: .tmp
   ORAS_CACHE: "{{ .TMP_DIR }}/oras-cache"
   CACHE_PATHS_FILE: "{{ .TMP_DIR }}/cache_paths.json"
-  LAST_CACHE_PULL_FILE: "{{ .TMP_DIR }}/last_cache_pull"
+  LAST_CACHE_PULL_FILE: "{{ .TMP_DIR }}/last_cache_paths.json"
 
   # TOOLS
   ORAS: "{{ .TOOL_DIR }}/oras"
@@ -304,8 +304,10 @@ tasks:
           #echo -e "${YELLOW}• calculating fingerprint for '$dir/image-*'... ${RESET}"
           
           # find all image-* directories in this test-fixtures directory and generate sha256sum
-          find "$dir" -type d -name 'image-*' -exec bash -c 'find "$0" -type f | sort | xargs sha256sum' {} \; > "$dir/cache.fingerprint"
-          
+          pushd "$dir" > /dev/null
+            find . -type d -name 'image-*' -exec bash -c 'find "$0" -type f | sort | xargs sha256sum' {} \; > "cache.fingerprint"
+          popd > /dev/null
+        
           # for debugging...
           #cat "$dir/cache.fingerprint" | sha256sum | awk '{print $1}'
         done
@@ -335,7 +337,7 @@ tasks:
       echo "desired cache:     $WANT_FINGERPRINT"
       
       if [ -f {{ .LAST_CACHE_PULL_FILE }} ]; then
-        LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }})
+        LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }} | {{ .YQ }} -r '.digest')
       else
         echo -e "${BOLD}${PURPLE}empty cache, refreshing cache...${RESET}"
         {{ .TASK }} clean-cache download-test-fixture-cache
@@ -360,6 +362,13 @@ tasks:
         {{ .TASK }} clean-cache download-test-fixture-cache
       else
         echo -e "${BOLD}${PURPLE}found newer cache, but does not match what we want (skipping cache refresh)${RESET}"
+      
+        git status
+      
+        {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .LAST_CACHE_PULL_FILE }} > .tmp/last_cache_lines
+        {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .CACHE_PATHS_FILE }} > .tmp/cache_lines
+        diff .tmp/last_cache_lines .tmp/cache_lines || true
+      
       fi
 
   build-fixtures:
@@ -389,7 +398,7 @@ tasks:
         sh: docker manifest inspect {{ .CACHE_IMAGE }} | jq -r '.annotations.fingerprint'
     cmds:
       - "ORAS_CACHE={{ .ORAS_CACHE }} {{ .ORAS }} pull {{ .CACHE_IMAGE }}"
-      - "echo '{{ .CACHE_DIGEST }}' > {{ .LAST_CACHE_PULL_FILE }}"
+      - "cp {{ .CACHE_PATHS_FILE }} {{ .LAST_CACHE_PULL_FILE }}"
 
   upload-test-fixture-cache:
     desc: Upload the test fixture cache to ghcr.io