ignore gitignored files

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

.github/scripts/find_cache_paths.py

@@ -49,7 +49,6 @@ def find_fingerprints_and_check_dirs(base_dir):
 
 
 def parse_fingerprint_contents(fingerprint_content):
-    """Parse the fingerprint content and return a dict of digest-path tuples."""
     input_map = {}
     for line in fingerprint_content.splitlines():
         digest, path = line.split()
@@ -68,7 +67,6 @@ def calculate_sha256(fingerprint_contents):
 
 
 def calculate_file_sha256(file_path):
-    """Calculate the sha256 digest of a file."""
     sha256_hash = hashlib.sha256()
     with open(file_path, 'rb') as f:
         for byte_block in iter(lambda: f.read(4096), b""):
@@ -96,19 +94,25 @@ def main(file_path: str | None):
     paths_with_digests = []
     for path in sorted(valid_paths):
         fingerprint_file = f"{path}.fingerprint"
-        if os.path.exists(fingerprint_file):
-            file_digest = calculate_file_sha256(fingerprint_file)
-
-            # Parse the fingerprint file to get the digest/path tuples
-            with open(fingerprint_file, 'r') as f:
-                fingerprint_content = f.read().strip()
-                input_map = parse_fingerprint_contents(fingerprint_content)
-
-            paths_with_digests.append({
-                "path": path,
-                "digest": file_digest,
-                "input": input_map
-            })
+        try:
+            if os.path.exists(fingerprint_file):
+                file_digest = calculate_file_sha256(fingerprint_file)
+
+                # Parse the fingerprint file to get the digest/path tuples
+                with open(fingerprint_file, 'r') as f:
+                    fingerprint_content = f.read().strip()
+                    input_map = parse_fingerprint_contents(fingerprint_content)
+
+                paths_with_digests.append({
+                    "path": path,
+                    "digest": file_digest,
+                    "input": input_map
+                })
+
+        except Exception as e:
+            show(f"Error processing {fingerprint_file}: {e}")
+            raise e
+
 
     output = {
         "digest": sha256_hash,

.github/scripts/fingerprint_docker_fixtures.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import hashlib
+
+BOLD = '\033[1m'
+YELLOW = '\033[0;33m'
+RESET = '\033[0m'
+
+
+def print_message(message):
+    print(f"{YELLOW}{message}{RESET}")
+
+
+def sha256sum(filepath):
+    h = hashlib.sha256()
+    with open(filepath, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def is_git_tracked_or_untracked(directory):
+    """Returns a sorted list of files in the directory that are tracked or not ignored by Git."""
+    result = subprocess.run(
+        ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
+        cwd=directory,
+        stdout=subprocess.PIPE,
+        text=True
+    )
+    return sorted(result.stdout.strip().splitlines())
+
+
+def find_test_fixture_dirs_with_images(base_dir):
+    """Find directories that contain 'test-fixtures' and at least one 'image-*' directory."""
+    for root, dirs, files in os.walk(base_dir):
+        if 'test-fixtures' in root:
+            image_dirs = [d for d in dirs if d.startswith('image-')]
+            if image_dirs:
+                yield os.path.realpath(root)
+
+
+def generate_fingerprints():
+    print_message("creating fingerprint files for docker fixtures...")
+
+    for test_fixture_dir in find_test_fixture_dirs_with_images('.'):
+        cache_fingerprint_path = os.path.join(test_fixture_dir, 'cache.fingerprint')
+
+        with open(cache_fingerprint_path, 'w') as fingerprint_file:
+            for image_dir in find_image_dirs(test_fixture_dir):
+                for file in is_git_tracked_or_untracked(image_dir):
+                    file_path = os.path.join(image_dir, file)
+                    checksum = sha256sum(file_path)
+                    fingerprint_file.write(f"{checksum}  {file}\n")
+
+
+def find_image_dirs(test_fixture_dir):
+    """Find all 'image-*' directories inside a given test-fixture directory."""
+    for root, dirs, files in os.walk(test_fixture_dir):
+        for dir_name in dirs:
+            if dir_name.startswith('image-'):
+                yield os.path.join(root, dir_name)
+
+
+if __name__ == "__main__":
+    generate_fingerprints()

Taskfile.yaml

@@ -289,32 +289,7 @@ tasks:
         # for debugging...
         # echo -e "generated all fixture fingerprints"
 
-      - |
-        BOLD='\033[1m'
-        YELLOW='\033[0;33m'
-        RESET='\033[0m'
-        
-        echo -e "${YELLOW}creating fingerprint files for docker fixtures...${RESET}"
-        
-        # generate sha256sum for test-fixture directories that contain at least one image-* directory
-        for dir in $(find . -type d -name 'test-fixtures' -exec bash -c '[ "$(find "$0" -type d -name "image-*")" ] && echo "$0"' {} \;); do
-          dir=$(echo "$dir" | sed 's#//*#/#g')
-
-          # for debugging...
-          #echo -e "${YELLOW}• calculating fingerprint for '$dir/image-*'... ${RESET}"
-          
-          # find all image-* directories in this test-fixtures directory and generate sha256sum
-          pushd "$dir" > /dev/null
-            find . -type d -name 'image-*' -exec bash -c 'find "$0" -type f | sort | xargs sha256sum' {} \; > "cache.fingerprint"
-          popd > /dev/null
-        
-          # for debugging...
-          #cat "$dir/cache.fingerprint" | sha256sum | awk '{print $1}'
-        done
-
-        # for debugging...
-        # echo -e "generated all docker fixture fingerprints"
-
+      - python .github/scripts/fingerprint_docker_fixtures.py
       - python .github/scripts/find_cache_paths.py {{ .CACHE_PATHS_FILE }} > /dev/null
 
 
@@ -339,7 +314,7 @@ tasks:
       if [ -f {{ .LAST_CACHE_PULL_FILE }} ]; then
         LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }} | {{ .YQ }} -r '.digest')
       else
-        echo -e "${BOLD}${PURPLE}empty cache, refreshing cache...${RESET}"
+        echo -e "${BOLD}${PURPLE}empty cache, downloading cache...${RESET}"
         {{ .TASK }} clean-cache download-test-fixture-cache
         exit 0
       fi
@@ -348,7 +323,7 @@ tasks:
 
       # if we already have the latest cache, skip the refresh
       if [ "$LAST_PULL_FINGERPRINT" = "$WANT_FINGERPRINT" ]; then
-          echo -e "${BOLD}${PURPLE}already have the latest cache (skipping cache refresh)${RESET}"
+          echo -e "${BOLD}${PURPLE}already have the latest cache (skipping cache download)${RESET}"
           exit 0
       fi
 
@@ -358,17 +333,16 @@ tasks:
       # which would be annoying to deal with in a development workflow).
 
       if [ "$LATEST_FINGERPRINT" = "$WANT_FINGERPRINT" ]; then
-        echo -e "${BOLD}${PURPLE}found newer cache! refreshing cache...${RESET}"
+        echo -e "${BOLD}${PURPLE}found newer cache! downloading cache...${RESET}"
         {{ .TASK }} clean-cache download-test-fixture-cache
       else
-        echo -e "${BOLD}${PURPLE}found newer cache, but does not match what we want (skipping cache refresh)${RESET}"
-      
-        git status
+        echo -e "${BOLD}${PURPLE}found different cache, but isn't clear if it's newer (skipping cache download and manually building)${RESET}"
       
         {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .LAST_CACHE_PULL_FILE }} > .tmp/last_cache_lines
         {{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .CACHE_PATHS_FILE }} > .tmp/cache_lines
         diff .tmp/last_cache_lines .tmp/cache_lines || true
       
+        {{ .TASK }} build-fixtures
       fi
 
   build-fixtures:

cmd/syft/internal/test/integration/test-fixtures/Makefile

@@ -1,6 +1,3 @@
-# change these if you want CI to not use previous stored cache
-CACHE_BUSTER := "fd8e5cd"
-
 FINGERPRINT_FILE := cache.fingerprint
 
 .DEFAULT_GOAL := fixtures
@@ -16,7 +13,6 @@ fingerprint: $(FINGERPRINT_FILE)
 .PHONY: $(FINGERPRINT_FILE)
 $(FINGERPRINT_FILE):
 	@find image-* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
-	@echo "$(CACHE_BUSTER)" >> $(FINGERPRINT_FILE)
 	@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
 
 # requirement 4: 'clean' goal to remove all generated test fixtures

test/cli/test-fixtures/Makefile

@@ -1,6 +1,3 @@
-# change these if you want CI to not use previous stored cache
-CACHE_BUSTER := "fd8e5cd"
-
 FINGERPRINT_FILE=cache.fingerprint
 
 .DEFAULT_GOAL := fixtures
@@ -16,7 +13,6 @@ fingerprint: $(FINGERPRINT_FILE)
 .PHONY: $(FINGERPRINT_FILE)
 $(FINGERPRINT_FILE):
 	@find image-* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
-	@echo "$(CACHE_BUSTER)" >> $(FINGERPRINT_FILE)
 	@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
 
 # requirement 4: 'clean' goal to remove all generated test fixtures

test/install/Makefile

@@ -1,7 +1,5 @@
 NAME=syft
 
-# CI cache busting values; change these if you want CI to not use previous stored cache
-CACHE_BUSTER=da88c94
 FINGERPRINT_FILE := cache.fingerprint
 
 # for local testing (not testing within containers) use the binny-managed version of cosign.
@@ -136,5 +134,4 @@ busybox-1.36:
 .PHONY: $(FINGERPRINT_FILE)
 $(FINGERPRINT_FILE):
 	@find ./environments/* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
-	@echo "$(CACHE_BUSTER)" >> $(FINGERPRINT_FILE)
 	@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'