Skip to content

Commit

Permalink
Allow passing e.g. --hints="@package-lock.json packages/*/package.jso…
Browse files Browse the repository at this point in the history
…n" to resolve the hint to the file content digest
  • Loading branch information
dimikot committed Dec 12, 2024
1 parent 1bd5d30 commit b25d0b3
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 2 deletions.
1 change: 1 addition & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"mtimes",
"myslot",
"oneline",
"strerror",
"tmpfs",
"topo"
]
Expand Down
48 changes: 47 additions & 1 deletion ci-storage
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ from __future__ import annotations
import argparse
import collections
import dataclasses
import glob
import hashlib
import os.path
import re
import shlex
Expand Down Expand Up @@ -131,7 +133,7 @@ def main():
)
slot_ids: list[str] = " ".join(args.slot_id).split()
local_dir: str = re.sub(r"/+$", "", args.local_dir)
hints: list[str] = " ".join(args.hint).split()
hints: list[str] = [hint for arg in args.hint for hint in expand_hint_arg(arg=arg)]
exclude: list[str] = [
line for line in "\n".join(args.exclude).splitlines() if line.strip()
]
Expand Down Expand Up @@ -631,6 +633,50 @@ def check_call(
print(elapsed)


#
# Given a hint argument value, expands it to hints:
# - If the argument starts with "@", then it expands to a digest of the content
# of all files matching the space-separated list of patterns after the "@".
# - If the argument contains multiple space-separated values, then it treats
# them as a list of hints.
#
def expand_hint_arg(*, arg: str) -> list[str]:
if arg.startswith("@"):
items = arg[1:].strip().split()
item_patterns = [item for item in items if is_wildcard(item)]
item_files = [item for item in items if not is_wildcard(item)]
files = sorted(
[
*item_files,
*[
file
for pattern in item_patterns
for file in glob.glob(pattern, recursive=True)
],
]
)
print(cmd_to_debug_prompt(["sha256sum", *files]))
m = hashlib.sha256()
for file in files:
try:
with open(file, "rb") as f:
m.update(f.read())
except OSError as e:
raise UserException(f"{e.strerror}: {e.filename}")
hint = "@" + m.hexdigest()[0:16]
print(f" {hint}")
return [hint]
else:
return arg.strip().split()


#
# Returns true if the path is a wildcard pattern.
#
def is_wildcard(path: str) -> bool:
return "*" in path or "?" in path


#
# Converts a command to a debug string like:
# - "$ some command"
Expand Down
14 changes: 14 additions & 0 deletions tests/0045-hints-with-patterns-expand-to-digest.test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/bin/bash
source ./common.sh

echo "a" > "$LOCAL_DIR/file-new-a"
echo "b" > "$LOCAL_DIR/file-new-b"
echo "c" > "$LOCAL_DIR/file-c"
ci-storage \
--slot-id=myslot1 \
--hint="aaa bbb" \
--hint="@$LOCAL_DIR/file-new-* $LOCAL_DIR/file-c $LOCAL_DIR/file-absent?" \
--hint=" ccc " \
store

grep -qF "hints=aaa bbb @f3bba3f02fb45a2a ccc" "$STORAGE_DIR/myslot1/.ci-storage.meta"
2 changes: 1 addition & 1 deletion tests/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ trap '
if [[ "$exitcode" != 0 ]]; then
echo
echo "FAILED! Last output was:"
echo "==================="
echo "========================"
cat $OUT
exit 1
fi
Expand Down

0 comments on commit b25d0b3

Please sign in to comment.