First version of a Golang version for APT package querying. (#118)

.github/workflows/pull_request.yml

@@ -0,0 +1,29 @@
+name: Pull Request
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+permissions:
+  contents: read
+
+jobs:
+  integrate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: "go.mod"
+
+      - name: Build and test
+        run: |
+          go build -v ./...
+          go test -v ./...
+
+      - name: Lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.52.2

.gitignore

@@ -0,0 +1 @@
+src/cmd/apt_query/apt_query*

.vscode/launch.json

@@ -0,0 +1,12 @@
+{  
+  "version": "0.2.0",
+  "configurations": [    
+    {
+      "name": "Launch Package",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${fileDirname}",
+    }
+  ]
+}

README.md

@@ -23,24 +23,24 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory
 
 There are three kinds of version labels you can use.
 
-* `@latest` - This will give you the latest release.
-* `@v#` - Major only will give you the latest release for that major version only (e.g. `v1`).
-* Branch
-  * `@master` - Most recent manual and automated tested code. Possibly unstable since it is pre-release.
-  * `@staging` - Most recent automated tested code and can sometimes contain experimental features. Is pulled from dev stable code.
-  * `@dev` - Very unstable and contains experimental features. Automated testing may not show breaks since CI is also updated based on code in dev.
+- `@latest` - This will give you the latest release.
+- `@v#` - Major only will give you the latest release for that major version only (e.g. `v1`).
+- Branch
+  - `@master` - Most recent manual and automated tested code. Possibly unstable since it is pre-release.
+  - `@staging` - Most recent automated tested code and can sometimes contain experimental features. Is pulled from dev stable code.
+  - `@dev` - Very unstable and contains experimental features. Automated testing may not show breaks since CI is also updated based on code in dev.
 
 ### Inputs
 
-* `packages` - Space delimited list of packages to install.
-* `version` - Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed.
-* `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information.
+- `packages` - Space delimited list of packages to install.
+- `version` - Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed.
+- `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information.
 
 ### Outputs
 
-* `cache-hit` - A boolean value to indicate a cache was found for the packages requested.
-* `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with equals delimit on the package version (i.e. \<package1>=<version1\>,\<package2>=\<version2>,...).
-* `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with equals delimit on the package version (i.e. \<package1>=<version1\>,\<package2>=\<version2>,...).
+- `cache-hit` - A boolean value to indicate a cache was found for the packages requested.
+- `package-version-list` - The main requested packages and versions that are installed. Represented as a comma delimited list with equals delimit on the package version (i.e. \<package1>=<version1\>,\<package2>=\<version2>,...).
+- `all-package-version-list` - All the pulled in packages and versions, including dependencies, that are installed. Represented as a comma delimited list with equals delimit on the package version (i.e. \<package1>=<version1\>,\<package2>=\<version2>,...).
 
 ### Cache scopes
 
@@ -54,7 +54,6 @@ This was a motivating use case for creating this action.
 name: Create Documentation
 on: push
 jobs:
-
   build_and_deploy_docs:
     runs-on: ubuntu-latest
     name: Build Doxygen documentation and deploy
@@ -65,7 +64,7 @@ jobs:
           packages: dia doxygen doxygen-doc doxygen-gui doxygen-latex graphviz mscgen
           version: 1.0
 
-      - name: Build        
+      - name: Build
         run: |
           cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}      
           cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
@@ -78,15 +77,16 @@ jobs:
 ```
 
 ```yaml
-...
-  install_doxygen_deps:
-    runs-on: ubuntu-latest    
-    steps:
-      - uses: actions/checkout@v2
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: dia doxygen doxygen-doc doxygen-gui doxygen-latex graphviz mscgen
-          version: 1.0
+
+---
+install_doxygen_deps:
+  runs-on: ubuntu-latest
+  steps:
+    - uses: actions/checkout@v2
+    - uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: dia doxygen doxygen-doc doxygen-gui doxygen-latex graphviz mscgen
+        version: 1.0
 ```
 
 ## Caveats
@@ -95,8 +95,8 @@ jobs:
 
 This action is based on the principle that most packages can be cached as a fileset. There are situations though where this is not enough.
 
-* Pre and post installation scripts needs to be ran from `/var/lib/dpkg/info/{package name}.[preinst, postinst]`.
-* The Debian package database needs to be queried for scripts above (i.e. `dpkg-query`).
+- Pre and post installation scripts needs to be ran from `/var/lib/dpkg/info/{package name}.[preinst, postinst]`.
+- The Debian package database needs to be queried for scripts above (i.e. `dpkg-query`).
 
 The `execute_install_scripts` argument can be used to attempt to execute the install scripts but they are no guaranteed to resolve the issue.
 
@@ -121,4 +121,4 @@ For more context and information see [issue #57](https://github.com/awalsh128/ca
 
 ### Cache Limits
 
-A repository can have up to 5GB of caches. Once the 5GB limit is reached, older caches will be evicted based on when the cache was last accessed.  Caches that are not accessed within the last week will also be evicted.
+A repository can have up to 5GB of caches. Once the 5GB limit is reached, older caches will be evicted based on when the cache was last accessed. Caches that are not accessed within the last week will also be evicted. To get more information on how to access and manage your actions's caches, see [GitHub Actions / Using workflows / Cache dependencies](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#viewing-cache-entries).

go.mod

@@ -0,0 +1,3 @@
+module awalsh128.com/cache-apt-pkgs-action
+
+go 1.20

lib.sh

@@ -99,57 +99,11 @@ function get_normalized_package_list {
   # Remove commas, and block scalar folded backslashes,
   # extraneous spaces at the middle, beginning and end
   # then sort.
-  packages=$(echo "${1}" \
+  local packages=$(echo "${1}" \
     | sed 's/[,\]/ /g; s/\s\+/ /g; s/^\s\+//g; s/\s\+$//g' \
     | sort -t' ')
-
-  # Validate package names and get versions.
-  log_err "resolving package versions..."
-  data=$(apt-cache --quiet=0 --no-all-versions show ${packages} 2>&1 | \
-    grep -E '^(Package|Version|N):')
-  log_err "resolved"
-
-  local ORIG_IFS="${IFS}"
-  IFS=$'\n'
-  declare -A missing
-  local package_versions=''
-  local package='' separator=''
-  for key_value in ${data}; do
-    local key="${key_value%%: *}"
-    local value="${key_value##*: }"
-
-    case $key in
-      Package)
-        package=$value
-        ;;
-      Version)
-        package_versions="${package_versions}${separator}"${package}=${value}""
-        separator=' '
-        ;;
-      N)
-        # Warning messages.
-        case $value in
-          'Unable to locate package '*)
-            package="${value#'Unable to locate package '}"
-            # Avoid duplicate messages.
-            if [ -z "${missing[$package]}" ]; then
-              package="${value#'Unable to locate package '}"
-              log_err "Package '${package}' not found."
-              missing[$package]=1
-            fi
-            ;;
-        esac
-        ;;
-    esac
-  done
-  IFS="${ORIG_IFS}"
-
-  if [ ${#missing[@]} -gt 0 ]; then
-    echo "aborted"
-    exit 5
-  fi
-
-  echo "${package_versions}"
+  local script_dir="$(dirname -- "$(realpath -- "${0}")")"
+  ${script_dir}/apt_query normalized-list ${packages}
 }
 
 ###############################################################################

pre_cache_action.sh

@@ -28,7 +28,9 @@ debug="${4}"
 input_packages="${@:5}"
 
 # Trim commas, excess spaces, and sort.
+log "Normalizing package list..."
 packages="$(get_normalized_package_list "${input_packages}")"
+log "done"
 
 # Create cache directory so artifacts can be saved.
 mkdir -p ${cache_dir}

src/cmd/apt_query/main.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"awalsh128.com/cache-apt-pkgs-action/src/internal/common"
+	"awalsh128.com/cache-apt-pkgs-action/src/internal/exec"
+	"awalsh128.com/cache-apt-pkgs-action/src/internal/logging"
+)
+
+type AptPackage struct {
+	Name    string
+	Version string
+}
+
+type AptPackages []AptPackage
+
+func (ps AptPackages) serialize() string {
+	tokens := []string{}
+	for _, p := range ps {
+		tokens = append(tokens, p.Name+"="+p.Version)
+	}
+	return strings.Join(tokens, " ")
+}
+
+// Gets the APT based packages as a sorted by name list (normalized).
+func getPackages(executor exec.Executor, names []string) AptPackages {
+	prefixArgs := []string{"--quiet=0", "--no-all-versions", "show"}
+	execution := executor.Exec("apt-cache", append(prefixArgs, names...)...)
+
+	err := execution.Error()
+	if err != nil {
+		logging.Fatal(err)
+	}
+
+	pkgs := []AptPackage{}
+	errorMessages := []string{}
+
+	for _, paragraph := range strings.Split(execution.Stdout, "\n\n") {
+		pkg := AptPackage{}
+		for _, line := range strings.Split(paragraph, "\n") {
+			if strings.HasPrefix(line, "Package: ") {
+				pkg.Name = strings.TrimSpace(strings.SplitN(line, ":", 2)[1])
+			} else if strings.HasPrefix(line, "Version: ") {
+				pkg.Version = strings.TrimSpace(strings.SplitN(line, ":", 2)[1])
+			} else if strings.HasPrefix(line, "N: Unable to locate package ") || strings.HasPrefix(line, "E: ") {
+				if !common.ContainsString(errorMessages, line) {
+					errorMessages = append(errorMessages, line)
+				}
+			}
+		}
+		if pkg.Name != "" {
+			pkgs = append(pkgs, pkg)
+		}
+	}
+
+	if len(errorMessages) > 0 {
+		logging.Fatalf("Errors encountered in apt-cache output (see below):\n%s", strings.Join(errorMessages, "\n"))
+	}
+
+	sort.Slice(pkgs, func(i, j int) bool {
+		return pkgs[i].Name < pkgs[j].Name
+	})
+
+	return pkgs
+}
+
+func getExecutor(replayFilename string) exec.Executor {
+	if len(replayFilename) == 0 {
+		return &exec.BinExecutor{}
+	}
+	return exec.NewReplayExecutor(replayFilename)
+}
+
+func main() {
+	debug := flag.Bool("debug", false, "Log diagnostic information to a file alongside the binary.")
+
+	replayFilename := flag.String("replayfile", "",
+		"Replay command output from a specified file rather than executing a binary."+
+			"The file should be in the same format as the log generated by the debug flag.")
+
+	flag.Parse()
+	unparsedFlags := flag.Args()
+
+	logging.Init(os.Args[0]+".log", *debug)
+
+	executor := getExecutor(*replayFilename)
+
+	if len(unparsedFlags) < 2 {
+		logging.Fatalf("Expected at least 2 non-flag arguments but found %d.", len(unparsedFlags))
+		return
+	}
+	command := unparsedFlags[0]
+	pkgNames := unparsedFlags[1:]
+
+	switch command {
+
+	case "normalized-list":
+		pkgs := getPackages(executor, pkgNames)
+		fmt.Println(pkgs.serialize())
+
+	default:
+		logging.Fatalf("Command '%s' not recognized.", command)
+	}
+}

src/cmd/apt_query/main_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"flag"
+	"testing"
+
+	"awalsh128.com/cache-apt-pkgs-action/src/internal/cmdtesting"
+)
+
+var createReplayLogs bool = false
+
+func init() {
+	flag.BoolVar(&createReplayLogs, "createreplaylogs", false, "Execute the test commands, save the command output for future replay and skip the tests themselves.")
+}
+
+func TestMain(m *testing.M) {
+	cmdtesting.TestMain(m)
+}
+
+func TestNormalizedList_MultiplePackagesExists_StdoutsAlphaSortedPackageNameVersionPairs(t *testing.T) {
+	result := cmdtesting.New(t, createReplayLogs).Run("normalized-list", "xdot", "rolldice")
+	result.ExpectSuccessfulOut("rolldice=1.16-1build1 xdot=1.2-3")
+}
+
+func TestNormalizedList_SamePackagesDifferentOrder_StdoutsMatch(t *testing.T) {
+	expected := "rolldice=1.16-1build1 xdot=1.2-3"
+
+	ct := cmdtesting.New(t, createReplayLogs)
+
+	result := ct.Run("normalized-list", "rolldice", "xdot")
+	result.ExpectSuccessfulOut(expected)
+
+	result = ct.Run("normalized-list", "xdot", "rolldice")
+	result.ExpectSuccessfulOut(expected)
+}
+
+func TestNormalizedList_MultiVersionWarning_StdoutSingleVersion(t *testing.T) {
+	var result = cmdtesting.New(t, createReplayLogs).Run("normalized-list", "libosmesa6-dev", "libgl1-mesa-dev")
+	result.ExpectSuccessfulOut("libgl1-mesa-dev=23.0.4-0ubuntu1~23.04.1 libosmesa6-dev=23.0.4-0ubuntu1~23.04.1")
+}
+
+func TestNormalizedList_SinglePackageExists_StdoutsSinglePackageNameVersionPair(t *testing.T) {
+	var result = cmdtesting.New(t, createReplayLogs).Run("normalized-list", "xdot")
+	result.ExpectSuccessfulOut("xdot=1.2-3")
+}
+
+func TestNormalizedList_VersionContainsColon_StdoutsEntireVersion(t *testing.T) {
+	var result = cmdtesting.New(t, createReplayLogs).Run("normalized-list", "default-jre")
+	result.ExpectSuccessfulOut("default-jre=2:1.17-74")
+}
+
+func TestNormalizedList_NonExistentPackageName_StderrsAptCacheErrors(t *testing.T) {
+	var result = cmdtesting.New(t, createReplayLogs).Run("normalized-list", "nonexistentpackagename")
+	result.ExpectError(
+		`Error encountered running apt-cache --quiet=0 --no-all-versions show nonexistentpackagename
+Exited with status code 100; see combined output below:
+N: Unable to locate package nonexistentpackagename
+N: Unable to locate package nonexistentpackagename
+E: No packages found`)
+}
+
+func TestNormalizedList_NoPackagesGiven_StderrsArgMismatch(t *testing.T) {
+	var result = cmdtesting.New(t, createReplayLogs).Run("normalized-list")
+	result.ExpectError("Expected at least 2 non-flag arguments but found 1.")
+}