Support glob patterns in the configuration file

This makes the configuration file more flexible.

Closes: #208

Signed-off-by: Ludovico de Nittis <ludovico.denittis@collabora.com>

README.md

@@ -241,8 +241,8 @@ Available configuration values:
 
 - `http-timeout` *DEPRECATED, see `store-options.<Location>.timeout`* - HTTP request timeout used in HTTP stores (not S3) in nanoseconds
 - `http-error-retry` *DEPRECATED, see `store-options.<Location>.error-retry` - Number of times to retry failed chunk requests from HTTP stores
-- `s3-credentials` - Defines credentials for use with S3 stores. Especially useful if more than one S3 store is used. The key in the config needs to be the URL scheme and host used for the store, excluding the path, but including the port number if used in the store URL. It is also possible to use a [standard aws credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) in order to store s3 credentials.
-- `store-options` - Allows customization of chunk and index stores, for example comression settings, timeouts, retry behavior and keys. Not all options are applicable to every store, some of these like `timeout` are ignored for local stores. Some of these options, such as the client certificates are overwritten with any values set in the command line. Note that the store location used in the command line needs to match the key under `store-options` exactly for these options to be used. Watch out for trailing `/` in URLs.
+- `s3-credentials` - Defines credentials for use with S3 stores. Especially useful if more than one S3 store is used. The key in the config needs to be the URL scheme and host used for the store, excluding the path, but including the port number if used in the store URL. The key can also contain glob patterns, and the available wildcards are `*`, `?` and `[…]`. Please refer to the [filepath.Match documentation] for additional information. It is also possible to use a [standard aws credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) in order to store s3 credentials.
+- `store-options` - Allows customization of chunk and index stores, for example compression settings, timeouts, retry behavior and keys. Not all options are applicable to every store, some of these like `timeout` are ignored for local stores. Some of these options, such as the client certificates are overwritten with any values set in the command line. Note that the store location used in the command line needs to match the key under `store-options` exactly for these options to be used. As for the `s3-credentials`, glob patterns are also supported. A configuration file where more than one key matches a single store location, is considered invalid.
   - `timeout` - Time limit for chunk read or write operation in nanoseconds. Default: 1 minute. If set to a negative value, timeout is infinite.
   - `error-retry` - Number of times to retry failed chunk requests. Default: 0.
   - `error-retry-base-interval` - Number of nanoseconds to wait before first retry attempt. Retry attempt number N for the same request will wait N times this interval. Default: 0.

cmd/desync/location.go

@@ -2,36 +2,42 @@ package main
 
 import (
 	"net/url"
-	"path"
 	"path/filepath"
+	"strings"
 )
 
 // Returns true if the two locations are equal. Locations can be URLs or local file paths.
 // It can handle Unix as well as Windows paths. Example
 // http://host/path/ is equal http://host/path (no trailing /) and /tmp/path is
 // equal \tmp\path on Windows.
-func locationMatch(loc1, loc2 string) bool {
-	u1, _ := url.Parse(loc1)
-	u2, _ := url.Parse(loc2)
-	// See if we have at least one URL, Windows drive letters come out as single-letter
-	// scheme so we need more here.
-	if len(u1.Scheme) > 1 || len(u2.Scheme) > 1 {
-		if u1.Scheme != u2.Scheme || u1.Host != u2.Host {
-			return false
-		}
-		// URL paths should only use /, use path (not filepath) package to clean them
-		// before comparing
-		return path.Clean(u1.Path) == path.Clean(u2.Path)
+func locationMatch(pattern, loc string) bool {
+	l, err := url.Parse(loc)
+	if err != nil {
+		return false
+	}
+
+	// See if we have a URL, Windows drive letters come out as single-letter
+	// scheme, so we need more here.
+	if len(l.Scheme) > 1 {
+		// URL paths should only use / as separator, remove the trailing one, if any
+		trimmedLoc := strings.TrimSuffix(loc, "/")
+		trimmedPattern := strings.TrimSuffix(pattern, "/")
+		m, _ := filepath.Match(trimmedPattern, trimmedLoc)
+		return m
 	}
 
-	// We're dealing with two paths.
-	p1, err := filepath.Abs(loc1)
+	// We're dealing with a path.
+	p1, err := filepath.Abs(pattern)
+	if err != nil {
+		return false
+	}
+	p2, err := filepath.Abs(loc)
 	if err != nil {
 		return false
 	}
-	p2, err := filepath.Abs(loc2)
+	m, err := filepath.Match(p1, p2)
 	if err != nil {
 		return false
 	}
-	return p1 == p2
+	return m
 }

cmd/desync/location_test.go

@@ -12,6 +12,29 @@ func TestLocationEquality(t *testing.T) {
 	require.True(t, locationMatch("http://host/path", "http://host/path"))
 	require.True(t, locationMatch("http://host/path/", "http://host/path/"))
 	require.True(t, locationMatch("http://host/path", "http://host/path/"))
+	require.True(t, locationMatch("https://host/", "https://host"))
+	require.True(t, locationMatch("https://host", "https://host/"))
+	require.True(t, locationMatch("https://host", "https://host"))
+	require.True(t, locationMatch("https://host/", "https://host/"))
+	require.True(t, locationMatch("s3+https://example.com", "s3+https://example.com"))
+
+	// Equal URLs with globs
+	require.True(t, locationMatch("https://host/path*", "https://host/path"))
+	require.True(t, locationMatch("https://host/path*", "https://host/path/"))
+	require.True(t, locationMatch("https://*", "https://example.com"))
+	require.True(t, locationMatch("https://example.com/path/*", "https://example.com/path/another"))
+	require.True(t, locationMatch("https://example.com/path/*", "https://example.com/path/another/"))
+	require.True(t, locationMatch("https://example.com/*/*/", "https://example.com/path/another/"))
+	require.True(t, locationMatch("https://example.com/*/", "https://example.com/2022.01/"))
+	require.True(t, locationMatch("https://*/*/*", "https://example.com/path/another/"))
+	require.True(t, locationMatch("https://example.*", "https://example.com"))
+	require.True(t, locationMatch("*://example.com", "https://example.com"))
+	require.True(t, locationMatch("http*://example.com", "https://example.com"))
+	require.True(t, locationMatch("http*://example.com", "http://example.com"))
+	require.True(t, locationMatch("https://exampl?.*", "https://example.com"))
+	require.True(t, locationMatch("http://examp??.com", "http://example.com"))
+	require.True(t, locationMatch("https://example.com/?", "https://example.com/a"))
+	require.True(t, locationMatch("https://example.com/fo[a-z]", "https://example.com/foo"))
 
 	// Not equal URLs
 	require.False(t, locationMatch("http://host:8080/path", "http://host/path"))
@@ -20,10 +43,23 @@ func TestLocationEquality(t *testing.T) {
 	require.False(t, locationMatch("http://host1/path", "http://host2/path"))
 	require.False(t, locationMatch("sftp://host/path", "http://host/path"))
 	require.False(t, locationMatch("ssh://host/path", "/path"))
+	require.False(t, locationMatch("ssh://host/path", "/host/path"))
+	require.False(t, locationMatch("ssh://host/path", "/ssh/host/path"))
+
+	// Not equal URLs with globs
+	require.False(t, locationMatch("*", "https://example.com/path"))
+	require.False(t, locationMatch("https://*", "https://example.com/path"))
+	require.False(t, locationMatch("https://example.com/*", "https://example.com/path/another"))
+	require.False(t, locationMatch("https://example.com/path/*", "https://example.com/path"))
+	require.False(t, locationMatch("http://*", "https://example.com"))
+	require.False(t, locationMatch("http?://example.com", "http://example.com"))
+	require.False(t, locationMatch("https://example.com/123?", "https://example.com/12345"))
+	require.False(t, locationMatch("*://example.com", "https://example.com/123"))
 
 	// Equal paths
 	require.True(t, locationMatch("/path", "/path/../path"))
 	require.True(t, locationMatch("//path", "//path"))
+	require.True(t, locationMatch("//path", "/path"))
 	require.True(t, locationMatch("./path", "./path"))
 	require.True(t, locationMatch("path", "path/"))
 	require.True(t, locationMatch("path/..", "."))
@@ -32,11 +68,48 @@ func TestLocationEquality(t *testing.T) {
 		require.True(t, locationMatch("/path/to/somewhere", "\\path\\to\\somewhere\\"))
 	}
 
+	// Equal paths with globs
+	require.True(t, locationMatch("/path*", "/path/../path"))
+	require.True(t, locationMatch("/path*", "/path_1"))
+	require.True(t, locationMatch("/path/*", "/path/to"))
+	require.True(t, locationMatch("/path/*", "/path/to/"))
+	require.True(t, locationMatch("/path/*/", "/path/to/"))
+	require.True(t, locationMatch("/path/*/", "/path/to"))
+	require.True(t, locationMatch("/path/to/../*", "/path/another"))
+	require.True(t, locationMatch("/*", "/path"))
+	require.True(t, locationMatch("*", "path"))
+	require.True(t, locationMatch("/pat?", "/path"))
+	require.True(t, locationMatch("/pat?/?", "/path/1"))
+	require.True(t, locationMatch("path/*", "path/to"))
+	require.True(t, locationMatch("path/?", "path/1"))
+	require.True(t, locationMatch("?", "a"))
+	if runtime.GOOS == "windows" {
+		require.True(t, locationMatch("c:\\path\\to\\*", "c:\\path\\to\\somewhere\\"))
+		require.True(t, locationMatch("/path/to/*", "\\path\\to\\here\\"))
+		require.True(t, locationMatch("c:\\path\\to\\?", "c:\\path\\to\\1\\"))
+		require.True(t, locationMatch("/path/to/?", "\\path\\to\\1\\"))
+	}
+
 	// Not equal paths
 	require.False(t, locationMatch("/path", "path"))
 	require.False(t, locationMatch("/path/to", "path/to"))
 	require.False(t, locationMatch("/path/to", "/path/to/.."))
 	if runtime.GOOS == "windows" {
 		require.False(t, locationMatch("c:\\path1", "c:\\path2"))
 	}
+
+	// Not equal paths with globs
+	require.False(t, locationMatch("/path*", "/dir"))
+	require.False(t, locationMatch("/path*", "path"))
+	require.False(t, locationMatch("/path*", "/path/to"))
+	require.False(t, locationMatch("/path/*", "/path"))
+	require.False(t, locationMatch("/path/to/../*", "/path/to/another"))
+	require.False(t, locationMatch("/pat?", "/pat"))
+	require.False(t, locationMatch("/pat?", "/dir"))
+	if runtime.GOOS == "windows" {
+		require.True(t, locationMatch("c:\\path\\to\\*", "c:\\path\\to\\"))
+		require.True(t, locationMatch("/path/to/*", "\\path\\to\\"))
+		require.True(t, locationMatch("c:\\path\\to\\?", "c:\\path\\to\\123\\"))
+		require.True(t, locationMatch("/path/to/?", "\\path\\to\\123\\"))
+	}
 }