Merge pull request #72 from gdubya/main

Add support for abfss://container@storage.endpoint/path URL syntax
duckdb · Aug 30, 2024 · eddc484 · eddc484
2 parents b3e9e08 + 690dc5a
commit eddc484
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 15 deletions.
diff --git a/src/azure_parsed_url.cpp b/src/azure_parsed_url.cpp
@@ -7,7 +7,8 @@ namespace duckdb {
 AzureParsedUrl ParseUrl(const std::string &url) {
 	constexpr auto invalid_url_format =
 	    "The URL %s does not match the expected formats: (azure|az)://<container>/[<path>] or the fully qualified one: "
-	    "(azure|az)://<storage account>.<endpoint>/<container>/[<path>]";
+	    "(abfss|azure|az)://<storage account>.<endpoint>/<container>/[<path>] "
+		"or abfss://<container>@<storage account>.<endpoint>/[<path>]";
 	bool is_fully_qualified;
 	std::string container, storage_account_name, endpoint, prefix, path;
 
@@ -22,26 +23,41 @@ AzureParsedUrl ParseUrl(const std::string &url) {
 	// they will be no more changes to path format.
 	const auto dot_pos = url.find('.', prefix_end_pos);
 	const auto slash_pos = url.find('/', prefix_end_pos);
+	const auto at_pos = url.find('@', prefix_end_pos);
 	if (slash_pos == std::string::npos) {
 		throw duckdb::IOException(invalid_url_format, url);
 	}
 
 	if (dot_pos != std::string::npos && dot_pos < slash_pos) {
-		// syntax is (azure|az)://<storage account>.<endpoint>/<container>/[<path>]
-		const auto container_slash_pos = url.find('/', dot_pos);
-		if (container_slash_pos == string::npos) {
-			throw IOException(invalid_url_format, url);
-		}
-		const auto path_slash_pos = url.find('/', container_slash_pos + 1);
-		if (path_slash_pos == string::npos) {
-			throw IOException(invalid_url_format, url);
-		}
-
 		is_fully_qualified = true;
-		storage_account_name = url.substr(prefix_end_pos, dot_pos - prefix_end_pos);
-		endpoint = url.substr(dot_pos + 1, container_slash_pos - dot_pos - 1);
-		container = url.substr(container_slash_pos + 1, path_slash_pos - container_slash_pos - 1);
-		path = url.substr(path_slash_pos + 1);
+
+		if (url.rfind(AzureDfsStorageFileSystem::PATH_PREFIX, 0) == 0 &&
+			at_pos != std::string::npos) {
+			// syntax is abfss://<container>@<storage account>.<endpoint>/[<path>]
+			const auto path_slash_pos = url.find('/', prefix_end_pos + 1);
+			if (path_slash_pos == string::npos) {
+				throw IOException(invalid_url_format, url);
+			}			
+
+			container = url.substr(prefix_end_pos, at_pos - prefix_end_pos);
+			storage_account_name = url.substr(at_pos + 1, dot_pos - at_pos - 1);
+			endpoint = url.substr(dot_pos + 1, path_slash_pos - dot_pos - 1);
+			path = url.substr(path_slash_pos + 1);
+		} else {
+			// syntax is (abfss|azure|az)://<storage account>.<endpoint>/<container>/[<path>]
+			const auto container_slash_pos = url.find('/', dot_pos);
+			if (container_slash_pos == string::npos) {
+				throw IOException(invalid_url_format, url);
+			}
+			const auto path_slash_pos = url.find('/', container_slash_pos + 1);
+			if (path_slash_pos == string::npos) {
+				throw IOException(invalid_url_format, url);
+			}
+			storage_account_name = url.substr(prefix_end_pos, dot_pos - prefix_end_pos);
+			endpoint = url.substr(dot_pos + 1, container_slash_pos - dot_pos - 1);
+			container = url.substr(container_slash_pos + 1, path_slash_pos - container_slash_pos - 1);
+			path = url.substr(path_slash_pos + 1);
+		}
 	} else {
 		// syntax is (azure|az)://<container>/[<path>]
 		// Storage account name will be retrieve from the variables or the secret information

diff --git a/test/sql/cloud/hierarchical_namespace.test b/test/sql/cloud/hierarchical_namespace.test
@@ -69,6 +69,12 @@ SELECT count(*) FROM 'abfss://${AZURE_STORAGE_ACCOUNT}.dfs.core.windows.net/test
 ----
 2317
 
+# Check fully qualified name abfss alternative syntax
+query I
+SELECT count(*) FROM 'abfss://testing-private@${AZURE_STORAGE_ACCOUNT}.dfs.core.windows.net/partitioned/l_receipmonth=*/l_shipmode=TRUCK/*.csv';
+----
+2317
+
 # Enable http info for the explain analyze statement
 statement ok
 SET azure_http_stats = true;