#98 Fixed detection of IMPORT LOCAL FILE (#99)

.github/workflows/go.yml

@@ -10,10 +10,10 @@ jobs:
     strategy:
       matrix:
         go: ["1.20", "1.21"]
-        db: ["7.1.23", "8.22.0"]
+        db: ["7.1.23", "8.23.0"]
     env:
       DEFAULT_GO: "1.21"
-      DEFAULT_DB: "8.22.0"
+      DEFAULT_DB: "8.23.0"
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-go-${{ matrix.go }}-db-${{ matrix.db }}
       cancel-in-progress: true

.project-keeper.yml

@@ -1,4 +1,4 @@
 sources:
   - type: golang
     path: go.mod
-version: 1.0.3
+version: 1.0.4

README.md

@@ -129,15 +129,14 @@ To rollback a transaction use `Rollback()`:
 err = transaction.Rollback()
 ```
 
-## Import local CSV files
+## Import Local CSV Files
 
-Use the sql driver to load data into your Exasol Database.
+Use the sql driver to load data from one or more CSV files into your Exasol Database. These files must be local to the machine where you execute the `IMPORT` statement.
 
-```
-!! Limitation !!
+**Limitations:**
+* Only import of CSV files is supported at the moment, FBV is not supported.
+* The `SECURE` option is not supported at the moment.
 
-Only import of CSV files is supported at the moment.
-```
 
 ```go
 result, err := exasol.Exec(`
@@ -148,6 +147,8 @@ IMPORT INTO CUSTOMERS FROM LOCAL CSV FILE './testData/data.csv' FILE './testData
 `)
 ```
 
+See also the [usage notes](https://docs.exasol.com/db/latest/sql/import.htm#UsageNotes) about the `file_src` element for local files of the `IMPORT` statement.
+
 ## Connection String
 
 The golang Driver uses the following URL structure for Exasol:

doc/changes/changes_1.0.4.md

@@ -0,0 +1,23 @@
+# Exasol Driver go 1.0.4, released 2023-10-23
+
+Code name: Fixed `IMPORT LOCAL CSV FILE`
+
+## Summary
+
+This release fixes the detection of `IMPORT LOCAL CSV FILE`. Before, the Go driver also detected this inside strings which broke e.g. running the following `INSERT` statement:
+
+```sql
+insert into table1 values ('import into {{dest.schema}}.{{dest.table}} ) from local csv file ''{{file.path}}'' ');
+```
+
+Thanks to [@cyrixsimon](https://github.com/cyrixsimon) and [@ssteinhauser](https://github.com/ssteinhauser) for reporting this.
+
+## Bugfixes
+
+* #98: Fixed detection of `IMPORT LOCAL CSV FILE`
+
+## Dependency Updates
+
+### Compile Dependency Updates
+
+* Updated `github.com/exasol/exasol-test-setup-abstraction-server/go-client:v0.3.3` to `v0.3.4`

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/exasol/error-reporting-go v0.2.0
-	github.com/exasol/exasol-test-setup-abstraction-server/go-client v0.3.3
+	github.com/exasol/exasol-test-setup-abstraction-server/go-client v0.3.4
 	github.com/gorilla/websocket v1.5.0
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/goleak v1.2.1
@@ -18,5 +18,4 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/stretchr/objx v0.5.1 // indirect
-	golang.org/x/net v0.17.0 // indirect
 )

internal/utils/helper.go

@@ -13,10 +13,6 @@ import (
 	"github.com/exasol/exasol-driver-go/pkg/errors"
 )
 
-var localImportRegex = regexp.MustCompile(`(?i)(FROM LOCAL CSV )`)
-var fileQueryRegex = regexp.MustCompile(`(?i)(FILE\s+(["|'])?(?P<File>[a-zA-Z0-9:<> \\\/._]+)(["|']? ?))`)
-var rowSeparatorQueryRegex = regexp.MustCompile(`(?i)(ROW\s+SEPARATOR\s+=\s+(["|'])?(?P<RowSeparator>[a-zA-Z]+)(["|']?))`)
-
 func NamedValuesToValues(namedValues []driver.NamedValue) ([]driver.Value, error) {
 	values := make([]driver.Value, len(namedValues))
 	for index, namedValue := range namedValues {
@@ -39,15 +35,30 @@ func BoolToPtr(b bool) *bool {
 	return &b
 }
 
+const WHITESPACE = `\s+`
+
+var localImportRegex = regexp.MustCompile(`(?ims)^\s*IMPORT[\s(]+.+FROM` + WHITESPACE + `LOCAL` + WHITESPACE + `CSV.*$`)
+
 func IsImportQuery(query string) bool {
 	return localImportRegex.MatchString(query)
 }
 
+const ROW_SEPARATOR_PLACEHOLDER = "RowSeparatorPlaceholder"
+const QUOTE = `["']`
+
+func namedGroup(name, regexp string) string {
+	return fmt.Sprintf("(?P<%s>%s)", name, regexp)
+}
+
+var rowSeparatorQueryRegex = regexp.MustCompile(`(?i)` +
+	`ROW` + WHITESPACE + `SEPARATOR` + WHITESPACE + `=` + WHITESPACE +
+	QUOTE + namedGroup(ROW_SEPARATOR_PLACEHOLDER, "[a-zA-Z]+") + QUOTE)
+
 func GetRowSeparator(query string) string {
 	r := rowSeparatorQueryRegex.FindStringSubmatch(query)
 	separator := "LF"
 	for i, name := range rowSeparatorQueryRegex.SubexpNames() {
-		if name == "RowSeparator" && len(r) >= i {
+		if name == ROW_SEPARATOR_PLACEHOLDER && len(r) >= i {
 			separator = r[i]
 		}
 	}
@@ -62,12 +73,17 @@ func GetRowSeparator(query string) string {
 	}
 }
 
+const FILE_PLACEHOLDER = "FilePlaceholder"
+
+var fileQueryRegex = regexp.MustCompile(`(?i)` + `FILE` + WHITESPACE +
+	QUOTE + namedGroup(FILE_PLACEHOLDER, `[a-zA-Z0-9:<> \\\/._-]+`) + QUOTE + ` ?`)
+
 func GetFilePaths(query string) ([]string, error) {
 	r := fileQueryRegex.FindAllStringSubmatch(query, -1)
 	var files []string
 	for _, matches := range r {
 		for i, name := range fileQueryRegex.SubexpNames() {
-			if name == "File" {
+			if name == FILE_PLACEHOLDER {
 				files = append(files, matches[i])
 			}
 		}
@@ -87,6 +103,9 @@ func OpenFile(path string) (*os.File, error) {
 }
 
 func UpdateImportQuery(query string, host string, port int) string {
+	if !IsImportQuery(query) {
+		return query
+	}
 	r := fileQueryRegex.FindAllStringSubmatch(query, -1)
 	for i, matches := range r {
 		if i == 0 {

internal/utils/helper_test.go

@@ -26,7 +26,27 @@ func TestNamedValuesToValuesInvalidName(t *testing.T) {
 }
 
 func TestIsImportQuery(t *testing.T) {
-	assert.True(t, IsImportQuery("IMPORT into <targettable> from local CSV file '/path/to/filename.csv' <optional options>;\n"))
+	tests := []struct {
+		name           string
+		query          string
+		expectedResult bool
+	}{
+		{name: "with options", query: "IMPORT into <targettable> from local CSV file '/path/to/filename.csv' <optional options>;\n", expectedResult: true},
+		{name: "upper case", query: "IMPORT INTO SCHEMA.TABLE FROM LOCAL CSV FILE '/path/to/filename.csv'", expectedResult: true},
+		{name: "with brackets", query: "IMPORT(something) INTO SCHEMA.TABLE FROM LOCAL CSV FILE '/path/to/filename.csv'", expectedResult: true},
+		{name: "lower case", query: "import into schema.table from local csv file '/path/to/filename.csv'", expectedResult: true},
+		{name: "with additional whitespace", query: " IMPORT \t INTO SCHEMA.TABLE\n\tFROM  LOCAL  CSV  FILE  '/path/to/filename.csv'", expectedResult: true},
+		{name: "FBV not supported", query: "IMPORT INTO SCHEMA.TABLE FROM LOCAL FBV FILE '/path/to/filename.fbf'", expectedResult: false},
+		{name: "select query unsupported", query: "select * from schema.table", expectedResult: false},
+		{name: "import in string with placeholders", query: "insert into table1 values ('import into {{dest.schema}}.{{dest.table}} ) from local csv file ''{{file.path}}'' ');", expectedResult: false},
+		{name: "import in string", query: "insert into table1 values ('import into schema.table from local csv file ''/path/to/filename.csv''');", expectedResult: false},
+		{name: "import in string with schema", query: "insert into schema.tab1 values ('IMPORT into schema.table FROM LOCAL CSV file ''/path/to/filename.csv'';')", expectedResult: false},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expectedResult, IsImportQuery(test.query))
+		})
+	}
 }
 
 func TestGetFilePathNotFound(t *testing.T) {
@@ -47,32 +67,48 @@ func TestOpenFile(t *testing.T) {
 }
 
 func TestUpdateImportQuery(t *testing.T) {
-	query := "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv'"
-	newQuery := UpdateImportQuery(query, "127.0.0.1", 4333)
-	assert.Equal(t, "IMPORT into table FROM CSV AT 'http://127.0.0.1:4333' FILE 'data.csv' ", newQuery)
-}
-
-func TestUpdateImportQueryMulti(t *testing.T) {
-	query := "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' file '/path/to/filename2.csv'"
-	newQuery := UpdateImportQuery(query, "127.0.0.1", 4333)
-	assert.Equal(t, "IMPORT into table FROM CSV AT 'http://127.0.0.1:4333' FILE 'data.csv' ", newQuery)
-}
-
-func TestUpdateImportQueryMulti2(t *testing.T) {
-	query := "IMPORT INTO table_1 FROM LOCAL CSV USER 'agent_007' IDENTIFIED BY 'secret' FILE 'tab1_part1.csv' FILE 'tab1_part2.csv' COLUMN SEPARATOR = ';' SKIP = 5;"
-	newQuery := UpdateImportQuery(query, "127.0.0.1", 4333)
-	assert.Equal(t, "IMPORT INTO table_1 FROM CSV AT 'http://127.0.0.1:4333' USER 'agent_007' IDENTIFIED BY 'secret' FILE 'data.csv' COLUMN SEPARATOR = ';' SKIP = 5;", newQuery)
+	tests := []struct {
+		name     string
+		query    string
+		expected string
+	}{
+		{name: "non import query",
+			query:    "select * from table",
+			expected: "select * from table"},
+		{name: "import statement in a string",
+			query:    "insert into tab1 values ('IMPORT into table FROM LOCAL CSV file ''/path/to/filename.csv'';')",
+			expected: "insert into tab1 values ('IMPORT into table FROM LOCAL CSV file ''/path/to/filename.csv'';')"},
+		{name: "non import query",
+			query:    "select * from table",
+			expected: "select * from table"},
+		{name: "single file",
+			query:    "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv'",
+			expected: "IMPORT into table FROM CSV AT 'http://127.0.0.1:4333' FILE 'data.csv' "},
+		{name: "multi",
+			query:    "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' file '/path/to/filename2.csv'",
+			expected: "IMPORT into table FROM CSV AT 'http://127.0.0.1:4333' FILE 'data.csv' "},
+		{name: "with options",
+			query:    "IMPORT INTO table_1 FROM LOCAL CSV USER 'agent_007' IDENTIFIED BY 'secret' FILE 'tab1_part1.csv' FILE 'tab1_part2.csv' COLUMN SEPARATOR = ';' SKIP = 5;",
+			expected: "IMPORT INTO table_1 FROM CSV AT 'http://127.0.0.1:4333' USER 'agent_007' IDENTIFIED BY 'secret' FILE 'data.csv' COLUMN SEPARATOR = ';' SKIP = 5;"},
+		{name: "with newline",
+			query:    "IMPORT INTO table_1\nFROM LOCAL CSV USER 'agent_007' IDENTIFIED BY 'secret' FILE 'tab1_part1.csv' FILE 'tab1_part2.csv' COLUMN SEPARATOR = ';'\r\nSKIP = 5;",
+			expected: "IMPORT INTO table_1\nFROM CSV AT 'http://127.0.0.1:4333' USER 'agent_007' IDENTIFIED BY 'secret' FILE 'data.csv' COLUMN SEPARATOR = ';'\r\nSKIP = 5;"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			updatedQuery := UpdateImportQuery(test.query, "127.0.0.1", 4333)
+			assert.Equal(t, test.expected, updatedQuery)
+		})
+	}
 }
 
 func TestGetFilePaths(t *testing.T) {
 	quotes := []struct {
 		name  string
 		value string
 	}{
-		{name: "SingleQuote",
-			value: "'"},
-		{name: "DoubleQuote",
-			value: `"`},
+		{name: "SingleQuote", value: "'"},
+		{name: "DoubleQuote", value: `"`},
 	}
 
 	tests := []struct {
@@ -82,15 +118,17 @@ func TestGetFilePaths(t *testing.T) {
 		{name: "Single file", paths: []string{"/path/to/filename.csv"}},
 		{name: "Multi file", paths: []string{"/path/to/filename.csv", "/path/to/filename2.csv"}},
 		{name: "Relative paths", paths: []string{"./tab1_part1.csv", "./tab1_part2.csv"}},
+		{name: "Local Dir", paths: []string{"tab1_part1.csv", "tab1_part2.csv"}},
 		{name: "Windows paths", paths: []string{"C:\\Documents\\Newsletters\\Summer2018.csv", "\\Program Files\\Custom Utilities\\StringFinder.csv"}},
 		{name: "Unix paths", paths: []string{"/Users/User/Documents/Data/test.csv"}},
+		{name: "With dash", paths: []string{"/Users/User/Documents/Data/test-1.csv"}},
 	}
 
 	for _, quote := range quotes {
-		for _, tt := range tests {
-			t.Run(fmt.Sprintf("%s %s", tt.name, quote.name), func(t *testing.T) {
+		for _, test := range tests {
+			t.Run(fmt.Sprintf("%s %s", test.name, quote.name), func(t *testing.T) {
 				var preparedPaths []string
-				for _, path := range tt.paths {
+				for _, path := range test.paths {
 					preparedPaths = append(preparedPaths, fmt.Sprintf("%s%s%s", quote.value, path, quote.value))
 				}
 
@@ -100,25 +138,36 @@ func TestGetFilePaths(t *testing.T) {
        			COLUMN SEPARATOR = ';'
        			SKIP = 5;`, strings.Join(preparedPaths, " FILE ")))
 				assert.NoError(t, err)
-				assert.ElementsMatch(t, tt.paths, foundPaths)
+				assert.ElementsMatch(t, test.paths, foundPaths)
 			})
 		}
 	}
 }
 
-func TestGetRowSeparatorLF(t *testing.T) {
-	query := "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR = 'LF'"
-	assert.Equal(t, GetRowSeparator(query), "\n")
-}
-
-func TestGetRowSeparatorCR(t *testing.T) {
-	query := "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR = 'CR'"
-	assert.Equal(t, GetRowSeparator(query), "\r")
-}
-
-func TestGetRowSeparatorCRLF(t *testing.T) {
-	query := "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR =  'CRLF'"
-	assert.Equal(t, GetRowSeparator(query), "\r\n")
+func TestGetRowSeparatorCompleteQuery(t *testing.T) {
+	tests := []struct {
+		name     string
+		query    string
+		expected string
+	}{
+		{name: "LF", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR = 'LF'", expected: "\n"},
+		{name: "CR", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR = 'CR'", expected: "\r"},
+		{name: "CRLF", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR = 'CRLF'", expected: "\r\n"},
+		{name: "only row separator fragment", query: "ROW SEPARATOR = 'CRLF'", expected: "\r\n"},
+		{name: "pipe as quote char not supported", query: "ROW SEPARATOR = |CRLF|", expected: "\n"},
+		{name: "unknown value returns default", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR = 'unknown'", expected: "\n"},
+		{name: "missing expression returns default", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv'", expected: "\n"},
+		{name: "trailing text", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR = 'CRLF' trailing text", expected: "\r\n"},
+		{name: "multiple spaces", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR \t = \t 'CRLF';", expected: "\r\n"},
+		{name: "no spaces returns default", query: "IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR='CRLF';", expected: "\n"},
+		{name: "with line breaks", query: "IMPORT into table\nFROM LOCAL CSV file '/path/to/filename.csv'\nROW\r\nSEPARATOR = 'CRLF';", expected: "\r\n"},
+		{name: "unknown query returns default", query: "select * from table", expected: "\n"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expected, GetRowSeparator(test.query))
+		})
+	}
 }
 
 func TestGetRowSeparator(t *testing.T) {
@@ -129,10 +178,13 @@ func TestGetRowSeparator(t *testing.T) {
 	}{
 		{name: "LF", separator: "LF", want: "\n"},
 		{name: "LF lowercase", separator: "lf", want: "\n"},
+		{name: "Lf mixed case returns default", separator: "Lf", want: "\n"},
 		{name: "CRLF", separator: "CRLF", want: "\r\n"},
 		{name: "CRLF lowercase", separator: "crlf", want: "\r\n"},
+		{name: "CrLf mixed case returns default", separator: "CrLf", want: "\n"},
 		{name: "CR", separator: "CR", want: "\r"},
 		{name: "CR lowercase", separator: "cr", want: "\r"},
+		{name: "Cr mixed case returns default", separator: "Cr", want: "\n"},
 	}
 	for _, tt := range tests {
 		query := fmt.Sprintf("IMPORT into table FROM LOCAL CSV file '/path/to/filename.csv' ROW SEPARATOR =  '%s'", tt.separator)

internal/version/version.go

@@ -1,3 +1,3 @@
 package version
 
-const DriverVersion = "v1.0.3"
+const DriverVersion = "v1.0.4"