urlencoded path params

Signed-off-by: Maciej Obuchowski <maciej.obuchowski@getindata.com>
MarquezProject · Jun 21, 2021 · da0ed70 · da0ed70
1 parent 9e41452
commit da0ed70
Show file tree

Hide file tree

Showing 21 changed files with 469 additions and 43 deletions.
diff --git a/api/src/main/java/marquez/common/models/NamespaceName.java b/api/src/main/java/marquez/common/models/NamespaceName.java
@@ -41,7 +41,8 @@ public NamespaceName(@NonNull final String value) {
     checkArgument(
         PATTERN.matcher(value).matches(),
         "namespace '%s' must contain only letters (a-z, A-Z), numbers (0-9), "
-            + "underscores (_), dashes (-), or dots (.) with a maximum length of %s characters.",
+            + "underscores (_), dashes (-), colons (:), slashes (/) or dots (.) with a maximum "
+            + "length of %s characters.",
         value,
         MAX_SIZE);
     this.value = value;

diff --git a/api/src/test/java/marquez/BaseIntegrationTest.java b/api/src/test/java/marquez/BaseIntegrationTest.java
@@ -204,7 +204,7 @@ protected static PostgresContainer createMarquezPostgres() {
   protected CompletableFuture<HttpResponse<String>> sendLineage(String body) {
     HttpRequest request =
         HttpRequest.newBuilder()
-            .uri(URI.create(baseUrl + "/api/v1/lineage"))
+            .uri(URI.create(baseUrl + "api/v1/lineage"))
             .header("Content-Type", "application/json")
             .POST(BodyPublishers.ofString(body))
             .build();

diff --git a/api/src/test/java/marquez/MarquezAppIntegrationTest.java b/api/src/test/java/marquez/MarquezAppIntegrationTest.java
@@ -35,49 +35,53 @@
 import marquez.client.models.StreamMeta;
 import marquez.client.models.Tag;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 @org.junit.jupiter.api.Tag("IntegrationTests")
 public class MarquezAppIntegrationTest extends BaseIntegrationTest {
 
-  @Test
-  public void testApp_createNamespace() {
+  @ParameterizedTest
+  @ValueSource(strings = {"DEFAULT", "database://localhost:1234", "s3://bucket", "bigquery:"})
+  public void testApp_createNamespace(String namespaceName) {
     final NamespaceMeta namespaceMeta =
         NamespaceMeta.builder().ownerName(OWNER_NAME).description(NAMESPACE_DESCRIPTION).build();
 
-    final Namespace namespace = client.createNamespace(NAMESPACE_NAME, namespaceMeta);
-    assertThat(namespace.getName()).isEqualTo(NAMESPACE_NAME);
+    final Namespace namespace = client.createNamespace(namespaceName, namespaceMeta);
+    assertThat(namespace.getName()).isEqualTo(namespaceName);
     assertThat(namespace.getCreatedAt()).isAfter(EPOCH);
     assertThat(namespace.getUpdatedAt()).isAfter(EPOCH);
     assertThat(namespace.getOwnerName()).isEqualTo(OWNER_NAME);
     assertThat(namespace.getDescription()).isEqualTo(Optional.of(NAMESPACE_DESCRIPTION));
 
     assertThat(
             client.listNamespaces().stream()
-                .filter(other -> other.getName().equals(NAMESPACE_NAME))
+                .filter(other -> other.getName().equals(namespaceName))
                 .count())
         .isEqualTo(1);
   }
 
-  @Test
-  public void testApp_createSource() {
+  @ParameterizedTest
+  @ValueSource(strings = {"test_source312", "s3://bucket", "asdf", "bigquery:"})
+  public void testApp_createSource(String sourceName) {
     final SourceMeta sourceMeta =
         SourceMeta.builder()
             .type(SOURCE_TYPE)
             .connectionUrl(CONNECTION_URL)
             .description(SOURCE_DESCRIPTION)
             .build();
 
-    final Source source = client.createSource(SOURCE_NAME, sourceMeta);
+    final Source source = client.createSource(sourceName, sourceMeta);
     assertThat(source.getType()).isEqualTo(SOURCE_TYPE);
-    assertThat(source.getName()).isEqualTo(SOURCE_NAME);
+    assertThat(source.getName()).isEqualTo(sourceName);
     assertThat(source.getCreatedAt()).isAfter(EPOCH);
     assertThat(source.getUpdatedAt()).isAfter(EPOCH);
     assertThat(source.getConnectionUrl()).isEqualTo(CONNECTION_URL);
     assertThat(source.getDescription()).isEqualTo(Optional.of(SOURCE_DESCRIPTION));
 
     assertThat(
             client.listSources().stream()
-                .filter(other -> other.getName().equals(SOURCE_NAME))
+                .filter(other -> other.getName().equals(sourceName))
                 .count())
         .isEqualTo(1);
   }

diff --git a/api/src/test/java/marquez/OpenLineageIntegrationTest.java b/api/src/test/java/marquez/OpenLineageIntegrationTest.java
@@ -34,6 +34,7 @@ public class OpenLineageIntegrationTest extends BaseIntegrationTest {
   public static String EVENT_UNICODE = "open_lineage/event_unicode.json";
   public static String EVENT_LARGE = "open_lineage/event_large.json";
   public static String NULL_NOMINAL_END_TIME = "open_lineage/null_nominal_end_time.json";
+  public static String EVENT_NAMESPACE_NAMING = "open_lineage/event_namespace_naming.json";
 
   public static List<String> data() {
     return Arrays.asList(
@@ -43,7 +44,8 @@ public static List<String> data() {
         EVENT_UNICODE,
         // FIXME: A very large event fails the test.
         // EVENT_LARGE,
-        NULL_NOMINAL_END_TIME);
+        NULL_NOMINAL_END_TIME,
+        EVENT_NAMESPACE_NAMING);
   }
 
   @ParameterizedTest

diff --git a/api/src/test/java/marquez/common/models/CommonModelGenerator.java b/api/src/test/java/marquez/common/models/CommonModelGenerator.java
@@ -31,7 +31,7 @@ public final class CommonModelGenerator extends Generator {
   private CommonModelGenerator() {}
 
   public static NamespaceName newNamespaceName() {
-    return NamespaceName.of("test_namespace" + newId());
+    return NamespaceName.of("s3://test_namespace" + newId());
   }
 
   public static OwnerName newOwnerName() {

diff --git a/api/src/test/java/marquez/db/NamespaceDaoTest.java b/api/src/test/java/marquez/db/NamespaceDaoTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package marquez.db;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import marquez.common.models.NamespaceName;
+import marquez.common.models.OwnerName;
+import marquez.jdbi.MarquezJdbiExternalPostgresExtension;
+import marquez.service.models.NamespaceMeta;
+import org.jdbi.v3.core.Jdbi;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(MarquezJdbiExternalPostgresExtension.class)
+public class NamespaceDaoTest {
+
+  private static NamespaceDao namespaceDao;
+
+  @BeforeAll
+  public static void setUpOnce(Jdbi jdbi) {
+    namespaceDao = jdbi.onDemand(NamespaceDao.class);
+  }
+
+  @Test
+  void testWriteAndReadNamespace() {
+    var namespaceName = NamespaceName.of("postgres://localhost:5432");
+    var namespaceMeta = new NamespaceMeta(new OwnerName("marquez"), null);
+    namespaceDao.upsertNamespaceMeta(namespaceName, namespaceMeta);
+
+    assertTrue(namespaceDao.exists(namespaceName.getValue()));
+  }
+}
diff --git a/api/src/test/resources/config.test.yml b/api/src/test/resources/config.test.yml
@@ -2,6 +2,7 @@ server:
   applicationConnectors:
   - type: http
     port: 8080
+    httpCompliance: RFC7230_LEGACY
   adminConnectors:
   - type: http
     port: 8081

diff --git a/api/src/test/resources/open_lineage/event_namespace_naming.json b/api/src/test/resources/open_lineage/event_namespace_naming.json
@@ -0,0 +1,172 @@
+{
+  "eventType": "COMPLETE",
+  "eventTime": "2020-12-28T19:51:01.641Z",
+  "run": {
+    "runId": "ea041791-68bc-4ae1-bd89-4c8106a157e4",
+    "facets": {
+      "nominalTime": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "nominalStartTime": "2020-12-17T03:00:00.001Z",
+        "nominalEndTime": "2020-12-17T04:00:00.001Z"
+      },
+      "parent": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "run": {
+          "runId": "3f5e83fa-3480-44ff-99c5-ff943904e5e8"
+        },
+        "job": {
+          "namespace": "my-scheduler-namespace",
+          "name": "myjob.mytask"
+        }
+      },
+      "additionalProp1": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "additionalProp1": {}
+      },
+      "additionalProp2": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "additionalProp1": {}
+      },
+      "additionalProp3": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "additionalProp1": {}
+      }
+    }
+  },
+  "job": {
+    "namespace": "my-scheduler-namespace",
+    "name": "myjob.mytask",
+    "facets": {
+      "documentation": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "description": "string"
+      },
+      "sourceCodeLocation": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "type": "git",
+        "url": "http://example.com"
+      },
+      "sql": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "additionalPropExample": {
+          "example": true
+        },
+        "query": "SELECT * FROM foo"
+      },
+      "additionalProp1": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "additionalProp1": {}
+      },
+      "additionalProp2": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "additionalProp1": {}
+      },
+      "additionalProp3": {
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+        "additionalProp1": {}
+      }
+    }
+  },
+  "inputs": [
+    {
+      "namespace": "s3://blob-source",
+      "name": "instance.schema.table",
+      "facets": {
+        "documentation": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "description": "canonical representation of entity Foo"
+        },
+        "schema": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "fields": [
+            {
+              "name": "column1",
+              "type": "VARCHAR",
+              "description": "string"
+            }
+          ]
+        },
+        "dataSource": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "name": "s3://blob-source",
+          "uri": "s3://blob-source"
+        },
+        "additionalProp1": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "additionalProp1": {}
+        },
+        "additionalProp2": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "additionalProp1": {}
+        },
+        "additionalProp3": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "additionalProp1": {}
+        }
+      }
+    }
+  ],
+  "outputs": [
+    {
+      "namespace": "s3://blob-sink",
+      "name": "instance.schema.table",
+      "facets": {
+        "documentation": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "description": "canonical representation of entity Foo"
+        },
+        "schema": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "fields": [
+            {
+              "name": "column1",
+              "type": "VARCHAR",
+              "description": "string"
+            }
+          ]
+        },
+        "dataSource": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "name": "s3://blob-sink",
+          "uri": "s3://blob-sink"
+        },
+        "additionalProp1": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "additionalProp1": {}
+        },
+        "additionalProp2": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "additionalProp1": {}
+        },
+        "additionalProp3": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.yml#MyCustomJobFacet",
+          "additionalProp1": {}
+        }
+      }
+    }
+  ],
+  "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client"
+}
diff --git a/clients/java/src/main/java/marquez/client/HttpBackend.java b/clients/java/src/main/java/marquez/client/HttpBackend.java
@@ -36,6 +36,11 @@ public URL getBaseUrl() {
 
   private URL url(String path) {
     try {
+      if (baseUrl.toString().endsWith("/") && path.startsWith("/")) {
+        path = path.substring(1);
+      } else if (!baseUrl.toString().endsWith("/") && !path.startsWith("/")) {
+        path = "/" + path;
+      }
       return new URL(this.baseUrl.toString() + path);
     } catch (MalformedURLException e) {
       throw new IllegalArgumentException("Invalid path " + path, e);