Merge branch 'linkedin:master' into feature/hive-database-identifier

datahub-project · Jun 8, 2021 · 1b252db · 1b252db
2 parents a7c988b + c0ace2c
commit 1b252db
Show file tree

Hide file tree

Showing 592 changed files with 42,829 additions and 38,009 deletions.
diff --git a/.github/workflows/docker-datahub-upgrade.yml b/.github/workflows/docker-datahub-upgrade.yml
@@ -0,0 +1,74 @@
+name: docker-datahub-upgrade docker
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+  pull_request:
+    branches:
+      - master
+    paths:
+      - './datahub-upgrade/**'
+      - '.github/workflows/docker-datahub-upgrade.yml'
+    paths_ignore:
+      - '**.md'
+      - '**.env'
+  release:
+    types: [published, edited]
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+      publish: ${{ steps.publish.outputs.publish }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Compute Tag
+        id: tag
+        run: |
+          echo "GITHUB_REF: $GITHUB_REF"
+          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
+          echo "SHORT_SHA: $SHORT_SHA"
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/.*$,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          echo "tag=$TAG"
+          echo "::set-output name=tag::$TAG"
+      - name: Check whether publishing enabled
+        id: publish
+        env:
+          ENABLE_PUBLISH: ${{ secrets.ORG_DOCKER_PASSWORD }}
+        run: |
+          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
+          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+  push_to_registries:
+    name: Build and Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Docker meta
+        id: docker_meta
+        uses: crazy-max/ghaction-docker-meta@v1
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            acryldata/datahub-upgrade
+          # add git short SHA as Docker tag
+          tag-custom: ${{ needs.setup.outputs.tag }}
+          tag-custom-only: true
+      - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.ORG_DOCKER_PASSWORD }}
+      - name: Build and Push image
+        uses: docker/build-push-action@v2
+        with:
+          file: ./docker/datahub-upgrade/Dockerfile
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-elasticsearch-setup.yml b/.github/workflows/docker-elasticsearch-setup.yml
@@ -3,10 +3,9 @@ on:
   push:
     branches:
       - master
-    paths:
-      - 'docker/elasticsearch-setup/**'
-      - '.github/workflows/docker-elasticsearch-setup.yml'
-      - 'gms/impl/src/main/resources/index/**'
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
   pull_request:
     branches:
       - master
@@ -31,7 +30,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -44,7 +43,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -60,6 +58,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
@@ -69,4 +68,4 @@ jobs:
         with:
           file: ./docker/elasticsearch-setup/Dockerfile
           tags: ${{ steps.docker_meta.outputs.tags }}
-          push: ${{ needs.setup.outputs.publish == 'true' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-frontend-react.yml b/.github/workflows/docker-frontend-react.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -45,7 +45,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -61,6 +60,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/.github/workflows/docker-gms.yml b/.github/workflows/docker-gms.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -45,7 +45,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -61,6 +60,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
@@ -70,4 +70,4 @@ jobs:
         with:
           file: ./docker/datahub-gms/Dockerfile
           tags: ${{ steps.docker_meta.outputs.tags }}
-          push: ${{ needs.setup.outputs.publish == 'true' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -45,7 +45,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -61,6 +60,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/.github/workflows/docker-kafka-setup.yml b/.github/workflows/docker-kafka-setup.yml
@@ -3,9 +3,9 @@ on:
   push:
     branches:
       - master
-    paths:
-      - 'docker/kafka-setup/**'
-      - '.github/workflows/docker-kafka-setup.yml'
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
   pull_request:
     branches:
       - master
@@ -29,7 +29,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -42,7 +42,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -58,6 +57,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
@@ -68,4 +68,4 @@ jobs:
           file: ./docker/kafka-setup/Dockerfile
           context: ./docker/kafka-setup
           tags: ${{ steps.docker_meta.outputs.tags }}
-          push: ${{ needs.setup.outputs.publish == 'true' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
diff --git a/.github/workflows/docker-mae-consumer.yml b/.github/workflows/docker-mae-consumer.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -45,7 +45,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -61,6 +60,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/.github/workflows/docker-mce-consumer.yml b/.github/workflows/docker-mce-consumer.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
-          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           echo "tag=$TAG"
           echo "::set-output name=tag::$TAG"
       - name: Check whether publishing enabled
@@ -45,7 +45,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -61,6 +60,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,9 @@
 *.iws
 *.ipr
 
+*gma-create-all.sql
+*gma-drop-all.sql
+
 # Pegasus & Avro
 **/src/mainGenerated*
 **/src/testGenerated*

diff --git a/README.md b/README.md
@@ -6,14 +6,16 @@ import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
 export const Logo = (props) => {
   const siteConfig = useDocusaurusContext().siteConfig;
   return (
-    <ThemedImage
-      alt="DataHub Logo"
-      sources={{
-          light: useBaseUrl(siteConfig.themeConfig.navbar.logo.src),
-          dark: useBaseUrl(siteConfig.themeConfig.navbar.logo.srcDark),
-      }}
-      {...props}
-    />
+    <div style={{display: "flex"}}>
+      <ThemedImage
+        alt="DataHub Logo"
+        sources={{
+            light: useBaseUrl(siteConfig.themeConfig.navbar.logo.src),
+            dark: useBaseUrl(siteConfig.themeConfig.navbar.logo.srcDark),
+        }}
+        {...props}
+      />
+    </div>
   )
 };
 
@@ -142,6 +144,7 @@ Here is a list of companies that are currently building POC or seriously evaluat
 - [Microsoft](https://microsoft.com)
 - [Morgan Stanley](https://www.morganstanley.com)
 - [Orange Telecom](https://www.orange.com)
+- [Plum Research](https://plumresearch.com)
 - [REEF Technology](https://reeftechnology.com)
 - [SpotHero](https://spothero.com)
 - [Sysco AS](https://sysco.no)

diff --git a/build.gradle b/build.gradle
@@ -9,7 +9,7 @@ buildscript {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
     classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
-    classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.1.2.RELEASE'
+    classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.1.4.RELEASE'
   }
 }
 
@@ -38,6 +38,7 @@ project.ext.externalDependency = [
     'cacheApi' : 'javax.cache:cache-api:1.1.0',
     'commonsIo': 'commons-io:commons-io:2.4',
     'commonsLang': 'commons-lang:commons-lang:2.6',
+    'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
     'ebean': 'io.ebean:ebean:11.33.3',
     'ebeanAgent': 'io.ebean:ebean-agent:11.27.1',
     'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.3',
@@ -62,6 +63,7 @@ project.ext.externalDependency = [
     'h2': 'com.h2database:h2:1.4.196',
     'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final',
     'httpClient': 'org.apache.httpcomponents:httpclient:4.5.9',
+    'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
     'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.9.7',
     'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.9.7',
     'javatuples': 'org.javatuples:javatuples:1.2',
@@ -79,6 +81,7 @@ project.ext.externalDependency = [
     'lombok': 'org.projectlombok:lombok:1.18.12',
     'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
     'mockito': 'org.mockito:mockito-core:3.0.0',
+    'mockitoInline': 'org.mockito:mockito-inline:3.0.0',
     'mysqlConnector': 'mysql:mysql-connector-java:5.1.47',
     'neo4jHarness': 'org.neo4j.test:neo4j-harness:3.4.11',
     'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:4.0.1',
@@ -101,12 +104,15 @@ project.ext.externalDependency = [
     'springCore': 'org.springframework:spring-core:5.2.3.RELEASE',
     'springJdbc': 'org.springframework:spring-jdbc:5.2.3.RELEASE',
     'springWeb': 'org.springframework:spring-web:5.2.3.RELEASE',
-    'springBootAutoconfigure': 'org.springframework.boot:spring-boot-autoconfigure:2.1.2.RELEASE',
-    'springBootStarterWeb': 'org.springframework.boot:spring-boot-starter-web:2.1.2.RELEASE',
-    'springBootStarterJetty': 'org.springframework.boot:spring-boot-starter-jetty:2.1.2.RELEASE',
+    'springBootAutoconfigure': 'org.springframework.boot:spring-boot-autoconfigure:2.1.4.RELEASE',
+    'springBootStarterWeb': 'org.springframework.boot:spring-boot-starter-web:2.1.4.RELEASE',
+    'springBootStarterJetty': 'org.springframework.boot:spring-boot-starter-jetty:2.1.4.RELEASE',
     'springKafka': 'org.springframework.kafka:spring-kafka:2.2.14.RELEASE',
     'springActuator': 'org.springframework.boot:spring-boot-starter-actuator:2.1.2.RELEASE',
-    'testng': 'org.testng:testng:6.9.9',
+    'testng': 'org.testng:testng:7.3.0',
+    'testContainers': 'org.testcontainers:testcontainers:1.15.1',
+    'testContainersJunit': 'org.testcontainers:junit-jupiter:1.15.1',
+    'testContainersElasticsearch': 'org.testcontainers:elasticsearch:1.15.3',
     'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14'
 ]