Merge pull request #15 from opendatamesh-initiative/12-adding-pipelin…

…e-status-tracking

12 adding pipeline status tracking

Dockerfile

@@ -11,6 +11,13 @@ ARG SPRING_PROPS
 ARG AZURE_ODM_APP_CLIENT_ID=none
 ARG AZURE_ODM_APP_CLIENT_SECRET=none
 ARG AZURE_TENANT_ID=none
+ARG DATABASE_URL=jdbc:postgresql://localhost:5432/odmpdb
+ARG DATABASE_USERNAME=usr
+ARG DATABASE_PASSWORD=pwd
+ARG FLYWAY_SCHEMA=ODMEXECUTOR
+ARG FLYWAY_SCRIPTS_DIR=postgresql
+ARG H2_CONSOLE_ENABLED=false
+ARG H2_CONSOLE_PATH=h2-console
 
 ENV SPRING_PROFILES_ACTIVE ${SPRING_PROFILES_ACTIVE}
 ENV JAVA_OPTS ${JAVA_OPTS}
@@ -19,6 +26,13 @@ ENV SPRING_PROPS ${SPRING_PROPS}
 ENV AZURE_ODM_APP_CLIENT_ID ${AZURE_ODM_APP_CLIENT_ID}
 ENV AZURE_ODM_APP_CLIENT_SECRET ${AZURE_ODM_APP_CLIENT_SECRET}
 ENV AZURE_TENANT_ID ${AZURE_TENANT_ID}
+ENV DATABASE_URL ${DATABASE_URL}
+ENV DATABASE_USERNAME ${DATABASE_USERNAME}
+ENV DATABASE_PASSWORD ${DATABASE_PASSWORD}
+ENV FLYWAY_SCHEMA ${FLYWAY_SCHEMA}
+ENV FLYWAY_SCRIPTS_DIR ${FLYWAY_SCRIPTS_DIR}
+ENV H2_CONSOLE_ENABLED ${H2_CONSOLE_ENABLED}
+ENV H2_CONSOLE_PATH ${H2_CONSOLE_PATH}
 
 EXPOSE $SPRING_PORT
 

README.md

@@ -129,13 +129,62 @@ You need to first execute the build locally by running the following command:
 mvn clean package spring-boot:repackage -DskipTests
 ```
 
+### Run database
+The image generated from Dockerfile contains only the application. It requires a database to run properly. The supported databases are MySql and Postgres. If you do not already have a database available, you can create one by running the following commands:
+
+**MySql**
+```bash
+docker run --name odm-executor-azuredevops-mysql-db -d -p 3306:3306  \
+   -e MYSQL_DATABASE=ODMEXECUTOR \
+   -e MYSQL_ROOT_PASSWORD=root \
+   mysql:8
+```
+
+**Postgres**
+```bash
+docker run --name odm-executor-azuredevops-postgres-db -d -p 5432:5432  \
+   -e POSTGRES_DB=odmpdb \
+   -e POSTGRES_USER=postgres \
+   -e POSTGRES_PASSWORD=postgres \
+   postgres:11-alpine
+```
+
+Check that the database has started correctly:
+
+**MySql**
+```bash
+docker logs odm-executor-azuredevops-mysql-db
+```
+
+*Postgres*
+```bash
+docker logs odm-executor-azuredevops-postgres-db
+```
+
 ### Build image
 Build the Docker image of the application and run it.
 
 *Before executing the following commands change properly the value of arguments.
 
+**MySql**
 ```bash
-docker build -t odm-executor-azuredevops-app . -f Dockerfile \
+docker build -t odm-executor-azuredevops-mysql-app . -f Dockerfile \
+   --build-arg DATABASE_URL=jdbc:mysql://localhost:3306/ODMEXECUTOR \
+   --build-arg DATABASE_USERNAME=root \
+   --build-arg DATABASE_PASSWORD=root \
+   --build-arg FLYWAY_SCRIPTS_DIR=mysql \
+   --build-arg AZURE_ODM_APP_CLIENT_ID=<azure-odm-app-client-id> \
+   --build-arg AZURE_ODM_APP_CLIENT_SECRET=<azure-odm-app-client-secret> \
+   --build-arg AZURE_TENANT_ID=<azure-tenant-id-value> 
+```
+
+**Postgres**
+```bash
+docker build -t odm-executor-azuredevops-postgres-app . -f Dockerfile \
+   --build-arg DATABASE_URL=jdbc:postgresql://localhost:5432/odmpdb \
+   --build-arg DATABASE_USERNAME=postgres \
+   --build-arg DATABASE_PASSWORD=postgres \
+   --build-arg FLYWAY_SCRIPTS_DIR=postgresql \
    --build-arg AZURE_ODM_APP_CLIENT_ID=<azure-odm-app-client-id> \
    --build-arg AZURE_ODM_APP_CLIENT_SECRET=<azure-odm-app-client-secret> \
    --build-arg AZURE_TENANT_ID=<azure-tenant-id-value> 
@@ -144,25 +193,37 @@ docker build -t odm-executor-azuredevops-app . -f Dockerfile \
 ### Run application
 Run the Docker image.
 
+*Note: Before executing the following commands remove the argument `--net host` if the database is not running on `localhost`*
+
+**MySql**
+```bash
+docker run --name odm-executor-azuredevops-mysql-app -p 9003:9003 --net host odm-executor-azuredevops-mysql-app
+```
+
+**Postgres**
 ```bash
-docker run --name odm-executor-azuredevops-app -p 9003:9003 odm-executor-azuredevops-app
+docker run --name odm-executor-azuredevops-postgres-app -p 9003:9003 --net host odm-executor-azuredevops-postgres-app
 ```
 
 ### Stop application
 
+*Before executing the following commands:
+* change the DB name to `odm-executor-azuredevops-postgres-db` if you are using postgres and not mysql
+* change the instance name to `odm-executor-azuredevops-postgres-app` if you are using postgres and not mysql
+
 ```bash
-docker stop odm-executor-azuredevops-app
+docker stop odm-executor-azuredevops-mysql-app
 ```
 To restart a stopped application execute the following commands:
 
 ```bash
-docker start odm-executor-azuredevops-app
+docker start odm-executor-azuredevops-mysql-app
 ```
 
 To remove a stopped application to rebuild it from scratch execute the following commands :
 
 ```bash
-docker rm odm-executor-azuredevops-app
+docker rm odm-executor-azuredevops-mysql-app
 ```
 
 ## Run with Docker Compose
@@ -172,8 +233,8 @@ docker rm odm-executor-azuredevops-app
 Clone the repository and move it to the project root folder
 
 ```bash
-git git clone https://github.com/opendatamesh-initiative/odm-platform-up-services-policy-opa.git
-cd odm-platform-up-services-policy-opa
+git git clone https://github.com/opendatamesh-initiative/odm-platform-up-services-executor-azuredevops.git
+cd odm-platform-up-services-executor-azuredevops
 ```
 
 ### Compile project
@@ -192,6 +253,10 @@ SPRING_PORT=9003
 AZURE_ODM_APP_CLIENT_ID=<azure-odm-app-client>
 AZURE_ODM_APP_CLIENT_SECRET=<azure-odm-app-client-secret>
 AZURE_TENANT_ID=<azure-tenant-id-value>
+DATABASE_NAME=odmpdb
+DATABASE_PASSWORD=pwd
+DATABASE_USERNAME=usr
+DATABASE_PORT=5432
 ```
 
 Then, build the docker-compose file:
@@ -226,4 +291,12 @@ docker-compose build --no-cache
 ## REST services
 You can invoke REST endpoints through *OpenAPI UI* available at the following url:
 
-* [http://localhost:9003//api/v1/up/executor/azure-devops/swagger-ui/index.html](http://localhost:9003/api/v1/up/executor/azure-devops/swagger-ui/index.html)
+* [http://localhost:9003//api/v1/up/executor/swagger-ui/index.html](http://localhost:9003/api/v1/up/executor/swagger-ui/index.html)
+
+## Database
+
+If the application is running using an in memory instance of H2 database you can check the database content through H2 Web Console available at the following url:
+
+* [http://localhost:9003/api/v1/up/executor/h2-console](http://localhost:9003/api/v1/up/executor/h2-console)
+
+In all cases you can also use your favourite sql client providing the proper connection parameters

azuredevops-api/src/main/java/org/opendatamesh/platform/up/executor/azuredevops/api/clients/AzureDevOpsAPIRoutes.java

@@ -0,0 +1,27 @@
+package org.opendatamesh.platform.up.executor.azuredevops.api.clients;
+
+import org.opendatamesh.platform.core.commons.clients.ODMApiRoutes;
+
+public enum AzureDevOpsAPIRoutes implements ODMApiRoutes {
+
+    AZURE_DEVOPS_PIPELINE("/{organization}/{project}/_apis/pipelines/{pipelineId}/runs?api-version=7.0"),
+
+    AZURE_DEVOPS_RUN("/{organization}/{project}/_apis/pipelines/{pipelineId}/runs/{runId}?api-version=7.0");
+
+    private final String path;
+
+    AzureDevOpsAPIRoutes(String path) {
+        this.path = path;
+    }
+
+    @Override
+    public String toString() {
+        return this.path;
+    }
+
+    @Override
+    public String getPath() {
+        return path;
+    }
+
+}

azuredevops-api/src/main/java/org/opendatamesh/platform/up/executor/azuredevops/api/clients/AzureDevOpsClient.java

@@ -3,12 +3,10 @@
 import org.opendatamesh.platform.core.commons.clients.ODMClient;
 import org.opendatamesh.platform.core.commons.oauth.OAuthTokenManager;
 import org.opendatamesh.platform.core.dpds.ObjectMapperFactory;
+import org.opendatamesh.platform.up.executor.azuredevops.api.resources.AzureRunResource;
 import org.opendatamesh.platform.up.executor.azuredevops.api.resources.PipelineResource;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import org.springframework.http.*;
 import org.springframework.stereotype.Service;
 
 @Service
@@ -36,27 +34,50 @@ public AzureDevOpsClient(
         );
     }
 
-    private String buildRunPipelineUri(String organization, String project, String pipelineId) {
+    public ResponseEntity<AzureRunResource> runPipeline(
+            PipelineResource pipelineResource, String organization, String project, String pipelineId
+    ){
+
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_JSON);
+        headers.setBearerAuth(oAuthTokenManager.getToken());
+
+        HttpEntity<PipelineResource> entity = new HttpEntity<>(pipelineResource, headers);
 
-        String pipelineUri = "/%s/%s/_apis/pipelines/%s/runs?api-version=7.0";
-        return String.format(pipelineUri, organization, project, pipelineId);
+        ResponseEntity<AzureRunResource> response = rest.postForEntity(
+                apiUrl(AzureDevOpsAPIRoutes.AZURE_DEVOPS_PIPELINE),
+                entity,
+                AzureRunResource.class,
+                organization,
+                project,
+                pipelineId
+        );
+
+        return response;
 
     }
 
-    public ResponseEntity<String> runPipeline(
-            PipelineResource pipelineResource, String organization, String project, String pipelineId
-    ){
+    public ResponseEntity<AzureRunResource> getAzureRun(String organization, String project, String pipelineId, Long runId){
 
         HttpHeaders headers = new HttpHeaders();
         headers.setContentType(MediaType.APPLICATION_JSON);
         headers.setBearerAuth(oAuthTokenManager.getToken());
 
-        String pipelineUri = buildRunPipelineUri(organization, project, pipelineId);
-        HttpEntity<PipelineResource> entity = new HttpEntity<>(pipelineResource, headers);
+        HttpEntity<PipelineResource> entity = new HttpEntity<>(headers);
 
-        ResponseEntity<String> response = rest.postForEntity(apiUrlFromString(pipelineUri), entity, String.class);
+        ResponseEntity<AzureRunResource> response = rest.exchange(
+                apiUrl(AzureDevOpsAPIRoutes.AZURE_DEVOPS_RUN),
+                HttpMethod.GET,
+                entity,
+                AzureRunResource.class,
+                organization,
+                project,
+                pipelineId,
+                runId
+        );
 
         return response;
 
     }
+
 }

azuredevops-api/src/main/java/org/opendatamesh/platform/up/executor/azuredevops/api/resources/AzureRunResource.java

@@ -0,0 +1,37 @@
+package org.opendatamesh.platform.up.executor.azuredevops.api.resources;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import java.util.Map;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AzureRunResource {
+
+    @JsonProperty("id")
+    private Long runId;
+
+    @JsonProperty("name")
+    private String name;
+
+    @JsonProperty("state")
+    private AzureRunState state;
+
+    @JsonProperty("result")
+    private AzureRunResult result;
+
+    @JsonProperty("variables")
+    private Map<String, AzureVariable> variables;
+
+    @JsonProperty("templateParameters")
+    private Map<String, String> templateParameters;
+
+    @JsonProperty("createdDate")
+    private String createdDate;
+
+    @JsonProperty("finishedDate")
+    private String finishedDate;
+
+}

azuredevops-api/src/main/java/org/opendatamesh/platform/up/executor/azuredevops/api/resources/AzureRunResult.java

@@ -0,0 +1,13 @@
+package org.opendatamesh.platform.up.executor.azuredevops.api.resources;
+
+public enum AzureRunResult {
+
+    canceled,
+
+    failed,
+
+    succeeded,
+
+    unknown
+
+}

azuredevops-api/src/main/java/org/opendatamesh/platform/up/executor/azuredevops/api/resources/AzureRunState.java

@@ -0,0 +1,8 @@
+package org.opendatamesh.platform.up.executor.azuredevops.api.resources;
+
+public enum AzureRunState {
+    canceling,
+    completed,
+    inProgress,
+    unknown
+}

azuredevops-api/src/main/java/org/opendatamesh/platform/up/executor/azuredevops/api/resources/AzureVariable.java

@@ -0,0 +1,17 @@
+package org.opendatamesh.platform.up.executor.azuredevops.api.resources;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AzureVariable {
+
+    @JsonProperty("isSecret")
+    private Boolean isSecret;
+
+    @JsonProperty("value")
+    private String value;
+
+}

azuredevops-api/src/main/java/org/opendatamesh/platform/up/executor/azuredevops/api/resources/PipelineResource.java

@@ -10,10 +10,10 @@
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class PipelineResource {
 
-    ResourcesResource resources;
+    private ResourcesResource resources;
 
     private Map<String, String> templateParameters;
 
-    List<String> stagesToSkip;
+    private List<String> stagesToSkip;
 
 }