Add (apache#115)

CHANGES.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## Unreleased
+* Add `ResolveAvroDependenciesTask` (#115)
 
 ## 0.19.1
 * Fix schema dependency resolution when types are referenced with a `{ "type": NAME }` block rather than just `NAME` (#107)

README.md

@@ -379,7 +379,22 @@ avro {
 }
 ```
 
-# Generating schema files
+# Resolving schema dependencies
+
+If desired, you can generate JSON schema with dependencies resolved.
+
+Example build:
+
+```groovy
+apply plugin: "com.commercehub.gradle.plugin.avro-base"
+
+tasks.register("resolveAvroDependencies", com.commercehub.gradle.plugin.avro.ResolveAvroDependenciesTask) {
+    source file("src/avro/normalized")
+    outputDir = file("build/avro/resolved")
+}
+```
+
+# Generating schema files from protocol/IDL
 
 If desired, you can generate JSON schema files.
 To do this, apply the plugin (either `avro` or `avro-base`), and define custom tasks as needed for the schema generation.

build.gradle

@@ -34,6 +34,7 @@ dependencies {
     implementation "org.apache.avro:avro-compiler:${compileAvroVersion}"
     testImplementation "org.spockframework:spock-core:1.3-groovy-2.5"
     testImplementation gradleTestKit()
+    testImplementation "uk.co.datumedge:hamcrest-json:0.2"
     testRuntimeOnly files(createClasspathManifest) // Add the classpath file to the test runtime classpath
 }
 

src/main/java/com/commercehub/gradle/plugin/avro/ProcessingState.java

@@ -103,4 +103,8 @@ int getProcessedTotal() {
     Iterable<? extends Schema> getSchemasForLocation(String path) {
         return typeStates.values().stream().filter(it -> it.hasLocation(path)).map(TypeState::getSchema).collect(Collectors.toList());
     }
+
+    Iterable<? extends Schema> getSchemas() {
+        return typeStates.values().stream().map(TypeState::getSchema).collect(Collectors.toSet());
+    }
 }

src/main/java/com/commercehub/gradle/plugin/avro/ResolveAvroDependenciesTask.java

@@ -0,0 +1,60 @@
+package com.commercehub.gradle.plugin.avro;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.regex.Pattern;
+import org.apache.avro.Schema;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.specs.NotSpec;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.TaskAction;
+
+import static com.commercehub.gradle.plugin.avro.Constants.SCHEMA_EXTENSION;
+
+/**
+ * Task to read Avro schema files, resolve their dependencies, and write out dependency-free Avro schema files.
+ */
+@CacheableTask
+public class ResolveAvroDependenciesTask extends OutputDirTask {
+    private final SchemaResolver resolver = new SchemaResolver(getProject(), getLogger());
+
+    @TaskAction
+    protected void process() {
+        getLogger().info("Found {} files", getInputs().getSourceFiles().getFiles().size());
+        failOnUnsupportedFiles();
+        processFiles();
+    }
+
+    private void failOnUnsupportedFiles() {
+        FileCollection unsupportedFiles = filterSources(new NotSpec<>(new FileExtensionSpec(SCHEMA_EXTENSION)));
+        if (!unsupportedFiles.isEmpty()) {
+            throw new GradleException(
+                String.format("Unsupported file extension for the following files: %s", unsupportedFiles));
+        }
+    }
+
+    private void processFiles() {
+        int processedFileCount = processSchemaFiles();
+        setDidWork(processedFileCount > 0);
+    }
+
+    private int processSchemaFiles() {
+        Set<File> inputFiles = filterSources(new FileExtensionSpec(SCHEMA_EXTENSION)).getFiles();
+        ProcessingState processingState = resolver.resolve(inputFiles);
+        for (Schema schema : processingState.getSchemas()) {
+            try {
+                String outputPath = schema.getNamespace().replaceAll(Pattern.quote("."), "/")
+                    + "/" + schema.getName() + "." + SCHEMA_EXTENSION;
+                File outputFile = new File(getOutputDir().get().getAsFile(), outputPath);
+                String schemaJson = schema.toString(true);
+                FileUtils.writeJsonFile(outputFile, schemaJson);
+                getLogger().debug("Wrote {}", outputFile.getPath());
+            } catch (IOException ex) {
+                throw new GradleException(String.format("Failed to write resolved schema definition for %s", schema.getFullName()), ex);
+            }
+        }
+        return processingState.getProcessedTotal();
+    }
+}

src/main/java/com/commercehub/gradle/plugin/avro/SchemaResolver.java

@@ -33,7 +33,7 @@ ProcessingState resolve(Iterable<File> files) {
         }
         Set<FileState> failedFiles = processingState.getFailedFiles();
         if (!failedFiles.isEmpty()) {
-            StringBuilder errorMessage = new StringBuilder("Could not compile schema definition files:");
+            StringBuilder errorMessage = new StringBuilder("Could not resolve schema definition files:");
             for (FileState fileState : failedFiles) {
                 String path = fileState.getPath();
                 String fileErrorMessage = fileState.getErrorMessage();
@@ -72,7 +72,7 @@ private void processSchemaFile(ProcessingState processingState, FileState fileSt
                 String typeName = duplicateTypeMatcher.group(1);
                 if (fileState.containsDuplicateTypeName(typeName)) {
                     throw new GradleException(
-                        String.format("Failed to compile schema definition file %s; contains duplicate type definition %s", path, typeName),
+                        String.format("Failed to resolve schema definition file %s; contains duplicate type definition %s", path, typeName),
                         ex);
                 } else {
                     fileState.setError(ex);
@@ -81,10 +81,10 @@ private void processSchemaFile(ProcessingState processingState, FileState fileSt
                     logger.debug("Identified duplicate type {} in {}; will re-process excluding it", typeName, path);
                 }
             } else {
-                throw new GradleException(String.format("Failed to compile schema definition file %s", path), ex);
+                throw new GradleException(String.format("Failed to resolve schema definition file %s", path), ex);
             }
         } catch (IOException ex) {
-            throw new GradleException(String.format("Failed to compile schema definition file %s", path), ex);
+            throw new GradleException(String.format("Failed to resolve schema definition file %s", path), ex);
         }
     }
 }

src/test/groovy/com/commercehub/gradle/plugin/avro/AvroPluginFunctionalSpec.groovy

@@ -127,7 +127,7 @@ class AvroPluginFunctionalSpec extends FunctionalSpec {
 
         then:
         result.task(":generateAvroJava").outcome == FAILED
-        result.output.contains("> Could not compile schema definition files:")
+        result.output.contains("> Could not resolve schema definition files:")
         result.output.contains("* $errorFilePath: \"enum\" is not a defined name. The type of the \"gender\" " +
                 "field must be a defined name or a {\"type\": ...} expression.")
     }

src/test/groovy/com/commercehub/gradle/plugin/avro/DuplicateHandlingFunctionalSpec.groovy

@@ -130,7 +130,7 @@ class DuplicateHandlingFunctionalSpec extends FunctionalSpec {
 
         then:
         result.task(":generateAvroJava").outcome == FAILED
-        result.output.contains("Failed to compile schema definition file $errorFilePath; " +
+        result.output.contains("Failed to resolve schema definition file $errorFilePath; " +
             "contains duplicate type definition example.avro.date")
     }
 

src/test/groovy/com/commercehub/gradle/plugin/avro/ResolveAvroDependenciesTaskFunctionalSpec.groovy

@@ -0,0 +1,39 @@
+package com.commercehub.gradle.plugin.avro
+
+import org.hamcrest.MatcherAssert
+import spock.lang.Subject
+import uk.co.datumedge.hamcrest.json.SameJSONAs
+
+import static org.gradle.testkit.runner.TaskOutcome.SUCCESS
+
+@Subject(ResolveAvroDependenciesTask)
+class ResolveAvroDependenciesTaskFunctionalSpec extends FunctionalSpec {
+    def "resolves dependencies"() {
+        def srcDir = testProjectDir.newFolder("src", "avro", "normalized")
+
+        given: "a build with the task declared"
+        applyAvroBasePlugin()
+        buildFile << """
+        |tasks.register("resolveAvroDependencies", com.commercehub.gradle.plugin.avro.ResolveAvroDependenciesTask) {
+        |    source file("src/avro/normalized")
+        |    outputDir = file("build/avro/resolved")
+        |}
+        |""".stripMargin()
+
+        and: "some normalized schema files"
+        copyResource("/examples/separate/Breed.avsc", srcDir)
+        copyResource("/examples/separate/Cat.avsc", srcDir)
+
+        when: "running the task"
+        def result = run("resolveAvroDependencies")
+
+        then: "the resolved schema files are generated"
+        result.task(":resolveAvroDependencies").outcome == SUCCESS
+        MatcherAssert.assertThat(
+            projectFile("build/avro/resolved/example/Cat.avsc").text,
+            SameJSONAs.sameJSONAs(getClass().getResourceAsStream("/examples/inline/Cat.avsc").text))
+        MatcherAssert.assertThat(
+            projectFile("build/avro/resolved/example/Breed.avsc").text,
+            SameJSONAs.sameJSONAs(getClass().getResourceAsStream("/examples/separate/Breed.avsc").text))
+    }
+}

src/test/groovy/com/commercehub/gradle/plugin/avro/SchemaResolverSpec.groovy

@@ -195,6 +195,6 @@ class SchemaResolverSpec extends Specification {
 
         then:
         def ex = thrown(GradleException)
-        ex.message == "Failed to compile schema definition file ${file.path}; contains duplicate type definition example.avro.date"
+        ex.message == "Failed to resolve schema definition file ${file.path}; contains duplicate type definition example.avro.date"
     }
 }