diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 44687f154fff4..312869e75b7cb 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -68,6 +68,7 @@ forbiddenApis { // No licenses for our benchmark deps (we don't ship benchmarks) dependencyLicenses.enabled = false +dependenciesInfo.enabled = false thirdPartyAudit.excludes = [ // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 7c6f3002f708f..b021bab6338b3 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -140,6 +140,7 @@ if (project != rootProject) { // build-tools is not ready for primetime with these... dependencyLicenses.enabled = false + dependenciesInfo.enabled = false forbiddenApisMain.enabled = false forbiddenApisTest.enabled = false jarHell.enabled = false diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 5a55549ca5b8f..c3297aa286599 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -85,6 +85,7 @@ class BuildPlugin implements Plugin { configureTest(project) configurePrecommit(project) + configureDependenciesInfo(project) } /** Performs checks on the build environment and prints information about the build environment. */ @@ -637,4 +638,9 @@ class BuildPlugin implements Plugin { project.test.mustRunAfter(precommit) project.dependencyLicenses.dependencies = project.configurations.runtime - project.configurations.provided } + + private static configureDependenciesInfo(Project project) { + Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class) + deps.dependencies = project.configurations.compile.allDependencies + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/ConcatFilesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/ConcatFilesTask.groovy new file mode 100644 index 0000000000000..7e561b18dcb8a --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/ConcatFilesTask.groovy @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle + +import org.gradle.api.DefaultTask +import org.gradle.api.file.FileTree +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction + +/** + * Concatenates a list of files into one and removes duplicate lines. + */ +public class ConcatFilesTask extends DefaultTask { + + /** List of files to concatenate */ + @InputFiles + FileTree files + + /** line to add at the top of the target file */ + @Input + @Optional + String headerLine + + @OutputFile + File target + + public ConcatFilesTask() { + description = 'Concat a list of files into one.' + } + + @TaskAction + public void concatFiles() { + final StringBuilder output = new StringBuilder() + + if (headerLine) { + output.append(headerLine).append('\n') + } + + final StringBuilder sb = new StringBuilder() + files.each { file -> + sb.append(file.getText('UTF-8')) + } + // Remove duplicate lines + sb.readLines().toSet().each { value -> + output.append(value).append('\n') + } + + target.setText(output.toString(), 'UTF-8') + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy new file mode 100644 index 0000000000000..ddd5248396ce3 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy @@ -0,0 +1,175 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle + +import org.gradle.api.DefaultTask +import org.gradle.api.artifacts.Dependency +import org.gradle.api.artifacts.DependencySet +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction + + +/** + * A task to gather information about the dependencies and export them into a csv file. + * + * The following information is gathered: + * + * + */ +public class DependenciesInfoTask extends DefaultTask { + + /** Dependencies to gather information from. */ + @Input + public DependencySet dependencies + + /** Directory to read license files */ + @InputDirectory + public File licensesDir = new File(project.projectDir, 'licenses') + + @OutputFile + File outputFile = new File(project.buildDir, "reports/dependencies/dependencies.csv") + + public DependenciesInfoTask() { + description = 'Create a CSV file with dependencies information.' + } + + @TaskAction + public void generateDependenciesInfo() { + final StringBuilder output = new StringBuilder() + + for (Dependency dependency : dependencies) { + // Only external dependencies are checked + if (dependency.group != null && dependency.group.contains("elasticsearch") == false) { + final String url = createURL(dependency.group, dependency.name, dependency.version) + final String licenseType = getLicenseType(dependency.group, dependency.name) + output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n") + } + } + outputFile.setText(output.toString(), 'UTF-8') + } + + /** + * Create an URL on Maven Central + * based on dependency coordinates. + */ + protected String createURL(final String group, final String name, final String version){ + final String baseURL = 'https://repo1.maven.org/maven2' + return "${baseURL}/${group.replaceAll('\\.' , '/')}/${name}/${version}" + } + + /** + * Read the LICENSE file associated with the dependency and determine a license type. + * + * The license type is one of the following values: + * + *
  • UNKNOWN if LICENSE file is not present for this dependency.
    • + *
  • one SPDX identifier if the LICENSE content matches with an SPDX license.
    • + *
  • Custom:URL if it's not an SPDX license, + * URL is the Github URL to the LICENSE file in elasticsearch repository.
    • + * + * + * @param group dependency group + * @param name dependency name + * @return SPDX identifier, UNKNOWN or a Custom license + */ + protected String getLicenseType(final String group, final String name) { + File license + + if (licensesDir.exists()) { + licensesDir.eachFileMatch({ it ==~ /.*-LICENSE.*/ }) { File file -> + String prefix = file.name.split('-LICENSE.*')[0] + if (group.contains(prefix) || name.contains(prefix)) { + license = file.getAbsoluteFile() + } + } + } + + if (license) { + final String content = license.readLines("UTF-8").toString() + final String spdx = checkSPDXLicense(content) + if (spdx == null) { + // License has not be identified as SPDX. + // As we have the license file, we create a Custom entry with the URL to this license file. + final gitBranch = System.getProperty('build.branch', 'master') + final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/" + return "Custom:${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}" + } + return spdx + } else { + return "UNKNOWN" + } + } + + /** + * Check the license content to identify an SPDX license type. + * + * @param licenseText LICENSE file content. + * @return SPDX identifier or null. + */ + private String checkSPDXLicense(final String licenseText) { + String spdx = null + + final String APACHE_2_0 = "Apache.*License.*(v|V)ersion 2.0" + final String BSD_2 = "BSD 2-clause.*License" + final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0" + final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1" + final String ICU = "ICU License - ICU 1.8.1 and later" + final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3" + final String MIT = "MIT License" + final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1" + + switch (licenseText) { + case ~/.*${APACHE_2_0}.*/: + spdx = 'Apache-2.0' + break + case ~/.*${MIT}.*/: + spdx = 'MIT' + break + case ~/.*${BSD_2}.*/: + spdx = 'BSD-2-Clause' + break + case ~/.*${LGPL_3}.*/: + spdx = 'LGPL-3.0' + break + case ~/.*${CDDL_1_0}.*/: + spdx = 'CDDL_1_0' + break + case ~/.*${CDDL_1_1}.*/: + spdx = 'CDDL_1_1' + break + case ~/.*${ICU}.*/: + spdx = 'ICU' + break + case ~/.*${MOZILLA_1_1}.*/: + spdx = 'MPL-1.1' + break + default: + break + } + return spdx + } +} diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle index f488c75b1dfe5..52559b553351e 100644 --- a/client/benchmark/build.gradle +++ b/client/benchmark/build.gradle @@ -67,3 +67,4 @@ dependencies { // No licenses for our benchmark deps (we don't ship benchmarks) dependencyLicenses.enabled = false +dependenciesInfo.enabled = false diff --git a/client/test/build.gradle b/client/test/build.gradle index e57d415e9eaab..a05443a702758 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -53,6 +53,7 @@ jarHell.enabled=false // TODO: should we have licenses for our test deps? dependencyLicenses.enabled = false +dependenciesInfo.enabled = false namingConventions.enabled = false diff --git a/distribution/build.gradle b/distribution/build.gradle index 26221cad5c7f6..aefbeba4b58d7 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -22,6 +22,7 @@ import org.apache.tools.ant.filters.FixCrLfFilter import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.EmptyDirTask +import org.elasticsearch.gradle.ConcatFilesTask import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.NoticeTask import org.elasticsearch.gradle.precommit.DependencyLicensesTask @@ -43,6 +44,17 @@ buildscript { Collection distributions = project.subprojects.findAll { it.path.contains(':tools') == false && it.path.contains(':bwc') == false } +/***************************************************************************** + * Third party dependencies report * + *****************************************************************************/ + +// Concatenates the dependencies CSV files into a single file +task generateDependenciesReport(type: ConcatFilesTask) { + files = fileTree(dir: project.rootDir, include: '**/dependencies.csv' ) + headerLine = "name,version,url,license" + target = new File(System.getProperty('csv')?: "${project.buildDir}/dependencies/es-dependencies.csv") +} + /***************************************************************************** * Notice file * *****************************************************************************/ diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle index 06a2a6ca45275..e1f6603360c8b 100644 --- a/qa/wildfly/build.gradle +++ b/qa/wildfly/build.gradle @@ -193,5 +193,6 @@ check.dependsOn(integTest) test.enabled = false dependencyLicenses.enabled = false +dependenciesInfo.enabled = false thirdPartyAudit.enabled = false diff --git a/test/build.gradle b/test/build.gradle index 594fa5bbb7059..2055896bda4e5 100644 --- a/test/build.gradle +++ b/test/build.gradle @@ -38,6 +38,7 @@ subprojects { // TODO: should we have licenses for our test deps? dependencyLicenses.enabled = false + dependenciesInfo.enabled = false // TODO: why is the test framework pulled in... forbiddenApisMain.enabled = false diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 215f6f2d394eb..4e6ef482210b9 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -46,6 +46,7 @@ forbiddenApisMain { // TODO: should we have licenses for our test deps? dependencyLicenses.enabled = false +dependenciesInfo.enabled = false thirdPartyAudit.excludes = [ // classes are missing