Skip to content

Commit

Permalink
Build: Add 3rd party dependencies report generation (#27727)
Browse files Browse the repository at this point in the history
* Adds task dependenciesInfo to BuildPlugin to generate a CSV file with dependencies information (name,version,url,license)
* Adds `ConcatFilesTask.groovy` to concatenates multiple files into one
* Adds task `:distribution:generateDependenciesReport` to concatenate `dependencies.csv` files into a single file (`es-dependencies.csv` by default)

 # Examples:
      $ gradle dependenciesInfo :distribution:generateDependenciesReport

 ## Use `csv` system property to customize the output file path
     $ gradle dependenciesInfo :distribution:generateDependenciesReport -Dcsv=/tmp/elasticsearch-dependencies.csv

 ## When branch is not master, use `build.branch` system property to generate correct licenses URLs
     $ gradle dependenciesInfo :distribution:generateDependenciesReport -Dbuild.branch=6.x -Dcsv=/tmp/elasticsearch-dependencies.csv
  • Loading branch information
mgreau authored Dec 26, 2017
1 parent 0b2c8c8 commit 771defb
Show file tree
Hide file tree
Showing 11 changed files with 270 additions and 0 deletions.
1 change: 1 addition & 0 deletions benchmarks/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ forbiddenApis {

// No licenses for our benchmark deps (we don't ship benchmarks)
dependencyLicenses.enabled = false
dependenciesInfo.enabled = false

thirdPartyAudit.excludes = [
// these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees)
Expand Down
1 change: 1 addition & 0 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ if (project != rootProject) {

// build-tools is not ready for primetime with these...
dependencyLicenses.enabled = false
dependenciesInfo.enabled = false
forbiddenApisMain.enabled = false
forbiddenApisTest.enabled = false
jarHell.enabled = false
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ class BuildPlugin implements Plugin<Project> {

configureTest(project)
configurePrecommit(project)
configureDependenciesInfo(project)
}

/** Performs checks on the build environment and prints information about the build environment. */
Expand Down Expand Up @@ -637,4 +638,9 @@ class BuildPlugin implements Plugin<Project> {
project.test.mustRunAfter(precommit)
project.dependencyLicenses.dependencies = project.configurations.runtime - project.configurations.provided
}

private static configureDependenciesInfo(Project project) {
Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
deps.dependencies = project.configurations.compile.allDependencies
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.gradle

import org.gradle.api.DefaultTask
import org.gradle.api.file.FileTree
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction

/**
* Concatenates a list of files into one and removes duplicate lines.
*/
public class ConcatFilesTask extends DefaultTask {

/** List of files to concatenate */
@InputFiles
FileTree files

/** line to add at the top of the target file */
@Input
@Optional
String headerLine

@OutputFile
File target

public ConcatFilesTask() {
description = 'Concat a list of files into one.'
}

@TaskAction
public void concatFiles() {
final StringBuilder output = new StringBuilder()

if (headerLine) {
output.append(headerLine).append('\n')
}

final StringBuilder sb = new StringBuilder()
files.each { file ->
sb.append(file.getText('UTF-8'))
}
// Remove duplicate lines
sb.readLines().toSet().each { value ->
output.append(value).append('\n')
}

target.setText(output.toString(), 'UTF-8')
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.gradle

import org.gradle.api.DefaultTask
import org.gradle.api.artifacts.Dependency
import org.gradle.api.artifacts.DependencySet
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction


/**
* A task to gather information about the dependencies and export them into a csv file.
*
* The following information is gathered:
* <ul>
* <li>name: name that identifies the library (groupId:artifactId)</li>
* <li>version</li>
* <li>URL: link to have more information about the dependency.</li>
* <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
* </ul>
*
*/
public class DependenciesInfoTask extends DefaultTask {

/** Dependencies to gather information from. */
@Input
public DependencySet dependencies

/** Directory to read license files */
@InputDirectory
public File licensesDir = new File(project.projectDir, 'licenses')

@OutputFile
File outputFile = new File(project.buildDir, "reports/dependencies/dependencies.csv")

public DependenciesInfoTask() {
description = 'Create a CSV file with dependencies information.'
}

@TaskAction
public void generateDependenciesInfo() {
final StringBuilder output = new StringBuilder()

for (Dependency dependency : dependencies) {
// Only external dependencies are checked
if (dependency.group != null && dependency.group.contains("elasticsearch") == false) {
final String url = createURL(dependency.group, dependency.name, dependency.version)
final String licenseType = getLicenseType(dependency.group, dependency.name)
output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
}
}
outputFile.setText(output.toString(), 'UTF-8')
}

/**
* Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
* based on dependency coordinates.
*/
protected String createURL(final String group, final String name, final String version){
final String baseURL = 'https://repo1.maven.org/maven2'
return "${baseURL}/${group.replaceAll('\\.' , '/')}/${name}/${version}"
}

/**
* Read the LICENSE file associated with the dependency and determine a license type.
*
* The license type is one of the following values:
* <u>
* <li><em>UNKNOWN</em> if LICENSE file is not present for this dependency.</li>
* <li><em>one SPDX identifier</em> if the LICENSE content matches with an SPDX license.</li>
* <li><em>Custom:URL</em> if it's not an SPDX license,
* URL is the Github URL to the LICENSE file in elasticsearch repository.</li>
* </ul>
*
* @param group dependency group
* @param name dependency name
* @return SPDX identifier, UNKNOWN or a Custom license
*/
protected String getLicenseType(final String group, final String name) {
File license

if (licensesDir.exists()) {
licensesDir.eachFileMatch({ it ==~ /.*-LICENSE.*/ }) { File file ->
String prefix = file.name.split('-LICENSE.*')[0]
if (group.contains(prefix) || name.contains(prefix)) {
license = file.getAbsoluteFile()
}
}
}

if (license) {
final String content = license.readLines("UTF-8").toString()
final String spdx = checkSPDXLicense(content)
if (spdx == null) {
// License has not be identified as SPDX.
// As we have the license file, we create a Custom entry with the URL to this license file.
final gitBranch = System.getProperty('build.branch', 'master')
final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/"
return "Custom:${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}"
}
return spdx
} else {
return "UNKNOWN"
}
}

/**
* Check the license content to identify an SPDX license type.
*
* @param licenseText LICENSE file content.
* @return SPDX identifier or null.
*/
private String checkSPDXLicense(final String licenseText) {
String spdx = null

final String APACHE_2_0 = "Apache.*License.*(v|V)ersion 2.0"
final String BSD_2 = "BSD 2-clause.*License"
final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0"
final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1"
final String ICU = "ICU License - ICU 1.8.1 and later"
final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3"
final String MIT = "MIT License"
final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1"

switch (licenseText) {
case ~/.*${APACHE_2_0}.*/:
spdx = 'Apache-2.0'
break
case ~/.*${MIT}.*/:
spdx = 'MIT'
break
case ~/.*${BSD_2}.*/:
spdx = 'BSD-2-Clause'
break
case ~/.*${LGPL_3}.*/:
spdx = 'LGPL-3.0'
break
case ~/.*${CDDL_1_0}.*/:
spdx = 'CDDL_1_0'
break
case ~/.*${CDDL_1_1}.*/:
spdx = 'CDDL_1_1'
break
case ~/.*${ICU}.*/:
spdx = 'ICU'
break
case ~/.*${MOZILLA_1_1}.*/:
spdx = 'MPL-1.1'
break
default:
break
}
return spdx
}
}
1 change: 1 addition & 0 deletions client/benchmark/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,4 @@ dependencies {

// No licenses for our benchmark deps (we don't ship benchmarks)
dependencyLicenses.enabled = false
dependenciesInfo.enabled = false
1 change: 1 addition & 0 deletions client/test/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ jarHell.enabled=false

// TODO: should we have licenses for our test deps?
dependencyLicenses.enabled = false
dependenciesInfo.enabled = false

namingConventions.enabled = false

Expand Down
12 changes: 12 additions & 0 deletions distribution/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import org.apache.tools.ant.filters.FixCrLfFilter
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.EmptyDirTask
import org.elasticsearch.gradle.ConcatFilesTask
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.NoticeTask
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
Expand All @@ -43,6 +44,17 @@ buildscript {
Collection distributions = project.subprojects.findAll {
it.path.contains(':tools') == false && it.path.contains(':bwc') == false }

/*****************************************************************************
* Third party dependencies report *
*****************************************************************************/

// Concatenates the dependencies CSV files into a single file
task generateDependenciesReport(type: ConcatFilesTask) {
files = fileTree(dir: project.rootDir, include: '**/dependencies.csv' )
headerLine = "name,version,url,license"
target = new File(System.getProperty('csv')?: "${project.buildDir}/dependencies/es-dependencies.csv")
}

/*****************************************************************************
* Notice file *
*****************************************************************************/
Expand Down
1 change: 1 addition & 0 deletions qa/wildfly/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -193,5 +193,6 @@ check.dependsOn(integTest)
test.enabled = false

dependencyLicenses.enabled = false
dependenciesInfo.enabled = false

thirdPartyAudit.enabled = false
1 change: 1 addition & 0 deletions test/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ subprojects {

// TODO: should we have licenses for our test deps?
dependencyLicenses.enabled = false
dependenciesInfo.enabled = false

// TODO: why is the test framework pulled in...
forbiddenApisMain.enabled = false
Expand Down
1 change: 1 addition & 0 deletions test/framework/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ forbiddenApisMain {

// TODO: should we have licenses for our test deps?
dependencyLicenses.enabled = false
dependenciesInfo.enabled = false

thirdPartyAudit.excludes = [
// classes are missing
Expand Down

0 comments on commit 771defb

Please sign in to comment.