Merge pull request #524 from IBM/develop

Release zAppBuild 3.9.0

README.md

@@ -30,6 +30,7 @@ The zAppBuild sample provides the following *language* build scripts by default:
 * ZunitConfig.groovy
 * CRB.groovy
 * Transfer.groovy (for transport non-buildable files like JCL or PROC into build libraries and register them as build output)
+* Easytrieve.groovy
 
 All language scripts both compile and optionally link-edit programs. The language build scripts are intended to be useful out of the box but depending on the complexity of your applications' build requirements, may require modifications to meet your development team's needs.  By following the examples used in the existing language build scripts of keeping all application specific references out of the build scripts and instead using configuration properties with strong default values, the zAppBuild sample can continue to be a generic build solution for all of your specific applications.
 

build-conf/Cobol.properties

@@ -62,16 +62,6 @@ cobol_compileErrorFeedbackXmlOptions=tracks space(200,40) dsorg(PS) blksize(2799
 # List of output datasets to document deletions
 cobol_outputDatasets=${cobol_loadPDS}
 
-#
-# Set filter used to exclude certain information from the link edit scanning.
-# The value contains a comma separated list of patterns.
-# example: A filter of *.SUB1, *.SUB2 will exclude modules SUB1 and SUB2
-#          from any dataset. To exclude member HELLO in PDS TEST.COBOL will
-#          be matched by the pattern TEST.COBOL.HELLO. The pattern TEST.COBOL.*
-#          will match any member in the data set TEST.COBOL.
-# The following filter excludes CICS and LE Library references.
-dbb.LinkEditScanner.excludeFilter = ${SDFHLOAD}.*, ${SCEELKED}.*
-
 #
 # additional libraries for compile SYSLIB concatenation, comma-separated, see definitions in application-conf
 # cobol_compileSyslibConcatenation=

build-conf/Easytrieve.properties

@@ -0,0 +1,45 @@
+# Releng properties used by language/Easytrieve.groovy
+
+#
+# Comma separated list of required build properties for easytrieve.groovy
+easytrieve_requiredBuildProperties=easytrieve_srcPDS,easytrieve_cpyPDS,easytrieve_objPDS,easytrieve_loadPDS,\
+easytrieve_compiler,easytrieve_linkEditor,easytrieve_tempOptions,applicationOutputsCollectionName,\
+SCEELKED
+
+#
+# easytrieve compiler name
+# Easytrieve EZTCOM
+# IBM Migration Utility: FSYTPA00
+easytrieve_compiler=FSYTPA00
+
+#
+# linker name
+easytrieve_linkEditor=IEWBLINK
+
+#
+# easytrieve source data sets
+easytrieve_srcPDS=${hlq}.EZT
+easytrieve_cpyPDS=${hlq}.MAC
+easytrieve_objPDS=${hlq}.OBJ
+easytrieve_dbrmPDS=${hlq}.DBRM
+
+#
+# easytrieve load data sets
+easytrieve_loadPDS=${hlq}.LOAD
+
+#
+# List the data sets that need to be created and their creation options
+# https://www.ibm.com/docs/en/mufz/5.1?topic=jcl-jcmucl2jtwo-step-translate-link-without-proc
+easytrieve_srcDatasets=${easytrieve_srcPDS},${easytrieve_cpyPDS},${easytrieve_objPDS},${easytrieve_dbrmPDS}
+easytrieve_srcOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(library)
+
+easytrieve_loadDatasets=${easytrieve_loadPDS}
+easytrieve_loadOptions=cyl space(1,1) dsorg(PO) recfm(U) blksize(32760) dsntype(library)
+
+easytrieve_tempOptions=cyl space(5,5) unit(vio) blksize(80) lrecl(80) recfm(f,b) new
+easytrieve_printTempOptions=cyl space(5,5) unit(vio) blksize(133) lrecl(133) recfm(f,b) new
+
+#
+# List of output datasets to document deletions
+easytrievel_outputDatasets=${easytrieve_loadPDS}
+

build-conf/README.md

@@ -31,6 +31,7 @@ SDSNLOAD | DB2 Load Library. Example: DB2.V9R1M0.SDSNLOAD
 SDSNEXIT | DB2 Exit Library. Example: DBC0CFG.SDSNEXIT
 SFELLOAD | Optional IDz Load Library. Example: FEL.V14R0M0.SFELLOAD
 SBZUSAMP | Optional z/OS Dynamic Test Runner IDz zUnit / WAZI VTP library containing necessary copybooks. Example : FEL.V14R2.SBZUSAMP
+EZETLOAD | Easytrieve Load Library. Example for IBM Migration Utility: SYS1.SFSYLOAD
 
 ### build.properties
 General properties used mainly by `build.groovy` but can also be a place to declare properties used by multiple language scripts.
@@ -336,6 +337,26 @@ transfer_xmlPDS | Sample dataset for xml members
 transfer_dsOptions | BPXWDYN creation options for creating 'source' type data sets
 transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets
 
+### Easytrieve.properties
+Build properties used by zAppBuild/language/Easytrieve.groovy
+
+Property | Description
+--- | ---
+easytrieve_requiredBuildProperties | Comma separated list of required build properties for language/Cobol.groovy
+easytrieve_srcPDS | Dataset to move Easytrieve source files to from USS
+easytrieve_cpyPDS | Dataset to move Easytrieve macros to from USS
+easytrieve_objPDS | Dataset to create object decks in from compile step
+easytrieve_dbrmPDS | Dataset to create DB2 DBRM modules in from compile step
+easytrieve_loadPDS | Dataset to create load modules in from link edit step
+easytrieve_srcDataSets | Comma separated list of 'source' type data sets
+easytrieve_srcOptions | BPXWDYN creation options for creating 'source' type data sets
+easytrieve_loadDatasets | Comma separated list of 'load module' type data sets
+easytrieve_loadOptions | BPXWDYN creation options for 'load module' type data sets
+easytrieve_tempOptions | BPXWDYN creation options for temporary data sets
+easytrieve_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property.
+easytrieve_compiler | MVS program name of the COBOL compiler
+easytrieve_linkEditor | MVS program name of the link editor
+
 ### language-conf/languageConfigProps01.properties
 Sample language configuration properties file used by dbb-zappbuild/utilities/BuildUtilities.groovy.
 

build-conf/REXX.properties

@@ -44,16 +44,6 @@ rexx_printTempOptions=cyl space(5,5) unit(vio) blksize(133) lrecl(133) recfm(f,b
 # List of output datasets to document deletions
 rexx_outputDatasets=${rexx_cexecPDS},${rexx_loadPDS}
 
-#
-# Set filter used to exclude certain information from the link edit scanning.
-# The value contains a comma separated list of patterns.
-# example: A filter of *.SUB1, *.SUB2 will exclude modules SUB1 and SUB2
-#          from any dataset. To exclude member HELLO in PDS TEST.rexx will
-#          be matched by the pattern TEST.rexx.HELLO. The pattern TEST.rexx.*
-#          will match any member in the data set TEST.rexx.
-# The following filter excludes CICS and LE Library references.
-dbb.LinkEditScanner.excludeFilter = ${SDFHLOAD}.*, ${SCEELKED}.*
-
 # rexx_dependenciesDatasetMapping - an optional dbb property mapping to map dependencies to different target datasets
 #  this property is used when dependencies are copied to the different build libraries, e.q dclgens going into to a dedicated library
 #  note, that a dependency file needs to match a single rule

build-conf/build.properties

@@ -19,7 +19,7 @@
 buildPropFiles=${systemDatasets},dependencyReport.properties,Assembler.properties,BMS.properties,\
 MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,Cobol.properties,\
 LinkEdit.properties,PLI.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,\
-CRB.properties,zCEE3.properties,zCEE2.properties
+CRB.properties,zCEE3.properties,zCEE2.properties,Easytrieve.properties
 
 #
 # Comma separated list of property files defining system datasets

build-conf/datasets.properties

@@ -75,5 +75,10 @@ SBZUSAMP=
 # REXX Compiler Data Sets. Example: REXX.V1R4.SFANLMD
 SFANLMD=
 
+# Easytrieve Load Library. Example: 
+#  Easytrieve: EZT.PRODUCT.LOADLIB 
+#  IBM Migration Utility: SYS1.SFSYLOAD
+EZETLOAD=
+
 # PD Tools Common Component load library. Example : PDTCC.V1R8.SIPVMODA 
 PDTCCMOD=

build-conf/defaultzAppBuildConf.properties

@@ -25,6 +25,7 @@ dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"C" :: c
 dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"ASM" :: asm, mac
 dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"CPP" :: cpp, hpp
 dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"PLI" :: pli, inc
+dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"EASY" :: ezt, eztm
 dbb.scannerMapping = "scannerClass":"ZUnitConfigScanner" :: bzucfg
 # Custom mappings
 dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"REXX" :: rexx
@@ -79,7 +80,7 @@ impactBuildOnBuildPropertyChanges=false
 impactBuildOnBuildPropertyList=[${assembler_impactPropertyList},${assembler_impactPropertyListCICS},${assembler_impactPropertyListSQL},${bms_impactPropertyList},${cobol_impactPropertyList},${cobol_impactPropertyListCICS},${cobol_impactPropertyListSQL},${dbdgen_impactPropertyList},${linkedit_impactPropertyList},${mfs_impactPropertyList},${pli_impactPropertyList},${pli_impactPropertyListCICS},${pli_impactPropertyListSQL},${psbgen_impactPropertyList}]
 
 
-# Set filter used to exclude certain information from the link edit scanning.
+# Set filters used to exclude certain information from the link edit (load module) scanning.
 # The value contains a comma separated list of patterns.
 # example: A filter of *.SUB1, *.SUB2 will exclude modules SUB1 and SUB2
 #          from any dataset. To exclude member HELLO in PDS TEST.ASM will

build.groovy

@@ -39,7 +39,7 @@ try {
 	println(errorMsg)
 	props.error = "true"
 	buildUtils.updateBuildResult(errorMsg:errorMsg)
-	finalizeBuildProcess(start:startTime, 0)
+	finalizeBuildProcess(start:startTime, count:0)
 }
 
 // create build list
@@ -270,6 +270,7 @@ options:
 	cli.v(longOpt:'verbose', 'Flag to turn on script trace')
 	cli.pv(longOpt:'preview', 'Supplemental flag indicating to run build in preview mode without processing the execute commands')
 	cli.cd(longOpt:'checkDatasets', 'Optional flag to validate the presense of the defined system datasets. ')
+	cli.cb(longOpt:'applicationCurrentBranch', args:1, 'Application\'s current Git branch. Used in pipeline builds to identify DBB Metadatastore objects.')
 
 	// scan options
 	cli.s(longOpt:'scanOnly', 'Flag indicating to only scan source files for application without building anything (deprecated use --scanSource)')
@@ -301,7 +302,7 @@ options:
 	cli.ccp(longOpt:'cccPort', args:1, argName:'cccPort', 'Headless Code Coverage Collector port (if not specified IDz will be used for reporting)')
 	cli.cco(longOpt:'cccOptions', args:1, argName:'cccOptions', 'Headless Code Coverage Collector Options')
 
-	// build framework options
+	// build reporting options
 	cli.re(longOpt:'reportExternalImpacts', 'Flag to activate analysis and report of external impacted files within DBB collections')
 
 	// IDE user build dependency file options
@@ -471,6 +472,7 @@ def populateBuildProperties(def opts) {
 	if (opts.m) props.mergeBuild = 'true'
 	if (opts.pv) props.preview = 'true'
 	if (opts.cd) props.checkDatasets = 'true'
+	if (opts.cb) props.applicationCurrentBranch = opts.cb
 
 	// scan options
 	if (opts.s) props.scanOnly = 'true'
@@ -516,7 +518,7 @@ def populateBuildProperties(def opts) {
 	if (opts.arguments()) props.buildFile = opts.arguments()[0].trim()
 
 	// set calculated properties
-	if (!props.userBuild) {
+	if (!props.userBuild && !props.applicationCurrentBranch) {
 		def gitDir = buildUtils.getAbsolutePath(props.application)
 		if ( gitUtils.isGitDetachedHEAD(gitDir) )
 			props.applicationCurrentBranch = gitUtils.getCurrentGitDetachedBranch(gitDir)
@@ -586,7 +588,11 @@ def createBuildList() {
 	}
 	// check if impact build
 	else if (props.impactBuild) {
-		println "** --impactBuild option selected. $action impacted programs for application ${props.application} "
+		if (props.baselineRef) {
+			println "** --impactBuild --baselineRef ${props.baselineRef} option selected. $action impacted programs for application ${props.application} "
+		} else {
+			println "** --impactBuild option selected. $action impacted programs for application ${props.application} "
+		}
 		if (metadataStore) {
 			(buildSet, changedFiles, deletedFiles, renamedFiles, changedBuildProperties) = impactUtils.createImpactBuildList()		}
 		else {
@@ -744,8 +750,9 @@ def finalizeBuildProcess(Map args) {
 					String gitchangedfilesKey = "$gitchangedfilesPrefix${buildUtils.relativizePath(dir)}"
 					def lastBuildResult= buildUtils.retrieveLastBuildResult()
 					if (lastBuildResult){
-						String baselineHash = lastBuildResult.getProperty(key)
-						String gitchangedfilesLink = props.gitRepositoryURL << "/" << props.gitRepositoryCompareService <<"/" << baselineHash << ".." << currenthash
+						String userBaselineRef = (props.baselineRef) ? buildUtils.getUserProvidedBaselineRef(dir).replaceAll("origin/","") : null
+						String baselineHash = (userBaselineRef) ? userBaselineRef : lastBuildResult.getProperty(key)
+						String gitchangedfilesLink = props.gitRepositoryURL << "/" << props.gitRepositoryCompareService <<"/" << baselineHash << "..." << currenthash
 						String gitchangedfilesLinkUrl = new URI(gitchangedfilesLink).normalize().toString()
 						if (props.verbose) println "** Setting property $gitchangedfilesKey : $gitchangedfilesLinkUrl"
 						buildResult.setProperty(gitchangedfilesKey, gitchangedfilesLink)