diff --git a/README.md b/README.md index 2d0c3b51..096fbc7a 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ The zAppBuild sample provides the following *language* build scripts by default: * ZunitConfig.groovy * CRB.groovy * Transfer.groovy (for transport non-buildable files like JCL or PROC into build libraries and register them as build output) +* Easytrieve.groovy All language scripts both compile and optionally link-edit programs. The language build scripts are intended to be useful out of the box but depending on the complexity of your applications' build requirements, may require modifications to meet your development team's needs. By following the examples used in the existing language build scripts of keeping all application specific references out of the build scripts and instead using configuration properties with strong default values, the zAppBuild sample can continue to be a generic build solution for all of your specific applications. diff --git a/build-conf/Cobol.properties b/build-conf/Cobol.properties index 1920e9ed..c391701c 100644 --- a/build-conf/Cobol.properties +++ b/build-conf/Cobol.properties @@ -62,16 +62,6 @@ cobol_compileErrorFeedbackXmlOptions=tracks space(200,40) dsorg(PS) blksize(2799 # List of output datasets to document deletions cobol_outputDatasets=${cobol_loadPDS} -# -# Set filter used to exclude certain information from the link edit scanning. -# The value contains a comma separated list of patterns. -# example: A filter of *.SUB1, *.SUB2 will exclude modules SUB1 and SUB2 -# from any dataset. To exclude member HELLO in PDS TEST.COBOL will -# be matched by the pattern TEST.COBOL.HELLO. The pattern TEST.COBOL.* -# will match any member in the data set TEST.COBOL. -# The following filter excludes CICS and LE Library references. -dbb.LinkEditScanner.excludeFilter = ${SDFHLOAD}.*, ${SCEELKED}.* - # # additional libraries for compile SYSLIB concatenation, comma-separated, see definitions in application-conf # cobol_compileSyslibConcatenation= diff --git a/build-conf/Easytrieve.properties b/build-conf/Easytrieve.properties new file mode 100644 index 00000000..cbbba937 --- /dev/null +++ b/build-conf/Easytrieve.properties @@ -0,0 +1,45 @@ +# Releng properties used by language/Easytrieve.groovy + +# +# Comma separated list of required build properties for easytrieve.groovy +easytrieve_requiredBuildProperties=easytrieve_srcPDS,easytrieve_cpyPDS,easytrieve_objPDS,easytrieve_loadPDS,\ +easytrieve_compiler,easytrieve_linkEditor,easytrieve_tempOptions,applicationOutputsCollectionName,\ +SCEELKED + +# +# easytrieve compiler name +# Easytrieve EZTCOM +# IBM Migration Utility: FSYTPA00 +easytrieve_compiler=FSYTPA00 + +# +# linker name +easytrieve_linkEditor=IEWBLINK + +# +# easytrieve source data sets +easytrieve_srcPDS=${hlq}.EZT +easytrieve_cpyPDS=${hlq}.MAC +easytrieve_objPDS=${hlq}.OBJ +easytrieve_dbrmPDS=${hlq}.DBRM + +# +# easytrieve load data sets +easytrieve_loadPDS=${hlq}.LOAD + +# +# List the data sets that need to be created and their creation options +# https://www.ibm.com/docs/en/mufz/5.1?topic=jcl-jcmucl2jtwo-step-translate-link-without-proc +easytrieve_srcDatasets=${easytrieve_srcPDS},${easytrieve_cpyPDS},${easytrieve_objPDS},${easytrieve_dbrmPDS} +easytrieve_srcOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(library) + +easytrieve_loadDatasets=${easytrieve_loadPDS} +easytrieve_loadOptions=cyl space(1,1) dsorg(PO) recfm(U) blksize(32760) dsntype(library) + +easytrieve_tempOptions=cyl space(5,5) unit(vio) blksize(80) lrecl(80) recfm(f,b) new +easytrieve_printTempOptions=cyl space(5,5) unit(vio) blksize(133) lrecl(133) recfm(f,b) new + +# +# List of output datasets to document deletions +easytrievel_outputDatasets=${easytrieve_loadPDS} + diff --git a/build-conf/README.md b/build-conf/README.md index 67396051..af677bbf 100644 --- a/build-conf/README.md +++ b/build-conf/README.md @@ -31,6 +31,7 @@ SDSNLOAD | DB2 Load Library. Example: DB2.V9R1M0.SDSNLOAD SDSNEXIT | DB2 Exit Library. Example: DBC0CFG.SDSNEXIT SFELLOAD | Optional IDz Load Library. Example: FEL.V14R0M0.SFELLOAD SBZUSAMP | Optional z/OS Dynamic Test Runner IDz zUnit / WAZI VTP library containing necessary copybooks. Example : FEL.V14R2.SBZUSAMP +EZETLOAD | Easytrieve Load Library. Example for IBM Migration Utility: SYS1.SFSYLOAD ### build.properties General properties used mainly by `build.groovy` but can also be a place to declare properties used by multiple language scripts. @@ -336,6 +337,26 @@ transfer_xmlPDS | Sample dataset for xml members transfer_dsOptions | BPXWDYN creation options for creating 'source' type data sets transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets +### Easytrieve.properties +Build properties used by zAppBuild/language/Easytrieve.groovy + +Property | Description +--- | --- +easytrieve_requiredBuildProperties | Comma separated list of required build properties for language/Cobol.groovy +easytrieve_srcPDS | Dataset to move Easytrieve source files to from USS +easytrieve_cpyPDS | Dataset to move Easytrieve macros to from USS +easytrieve_objPDS | Dataset to create object decks in from compile step +easytrieve_dbrmPDS | Dataset to create DB2 DBRM modules in from compile step +easytrieve_loadPDS | Dataset to create load modules in from link edit step +easytrieve_srcDataSets | Comma separated list of 'source' type data sets +easytrieve_srcOptions | BPXWDYN creation options for creating 'source' type data sets +easytrieve_loadDatasets | Comma separated list of 'load module' type data sets +easytrieve_loadOptions | BPXWDYN creation options for 'load module' type data sets +easytrieve_tempOptions | BPXWDYN creation options for temporary data sets +easytrieve_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. +easytrieve_compiler | MVS program name of the COBOL compiler +easytrieve_linkEditor | MVS program name of the link editor + ### language-conf/languageConfigProps01.properties Sample language configuration properties file used by dbb-zappbuild/utilities/BuildUtilities.groovy. diff --git a/build-conf/REXX.properties b/build-conf/REXX.properties index e0dd7a52..6277cfe7 100644 --- a/build-conf/REXX.properties +++ b/build-conf/REXX.properties @@ -44,16 +44,6 @@ rexx_printTempOptions=cyl space(5,5) unit(vio) blksize(133) lrecl(133) recfm(f,b # List of output datasets to document deletions rexx_outputDatasets=${rexx_cexecPDS},${rexx_loadPDS} -# -# Set filter used to exclude certain information from the link edit scanning. -# The value contains a comma separated list of patterns. -# example: A filter of *.SUB1, *.SUB2 will exclude modules SUB1 and SUB2 -# from any dataset. To exclude member HELLO in PDS TEST.rexx will -# be matched by the pattern TEST.rexx.HELLO. The pattern TEST.rexx.* -# will match any member in the data set TEST.rexx. -# The following filter excludes CICS and LE Library references. -dbb.LinkEditScanner.excludeFilter = ${SDFHLOAD}.*, ${SCEELKED}.* - # rexx_dependenciesDatasetMapping - an optional dbb property mapping to map dependencies to different target datasets # this property is used when dependencies are copied to the different build libraries, e.q dclgens going into to a dedicated library # note, that a dependency file needs to match a single rule diff --git a/build-conf/build.properties b/build-conf/build.properties index 802e8c56..97c5976a 100644 --- a/build-conf/build.properties +++ b/build-conf/build.properties @@ -19,7 +19,7 @@ buildPropFiles=${systemDatasets},dependencyReport.properties,Assembler.properties,BMS.properties,\ MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,Cobol.properties,\ LinkEdit.properties,PLI.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,\ -CRB.properties,zCEE3.properties,zCEE2.properties +CRB.properties,zCEE3.properties,zCEE2.properties,Easytrieve.properties # # Comma separated list of property files defining system datasets diff --git a/build-conf/datasets.properties b/build-conf/datasets.properties index ad5d366c..abd09915 100644 --- a/build-conf/datasets.properties +++ b/build-conf/datasets.properties @@ -75,5 +75,10 @@ SBZUSAMP= # REXX Compiler Data Sets. Example: REXX.V1R4.SFANLMD SFANLMD= +# Easytrieve Load Library. Example: +# Easytrieve: EZT.PRODUCT.LOADLIB +# IBM Migration Utility: SYS1.SFSYLOAD +EZETLOAD= + # PD Tools Common Component load library. Example : PDTCC.V1R8.SIPVMODA PDTCCMOD= diff --git a/build-conf/defaultzAppBuildConf.properties b/build-conf/defaultzAppBuildConf.properties index 9b1b6cc9..e6b1a739 100644 --- a/build-conf/defaultzAppBuildConf.properties +++ b/build-conf/defaultzAppBuildConf.properties @@ -25,6 +25,7 @@ dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"C" :: c dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"ASM" :: asm, mac dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"CPP" :: cpp, hpp dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"PLI" :: pli, inc +dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"EASY" :: ezt, eztm dbb.scannerMapping = "scannerClass":"ZUnitConfigScanner" :: bzucfg # Custom mappings dbb.scannerMapping = "scannerClass":"DependencyScanner", "languageHint":"REXX" :: rexx @@ -79,7 +80,7 @@ impactBuildOnBuildPropertyChanges=false impactBuildOnBuildPropertyList=[${assembler_impactPropertyList},${assembler_impactPropertyListCICS},${assembler_impactPropertyListSQL},${bms_impactPropertyList},${cobol_impactPropertyList},${cobol_impactPropertyListCICS},${cobol_impactPropertyListSQL},${dbdgen_impactPropertyList},${linkedit_impactPropertyList},${mfs_impactPropertyList},${pli_impactPropertyList},${pli_impactPropertyListCICS},${pli_impactPropertyListSQL},${psbgen_impactPropertyList}] -# Set filter used to exclude certain information from the link edit scanning. +# Set filters used to exclude certain information from the link edit (load module) scanning. # The value contains a comma separated list of patterns. # example: A filter of *.SUB1, *.SUB2 will exclude modules SUB1 and SUB2 # from any dataset. To exclude member HELLO in PDS TEST.ASM will diff --git a/build.groovy b/build.groovy index 30358bf4..5fde7cf1 100644 --- a/build.groovy +++ b/build.groovy @@ -39,7 +39,7 @@ try { println(errorMsg) props.error = "true" buildUtils.updateBuildResult(errorMsg:errorMsg) - finalizeBuildProcess(start:startTime, 0) + finalizeBuildProcess(start:startTime, count:0) } // create build list @@ -270,6 +270,7 @@ options: cli.v(longOpt:'verbose', 'Flag to turn on script trace') cli.pv(longOpt:'preview', 'Supplemental flag indicating to run build in preview mode without processing the execute commands') cli.cd(longOpt:'checkDatasets', 'Optional flag to validate the presense of the defined system datasets. ') + cli.cb(longOpt:'applicationCurrentBranch', args:1, 'Application\'s current Git branch. Used in pipeline builds to identify DBB Metadatastore objects.') // scan options cli.s(longOpt:'scanOnly', 'Flag indicating to only scan source files for application without building anything (deprecated use --scanSource)') @@ -301,7 +302,7 @@ options: cli.ccp(longOpt:'cccPort', args:1, argName:'cccPort', 'Headless Code Coverage Collector port (if not specified IDz will be used for reporting)') cli.cco(longOpt:'cccOptions', args:1, argName:'cccOptions', 'Headless Code Coverage Collector Options') - // build framework options + // build reporting options cli.re(longOpt:'reportExternalImpacts', 'Flag to activate analysis and report of external impacted files within DBB collections') // IDE user build dependency file options @@ -471,6 +472,7 @@ def populateBuildProperties(def opts) { if (opts.m) props.mergeBuild = 'true' if (opts.pv) props.preview = 'true' if (opts.cd) props.checkDatasets = 'true' + if (opts.cb) props.applicationCurrentBranch = opts.cb // scan options if (opts.s) props.scanOnly = 'true' @@ -516,7 +518,7 @@ def populateBuildProperties(def opts) { if (opts.arguments()) props.buildFile = opts.arguments()[0].trim() // set calculated properties - if (!props.userBuild) { + if (!props.userBuild && !props.applicationCurrentBranch) { def gitDir = buildUtils.getAbsolutePath(props.application) if ( gitUtils.isGitDetachedHEAD(gitDir) ) props.applicationCurrentBranch = gitUtils.getCurrentGitDetachedBranch(gitDir) @@ -586,7 +588,11 @@ def createBuildList() { } // check if impact build else if (props.impactBuild) { - println "** --impactBuild option selected. $action impacted programs for application ${props.application} " + if (props.baselineRef) { + println "** --impactBuild --baselineRef ${props.baselineRef} option selected. $action impacted programs for application ${props.application} " + } else { + println "** --impactBuild option selected. $action impacted programs for application ${props.application} " + } if (metadataStore) { (buildSet, changedFiles, deletedFiles, renamedFiles, changedBuildProperties) = impactUtils.createImpactBuildList() } else { @@ -744,8 +750,9 @@ def finalizeBuildProcess(Map args) { String gitchangedfilesKey = "$gitchangedfilesPrefix${buildUtils.relativizePath(dir)}" def lastBuildResult= buildUtils.retrieveLastBuildResult() if (lastBuildResult){ - String baselineHash = lastBuildResult.getProperty(key) - String gitchangedfilesLink = props.gitRepositoryURL << "/" << props.gitRepositoryCompareService <<"/" << baselineHash << ".." << currenthash + String userBaselineRef = (props.baselineRef) ? buildUtils.getUserProvidedBaselineRef(dir).replaceAll("origin/","") : null + String baselineHash = (userBaselineRef) ? userBaselineRef : lastBuildResult.getProperty(key) + String gitchangedfilesLink = props.gitRepositoryURL << "/" << props.gitRepositoryCompareService <<"/" << baselineHash << "..." << currenthash String gitchangedfilesLinkUrl = new URI(gitchangedfilesLink).normalize().toString() if (props.verbose) println "** Setting property $gitchangedfilesKey : $gitchangedfilesLinkUrl" buildResult.setProperty(gitchangedfilesKey, gitchangedfilesLink) diff --git a/docs/BUILD.md b/docs/BUILD.md index fe196c01..af75f058 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -179,84 +179,91 @@ If buildFile is a text file (*.txt), then it is assumed to be a build list file. Options: required options: - -w,--workspace Absolute path to workspace (root) directory - containing all required source directories - -a,--application Application directory name (relative to workspace) - -o,--outDir Absolute path to the build output root directory - -h,--hlq High level qualifier for partition data sets + -w,--workspace Absolute path to workspace (root) directory + containing all required source directories + -a,--application Application directory name (relative to workspace) + -o,--outDir Absolute path to the build output root directory + -h,--hlq High level qualifier for partition data sets build options: - -f,--fullBuild Flag indicating to build all programs for - the application - -i,--impactBuild Flag indicating to build only programs impacted - by changed files since last successful build. - -b,--baselineRef Comma seperated list of git references to overwrite - the baselineHash hash in an impactBuild scenario. - -m,--mergeBuild Flag indicating to build only source code changes which will be - merged back to the mainBuildBranch. - - -s,--scanOnly Flag indicating to only scan source files for application without building anything (deprecated use --scanSource) - -ss,--scanSource Flag indicating to only scan source files for application without building anything - -sl,--scanLoad Flag indicating to only scan load modules for application without building anything - -sa,--scanAll Flag indicating to scan both source files and load modules for application without building anything - -pv,--preview Supplemental flag indicating to run build in preview mode without processing the execute commands - - - -r,--reset Deletes the application's dependency collections - and build result group from the DBB repository - -v,--verbose Flag to turn on script trace - -d,--debug Flag to build modules for debugging with - IBM Debug for z/OS - -l,--logEncoding Encoding of output logs. Default is EBCDIC - directory for user build - -zTest,--runzTests Specify if zUnit Tests should be run - - -p,--propFiles Comma separated list of additional property files - to load. Absolute paths or relative to workspace - -po,--propOverwrites Comma separated list of key=value pairs for set and overwrite build properties - - -cc,--ccczUnit Flag to indicate to collect code coverage reports during zUnit step - -cch,--cccHost Headless Code Coverage Collector host (if not specified IDz will be used for reporting) - -ccp,--cccPort Headless Code Coverage Collector port (if not specified IDz will be used for reporting) - -cco,--cccOptions Headless Code Coverage Collector Options - - -re,--reportExternalImpacts Flag to activate analysis and report of external impacted files within DBB collections - -cd,--checkDatasets Flag to enable validation of the defined system dataset definitions. + -f,--fullBuild Flag indicating to build all programs for + the application + -i,--impactBuild Flag indicating to build only programs impacted + by changed files since last successful build. + -b,--baselineRef Comma seperated list of git references to overwrite + the baselineHash hash in an impactBuild scenario. + -m,--mergeBuild Flag indicating to build only source code changes which will be + merged back to the mainBuildBranch. + + -s,--scanOnly Flag indicating to only scan source files for application without building anything (deprecated use --scanSource) + -ss,--scanSource Flag indicating to only scan source files for application without building anything + -sl,--scanLoad Flag indicating to only scan load modules for application without building anything + -sa,--scanAll Flag indicating to scan both source files and load modules for application without building anything + -pv,--preview Supplemental flag indicating to run build in preview mode without processing the execute commands + + -cb,--applicationCurrentBranch Application's current Git branch. Used in pipeline builds to identify DBB Metadatastore objects. + + + -r,--reset Deletes the application's dependency collections + and build result group from the DBB repository + -v,--verbose Flag to turn on script trace + -d,--debug Flag to build modules for debugging with + IBM Debug for z/OS + -l,--logEncoding Encoding of output logs. Default is EBCDIC + directory for user build + -zTest,--runzTests Specify if zUnit Tests should be run + + -p,--propFiles Comma separated list of additional property files + to load. Absolute paths or relative to workspace + -po,--propOverwrites Comma separated list of key=value pairs for set and overwrite build properties + + -cc,--ccczUnit Flag to indicate to collect code coverage reports during zUnit step + -cch,--cccHost Headless Code Coverage Collector host (if not specified IDz will be used for reporting) + -ccp,--cccPort Headless Code Coverage Collector port (if not specified IDz will be used for reporting) + -cco,--cccOptions Headless Code Coverage Collector Options + + -re,--reportExternalImpacts Flag to activate analysis and report of external impacted files within DBB collections + -cd,--checkDatasets Flag to enable validation of the defined system dataset definitions. Db2 MetadataStore configuration options - -url,--url Db2 JDBC URL for the MetadataStore. - Example: jdbc:db2: - -id,--id Db2 user id for the MetadataStore - -pw,--pw Db2 password (encrypted with DBB Password Utility) for the MetadataStore - -pf,--pwFile Absolute or relative (from workspace) path to file containing Db2 password + -url,--url Db2 JDBC URL for the MetadataStore. + Example: jdbc:db2: + -id,--id Db2 user id for the MetadataStore + -pw,--pw Db2 password (encrypted with DBB Password Utility) for the MetadataStore + -pf,--pwFile Absolute or relative (from workspace) path to file containing Db2 password IDz/ZOD User Build options - -u,--userBuild Flag indicating running a user build - -dz,--debugzUnitTestcase Flag indicating to start a debug session for zUnit Test configurations as part of user build - -e,--errPrefix Unique id used for IDz error message datasets - -df,--dependencyFile Absolute or relative path (from workspace) to user build JSON file containing dependency information. + -u,--userBuild Flag indicating running a user build + -dz,--debugzUnitTestcase Flag indicating to start a debug session for zUnit Test configurations as part of user build + -e,--errPrefix Unique id used for IDz error message datasets + -df,--dependencyFile Absolute or relative path (from workspace) to user build JSON file containing dependency information. utility options - -help,--help Prints this message + -help,--help Prints this message ``` ## Invocation Samples including console log -- [Build a Single Program](#build-a-single-program) -- [Build a List of Programs](#build-a-list-of-programs) -- [Perform Full Build to build all files](#perform-full-build-to-build-all-files) -- [Perform Impact Build](#perform-impact-build) -- [Perform Impact Build for topic branches](#perform-impact-build-for-topic-branches) -- [Perform Impact Build by providing baseline reference for the analysis of changed files](#perform-impact-build-by-providing-baseline-reference-for-the-analysis-of-changed-files) -- [Perform a Merge build](#perform-a-merge-build) -- [Perform a Build in Preview Mode](#perform-a-build-in-preview-mode) -- [Perform a Scan Source build](#perform-a-scan-source-build) -- [Perform a Scan Source + Outputs build](#perform-a-scan-source--outputs-build) -- [Dynamically Overwrite build properties](#dynamically-overwrite-build-properties) -- [Validate System Datasets](#validate-system-datasets) +- [Building Applications with zAppBuild](#building-applications-with-zappbuild) + - [Common Pipeline Invocation Examples](#common-pipeline-invocation-examples) + - [Common User Build Invocation Examples](#common-user-build-invocation-examples) + - [Command Line Options Summary](#command-line-options-summary) + - [Invocation Samples including console log](#invocation-samples-including-console-log) + - [Build a Single Program](#build-a-single-program) + - [Build a List of Programs](#build-a-list-of-programs) + - [Perform Full Build to build all files](#perform-full-build-to-build-all-files) + - [Perform Impact Build](#perform-impact-build) + - [Perform Impact Build for topic branches](#perform-impact-build-for-topic-branches) + - [Perform Impact Build by providing baseline reference for the analysis of changed files](#perform-impact-build-by-providing-baseline-reference-for-the-analysis-of-changed-files) + - [Perform a Merge build](#perform-a-merge-build) + - [Perform a Build in Preview Mode](#perform-a-build-in-preview-mode) + - [Perform a Scan Source build](#perform-a-scan-source-build) + - [Perform a Scan Source + Outputs build](#perform-a-scan-source--outputs-build) + - [Dynamically Overwrite build properties](#dynamically-overwrite-build-properties) + - [Validate System Datasets](#validate-system-datasets) ### Build a Single Program diff --git a/languages/Easytrieve.groovy b/languages/Easytrieve.groovy new file mode 100644 index 00000000..16f9c80b --- /dev/null +++ b/languages/Easytrieve.groovy @@ -0,0 +1,273 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.dependency.* +import com.ibm.dbb.build.* +import groovy.transform.* + + +// define script properties +@Field BuildProperties props = BuildProperties.getInstance() +@Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) +@Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) + + +/*** + * + * Sample language script to compile and link Easytrieve source code + * + * https://www.ibm.com/docs/en/mufz/5.1?topic=jcl-jcmucl2jtwo-step-translate-link-without-proc + * + */ + +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") + +//verify required build properties +buildUtils.assertBuildProperties(props.easytrieve_requiredBuildProperties) + +// create language datasets +def langQualifier = "easytrieve" +buildUtils.createLanguageDatasets(langQualifier) + + +//sort the build list based on build file rank if provided +List sortedList = buildUtils.sortBuildList(argMap.buildList, 'easytrieve_fileBuildRank') +int currentBuildFileNumber = 1 + +//iterate through build list + +sortedList.each { buildFile -> + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" + + // configure dependency resolution and create logical file + + String dependencySearch = props.getFileProperty('easytrieve_dependencySearch', buildFile) + SearchPathDependencyResolver dependencyResolver = new SearchPathDependencyResolver(dependencySearch) + + // copy source files and dependency files to data sets + buildUtils.copySourceFiles(buildFile, props.easytrieve_srcPDS, 'easytrieve_dependenciesDatasetMapping', null, dependencyResolver) + + // Get logical file + LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) + + + // create mvs commands + String member = CopyToPDS.createMemberName(buildFile) + String needsLinking = props.getFileProperty('easytrieve_linkEdit', buildFile) + + File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.ezt.log") + if (logFile.exists()) + logFile.delete() + + MVSExec compile = createCompileCommand(buildFile, logicalFile, member, logFile) + MVSExec linkEdit + if (needsLinking.toBoolean()) linkEdit = createLinkEditCommand(buildFile, logicalFile, member, logFile) + + // execute mvs commands in a mvs job + MVSJob job = new MVSJob() + job.start() + + // compile the easytrieve program + int rc = compile.execute() + //int rc = 0 + int maxRC = props.getFileProperty('easytrieve_compileMaxRC', buildFile).toInteger() + + if (rc > maxRC) { + String errorMsg = "*! The compile return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)" + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile]) + } + else { + // if this program needs to be link edited . . . + + if (needsLinking.toBoolean()) { + rc = linkEdit.execute() + //rc = 0 + maxRC = props.getFileProperty('easytrieve_linkEditMaxRC', buildFile).toInteger() + + if (rc > maxRC) { + String errorMsg = "*! The link edit return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)" + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile]) + } + else { + // only scan the load module if load module scanning turned on for file + String scanLoadModule = props.getFileProperty('easytrieve_scanLoadModule', buildFile) + if (scanLoadModule && scanLoadModule.toBoolean()) + impactUtils.saveStaticLinkDependencies(buildFile, props.linkedit_loadPDS, logicalFile) + } + } + + } + + + // clean up passed DD statements + job.stop() +} + +// end script + + +//******************************************************************** +//* Method definitions +//******************************************************************** + +/* + * createEasytrieveParms - Builds up the EZT compiler parameter list from build and file properties + */ +def createEasytrieveParms(String buildFile, LogicalFile logicalFile) { + def parms = props.getFileProperty('easytrieve_compileParms', buildFile) ?: "" + + if (parms.startsWith(',')) + parms = parms.drop(1) + + if (props.verbose) println "Easytrieve compiler parms for $buildFile = $parms" + return parms +} + +/* + * createCompileCommand - creates a MVSExec command for compiling the EZT program (buildFile) + */ +def createCompileCommand(String buildFile, LogicalFile logicalFile, String member, File logFile) { + String parms = createEasytrieveParms(buildFile, logicalFile) + String compiler = props.getFileProperty('easytrieve_compiler', buildFile) + + // define the MVSExec command to compile the program + MVSExec compile = new MVSExec().file(buildFile).pgm(compiler).parm(parms) + + // add DD statements to the compile command + compile.dd(new DDStatement().name("SYSIN").dsn("${props.easytrieve_srcPDS}($member)").options('shr').report(true)) + compile.dd(new DDStatement().name("SYSPRINT").options(props.easytrieve_printTempOptions)) + (1..2).toList().each { num -> + compile.dd(new DDStatement().name("SYSUT$num").options(props.easytrieve_tempOptions)) + } + + // define object dataset allocation + compile.dd(new DDStatement().name("SYSLIN").dsn("${props.easytrieve_objPDS}($member)").options('shr').output(true)) + + // add a syslib to the compile command + compile.dd(new DDStatement().name("PANDD").dsn(props.easytrieve_cpyPDS).options("shr")) + + // add custom concatenation + def easytrieveSyslibConcatenation = props.getFileProperty('easytrieve_easytrieveSyslibConcatenation', buildFile) ?: "" + if (easytrieveSyslibConcatenation) { + def String[] syslibDatasets = easytrieveSyslibConcatenation.split(','); + for (String syslibDataset : syslibDatasets ) + compile.dd(new DDStatement().dsn(syslibDataset).options("shr")) + } + + // add a tasklib to the compile command with optional concatenations + compile.dd(new DDStatement().name("TASKLIB").dsn(props.EZETLOAD).options("shr")) + + // add optional DBRMLIB if build file contains DB2 code + if (buildUtils.isSQL(logicalFile)) + compile.dd(new DDStatement().name("DBRMLIB").dsn("$props.easytrieve_dbrmPDS($member)").options('shr').output(true).deployType('DBRM')) + + // add a copy command to the compile command to copy the SYSPRINT from the temporary dataset to an HFS log file + compile.copy(new CopyToHFS().ddName("SYSPRINT").file(logFile).hfsEncoding(props.logEncoding)) + + return compile +} + + +/* + * createLinkEditCommand - creates a MVSExec command for link editing the EZT object module produced by the compile + */ +def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String member, File logFile) { + String parms = props.getFileProperty('easytrieve_linkEditParms', buildFile) + String linker = props.getFileProperty('easytrieve_linkEditor', buildFile) + String linkEditStream = props.getFileProperty('easytrieve_linkEditStream', buildFile) + + // obtain githash for buildfile + String easytrieve_storeSSI = props.getFileProperty('easytrieve_storeSSI', buildFile) + if (easytrieve_storeSSI && easytrieve_storeSSI.toBoolean() && (props.mergeBuild || props.impactBuild || props.fullBuild)) { + String ssi = buildUtils.getShortGitHash(buildFile) + if (ssi != null) parms = parms + ",SSI=$ssi" + } + + if (props.verbose) println "*** Link-Edit parms for $buildFile = $parms" + + // define the MVSExec command to link edit the program + MVSExec linkedit = new MVSExec().file(buildFile).pgm(linker).parm(parms) + + // Assemble linkEditInstream to define SYSIN as instreamData + String sysin_linkEditInstream = '' + + // appending configured linkEdit stream if specified + if (linkEditStream) { + sysin_linkEditInstream += " " + linkEditStream.replace("\\n","\n").replace('@{member}',member) + } + + // appending IDENTIFY statement to link phase for traceability of load modules + // this adds an IDRU record, which can be retrieved with amblist + def identifyLoad = props.getFileProperty('easytrieve_identifyLoad', buildFile) + + if (identifyLoad && identifyLoad.toBoolean()) { + String identifyStatement = buildUtils.generateIdentifyStatement(buildFile, props.easytrieve_loadOptions) + if (identifyStatement != null ) { + sysin_linkEditInstream += identifyStatement + } + } + + // appending mq stub according to file flags + if(buildUtils.isMQ(logicalFile)) { + // include mq stub program + // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs + sysin_linkEditInstream += buildUtils.getMqStubInstruction(logicalFile) + } + + // appending debug exit to link instructions + if (props.debug && linkDebugExit!= null) { + sysin_linkEditInstream += " " + linkDebugExit.replace("\\n","\n").replace('@{member}',member) + } + + // Define SYSIN dd as instream data + if (sysin_linkEditInstream) { + if (props.verbose) println("*** Generated linkcard input stream: \n $sysin_linkEditInstream") + linkedit.dd(new DDStatement().name("SYSIN").instreamData(sysin_linkEditInstream)) + } + + // add SYSLIN along the reference to SYSIN if configured through sysin_linkEditInstream + linkedit.dd(new DDStatement().name("SYSLIN").dsn("${props.easytrieve_objPDS}($member)").options('shr')) + if (sysin_linkEditInstream) linkedit.dd(new DDStatement().ddref("SYSIN")) + + // add DD statements to the linkedit command + String deployType = buildUtils.getDeployType("easytrieve", buildFile, logicalFile) + linkedit.dd(new DDStatement().name("SYSLMOD").dsn("${props.easytrieve_loadPDS}($member)").options('shr').output(true).deployType(deployType)) + linkedit.dd(new DDStatement().name("SYSPRINT").options(props.easytrieve_printTempOptions)) + linkedit.dd(new DDStatement().name("SYSUT1").options(props.easytrieve_tempOptions)) + + // add a syslib to the compile command with optional concatenation + linkedit.dd(new DDStatement().name("SYSLIB").dsn(props.easytrieve_objPDS).options("shr")) + + // add custom concatenation + def linkEditSyslibConcatenation = props.getFileProperty('easytrieve_linkEditSyslibConcatenation', buildFile) ?: "" + if (linkEditSyslibConcatenation) { + def String[] syslibDatasets = linkEditSyslibConcatenation.split(','); + for (String syslibDataset : syslibDatasets ) + linkedit.dd(new DDStatement().dsn(syslibDataset).options("shr")) + } + + linkedit.dd(new DDStatement().dsn(props.SCEELKED).options("shr")) + + // Add Debug Dataset to find the debug exit to SYSLIB + if (props.debug && props.SEQAMOD) + linkedit.dd(new DDStatement().dsn(props.SEQAMOD).options("shr")) + + if (buildUtils.isCICS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + + if (buildUtils.isSQL(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) + + if (buildUtils.isMQ(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SCSQLOAD).options("shr")) + + // add a copy command to the linkedit command to append the SYSPRINT from the temporary dataset to the HFS log file + linkedit.copy(new CopyToHFS().ddName("SYSPRINT").file(logFile).hfsEncoding(props.logEncoding).append(true)) + + return linkedit +} diff --git a/languages/README.md b/languages/README.md index efc1a428..0bff627f 100644 --- a/languages/README.md +++ b/languages/README.md @@ -12,6 +12,7 @@ zAppBuild comes with a number of language specific build scripts. These script * MFS.groovy * zCEE3.groovy * ZunitConfig.groovy +* Easytrieve.groovy All language scripts both compile and optionally link-edit programs. The language build scripts are intended to be useful out of the box but depending on the complexity of your applications' build requirements, may require modifications to meet your development team's needs. By following the examples used in the existing language build scripts of keeping all application specific references out of the build scripts and instead using configuration properties with strong default values, the zAppBuild sample can continue to be a generic build solution for all of your specific applications. diff --git a/samples/application-conf/Easytrieve.properties b/samples/application-conf/Easytrieve.properties new file mode 100644 index 00000000..78c46a8b --- /dev/null +++ b/samples/application-conf/Easytrieve.properties @@ -0,0 +1,69 @@ +# Application properties used by zAppBuild/language/Easytrieve.groovy + +# +# default Easytrieve program build rank - used to sort language build file list +# leave empty - overridden by file properties if sorting needed +easytrieve_fileBuildRank= + +# +# Easytrieve dependencySearch configuration +# searchPath defined in application.properties +easytrieve_dependencySearch=${eztMacSearch} + +# +# default Easytrieve maximum RCs allowed +# can be overridden by file properties +easytrieve_compileMaxRC=4 +easytrieve_linkEditMaxRC=4 + +# +# default Easytrieve compiler parameters +# can be overridden by file properties +easytrieve_compileParms= + +# +# default LinkEdit parameters +# can be overridden by file properties +easytrieve_linkEditParms=LIST,LET,XREF,MAP,AMODE(31),RMODE(24) + +# Optional linkEditStream defining additional link instructions via SYSIN dd +# cobol_linkEditStream= INCLUDE SYSLIB(EZTJT) \n +easytrieve_linkEditStream= + +# +# execute link edit step +# can be overridden by file properties +easytrieve_linkEdit=true + +# +# store abbrev git hash in ssi field +# available for buildTypes impactBuild, mergeBuild and fullBuild +# can be overridden by file properties +easytrieve_storeSSI=true + +# +# flag to generate IDENTIFY statement during link edit phase +# to create an user data record (IDRU) to "sign" the load module with +# an identify String: / +# to increase traceability +# +# can be overridden by file properties +# default: true +easytrieve_identifyLoad=true + +# +# default deployType +easytrieve_deployType=EZTLOAD + +# +# scan link edit load module for link dependencies +# can be overridden by file properties +easytrieve_scanLoadModule=false + +# +# additional libraries for compile SYSLIB concatenation, comma-separated +easytrieve_compileSyslibConcatenation= + +# +# additional libraries for linkEdit SYSLIB concatenation, comma-separated +easytrieve_linkEditSyslibConcatenation= \ No newline at end of file diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index b86d2b5c..a7e4ad4c 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -310,6 +310,21 @@ rexx_cexec_deployType | default deployType CEXEC | true rexx_compileSyslibConcatenation | A comma-separated list of libraries to be concatenated in syslib during compile step | true rexx_linkEditSyslibConcatenation | A comma-separated list of libraries to be concatenated in syslib during linkEdit step | true +### Eastrieve.properties +Application properties used by zAppBuild/language/Easytrieve.groovy + +Property | Description | Overridable +--- | --- | --- +easytrieve_compileMaxRC | Default compile maximum RC allowed. | true +easytrieve_linkEditMaxRC | Default link edit maximum RC allowed. | true +easytrieve_dependencySearch | Default dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true +easytrieve_compileParms | Default base compile parameters. | true +easytrieve_linkEdit | Flag indicating to execute the link edit step to produce a compiled easytrieve for the source file. | true +easytrieve_linkEditParms | Default link edit parameters. | true +easytrieve_deployType | default deployType | true +easytrieve_compileSyslibConcatenation | A comma-separated list of libraries to be concatenated in syslib during compile step | true +easytrieve_linkEditSyslibConcatenation | A comma-separated list of libraries to be concatenated in syslib during linkEdit step | true + ### Transfer.properties Application properties used by zAppBuild/language/Transfer.groovy diff --git a/samples/application-conf/application.properties b/samples/application-conf/application.properties index 87e2a66c..2ad1a2ba 100644 --- a/samples/application-conf/application.properties +++ b/samples/application-conf/application.properties @@ -114,7 +114,7 @@ resolveSubsystems=false # # this configuration is used when running zAppBuild with the --impactBuild option # to calculate impacted files based on the identified changed files -impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${zunitTestConfigSearch}${zunitTestcasePgmSearch} +impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${zunitTestConfigSearch}${zunitTestcasePgmSearch}${eztMacSearch} # # copybookSearch @@ -210,3 +210,9 @@ zunitPlayfileSearch = search:[SYSPLAY:]${workspace}/?path=${application}/testpla # see also build-conf/build.properties -> createTestcaseDependency # used in impact analysis zunitTestcasePgmSearch = search:[SYSPROG:PROGRAMDEPENDENCY]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli + +# +# eztMacSearch +# searchPath to locate EZT macros +# used in dependency resolution and impact analysis +eztMacSearch = search:${workspace}/?path=${application}/eztmacros/*.eztm diff --git a/samples/application-conf/file.properties b/samples/application-conf/file.properties index d54978a0..e3df501f 100644 --- a/samples/application-conf/file.properties +++ b/samples/application-conf/file.properties @@ -10,6 +10,7 @@ dbb.scriptMapping = DBDgen.groovy :: **/dbd/*.asm dbb.scriptMapping = Cobol.groovy :: **/*.cbl dbb.scriptMapping = LinkEdit.groovy :: **/*.lnk dbb.scriptMapping = PLI.groovy :: **/*.pli +dbb.scriptMapping = Easytrieve.groovy :: **/*.ezt dbb.scriptMapping = ZunitConfig.groovy :: **/*.bzucfg dbb.scriptMapping = Transfer.groovy :: **/*.jcl, **/*.xml dbb.scriptMapping = zCEE3.groovy :: **/openapi.yaml @@ -134,6 +135,9 @@ isIMS = true :: **/cobol/DLIBATCH.cbl # file mapping for generated zUnit Test case programs (PL/I) to use a seperate set of libraries # pli_testcase = true :: **/testcase/*.pli +# skip creating IDENTIFY binder control card statement for generated zunit test cases +# cobol_identifyLoad=false **/testcase/*.cbl, **/testcase/*.pli + # mapping for overwriting the impactResolution rules in application.properties # impactResolutionRules=[${copybookRule},${linkRule}] :: **/copy/*.cpy,**/cobol/*.cbl diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index f78d3003..d64f9458 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -605,6 +605,45 @@ def retrieveLastBuildResult(){ return lastBuildResult } +/* + * Returns the user-provided baseline hash for configurations passed in with the baselineRef cli option + * returns null if not defined + */ + +def getUserProvidedBaselineRef(String dir) { + + String hash + String relDir = relativizePath(dir) + + if (props.baselineRef) { + + String[] baselineMap = (props.baselineRef).split(",") + baselineMap.each{ + // case: baselineRef (gitref) + if(it.split(":").size()==1 && relDir.equals(props.application)){ + if (props.verbose) println "*** Baseline hash for directory $relDir retrieved from overwrite." + hash = it + } + // case: baselineRef (folder:gitref) + else if(it.split(":").size()>1){ + (appSrcDir, gitReference) = it.split(":") + if (appSrcDir.equals(relDir)){ + if (props.verbose) println "*** Baseline hash for directory $relDir retrieved from overwrite." + hash = gitReference + } + } else { + // No user-provided baseline ref found for dir + } + } + } else { + // No baseline ref defined + } + + return hash +} + + + /* * returns the deployType for a logicalFile depending on the * isCICS, isIMS and isDLI setting diff --git a/utilities/GitUtilities.groovy b/utilities/GitUtilities.groovy index 253c0b57..81c4c1c8 100644 --- a/utilities/GitUtilities.groovy +++ b/utilities/GitUtilities.groovy @@ -78,7 +78,7 @@ def getCurrentGitDetachedBranch(String gitDir) { // expecting references with "origin" as segment def origin = "origin/" if (gitBranchesArray.count {it.contains(origin)} > 1 ) { - String warningMsg = "*! (GitUtils.getCurrentGitDetachedBranch) Warning obtaining branch name for ($gitDir). Multiple references point to the same commit. ($gitBranchString)" + String warningMsg = "*! (GitUtils.getCurrentGitDetachedBranch) Issue when obtaining the current branch for ($gitDir): multiple references ($gitBranchString) point to the same commit. Consider passing the current Git branch with the '--applicationCurrentBranch' CLI parameter." println(warningMsg) updateBuildResult(warningMsg:warningMsg) } diff --git a/utilities/ImpactUtilities.groovy b/utilities/ImpactUtilities.groovy index 41a5eef0..23180eaf 100644 --- a/utilities/ImpactUtilities.groovy +++ b/utilities/ImpactUtilities.groovy @@ -324,28 +324,16 @@ def calculateChangedFiles(BuildResult lastBuildResult, boolean calculateConcurre // get the baseline hash for all build directories directories.each { dir -> dir = buildUtils.getAbsolutePath(dir) + if (props.verbose) println "** Getting baseline hash for directory $dir" String key = "$hashPrefix${buildUtils.relativizePath(dir)}" String relDir = buildUtils.relativizePath(dir) + String hash // retrieve baseline reference overwrite if set if (props.baselineRef){ - String[] baselineMap = (props.baselineRef).split(",") - baselineMap.each{ - // case: baselineRef (gitref) - if(it.split(":").size()==1 && relDir.equals(props.application)){ - if (props.verbose) println "*** Baseline hash for directory $relDir retrieved from overwrite." - hash = it - } - // case: baselineRef (folder:gitref) - else if(it.split(":").size()>1){ - (appSrcDir, gitReference) = it.split(":") - if (appSrcDir.equals(relDir)){ - if (props.verbose) println "*** Baseline hash for directory $relDir retrieved from overwrite." - hash = gitReference - } - } - } + // get user-provided baseline configuration from cli config + hash = buildUtils.getUserProvidedBaselineRef(dir) // for build directories which are not specified in baselineRef mapping, return the info from lastBuildResult if (hash == null && lastBuildResult) { hash = lastBuildResult.getProperty(key) diff --git a/version.properties b/version.properties index ed363131..b5116936 100644 --- a/version.properties +++ b/version.properties @@ -2,7 +2,7 @@ # delivered via the public github repository # https://github.com/IBM/dbb-zappbuild. # Helps to understand from which version a copy was taken. -zappbuild_baseVersion=3.8.0 +zappbuild_baseVersion=3.9.0 # use this property to add you own version suffix to indicate # contributed enhancements and your own revisions