From 71ba3139d29fe427d74903c8ee739c378de6b6d7 Mon Sep 17 00:00:00 2001 From: Flook Peter Date: Wed, 20 Mar 2024 16:50:21 +0800 Subject: [PATCH] Push logs to file, define folder for logs for each OS --- .github/workflows/build.yml | 4 ++-- app/src/main/resources/log4j2.properties | 9 +++++++++ .../datacaterer/core/ui/plan/ConnectionRepository.scala | 2 +- .../datacaterer/core/ui/plan/PlanRepository.scala | 9 ++++++++- misc/jpackage/jpackage-linux.cfg | 3 ++- misc/jpackage/jpackage-mac.cfg | 3 ++- misc/jpackage/jpackage-windows.cfg | 5 +++-- misc/jpackage/jpackage.cfg | 2 +- 8 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3dff72b..f70b4b9b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,13 +83,13 @@ jobs: with: name: jars path: app/build/libs/ - - name: Package jar as exe + - name: Package jar as msi run: 'jpackage "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-windows.cfg"' - name: Upload installer uses: actions/upload-artifact@v4 with: name: data-caterer-windows - path: "DataCaterer-1.0.0.exe" + path: "DataCaterer-1.0.0.msi" overwrite: true linux: diff --git a/app/src/main/resources/log4j2.properties b/app/src/main/resources/log4j2.properties index 04118980..4e891eb0 100644 --- a/app/src/main/resources/log4j2.properties +++ b/app/src/main/resources/log4j2.properties @@ -1,10 +1,19 @@ rootLogger.level=${env:LOG_LEVEL:-info} +rootLogger.appenderRefs=console,file rootLogger.appenderRef.stdout.ref=console +rootLogger.appenderRef.logfile.ref=file +# Console log appender.console.type=Console appender.console.name=console appender.console.target=SYSTEM_OUT appender.console.layout.type=PatternLayout appender.console.layout.pattern=%d{dd/MM/yyyy HH:mm:ss} [%-5p] %c: %m%n%ex +# File log +appender.file.type=File +appender.file.name=file +appender.file.fileName=${sys:LOG_FOLDER:-logs}/data-caterer.log +appender.file.layout.type=PatternLayout +appender.file.layout.pattern=%d{dd/MM/yyyy HH:mm:ss} [%-5p] %c: %m%n%ex # Settings to quiet third party logs that are too verbose logger.spark.name=org.apache.spark logger.spark.level=error diff --git a/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/ConnectionRepository.scala b/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/ConnectionRepository.scala index 8fbfe1e2..00fce00a 100644 --- a/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/ConnectionRepository.scala +++ b/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/ConnectionRepository.scala @@ -65,7 +65,7 @@ object ConnectionRepository extends JsonSupport { } def getConnection(name: String): Connection = { - LOGGER.info(s"Getting connection details, connection-name=$name") + LOGGER.debug(s"Getting connection details, connection-name=$name") val connectionFile = Path.of(s"$connectionSaveFolder/$name.csv") val connection = Connection.fromString(Files.readString(connectionFile)) val mappedPw = connection.options.map(o => if (o._1 == "password") (o._1, "***") else o) diff --git a/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanRepository.scala b/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanRepository.scala index 9ee0c9fd..86d4422d 100644 --- a/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanRepository.scala +++ b/app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanRepository.scala @@ -8,7 +8,7 @@ import io.github.datacatering.datacaterer.core.plan.PlanProcessor import io.github.datacatering.datacaterer.core.ui.config.UiConfiguration.INSTALL_DIRECTORY import io.github.datacatering.datacaterer.core.ui.mapper.UiMapper import io.github.datacatering.datacaterer.core.ui.model.{JsonSupport, PlanRunExecution, PlanRunRequest, PlanRunRequests} -import io.github.datacatering.datacaterer.core.ui.plan.ConnectionRepository.getClass +import io.github.datacatering.datacaterer.core.ui.plan.ConnectionRepository.{LOGGER, getClass} import io.github.datacatering.datacaterer.core.ui.plan.PlanResponseHandler.{KO, OK, Response} import org.apache.log4j.Logger import org.joda.time.{DateTime, Seconds} @@ -117,6 +117,7 @@ object PlanRepository extends JsonSupport { } private def savePlanRunExecution(planRunRequest: PlanRunRequest, planRunExecution: PlanRunExecution): Unit = { + LOGGER.debug(s"Saving plan run execution details, plan-name=${planRunRequest.name}, plan-run-id=${planRunRequest.id}") savePlan(planRunRequest) try { Path.of(executionSaveFolder).toFile.mkdirs() @@ -128,6 +129,7 @@ object PlanRepository extends JsonSupport { } private def savePlan(planRunRequest: PlanRunRequest): Unit = { + LOGGER.debug(s"Saving plan details, plan-name=${planRunRequest.name}") try { Path.of(planSaveFolder).toFile.mkdirs() val planFile = Path.of(s"$planSaveFolder/${planRunRequest.name}.json") @@ -138,6 +140,7 @@ object PlanRepository extends JsonSupport { } private def getPlans: PlanRunRequests = { + LOGGER.debug("Getting all plans") val planFolder = Path.of(planSaveFolder) if (!planFolder.toFile.exists()) planFolder.toFile.mkdirs() val plans = Files.list(planFolder) @@ -152,12 +155,14 @@ object PlanRepository extends JsonSupport { } private def getPlan(name: String): PlanRunRequest = { + LOGGER.debug(s"Getting plan details, plan-name=$name") val planFile = Path.of(s"$planSaveFolder/$name.json") Files.readString(planFile).parseJson.convertTo[PlanRunRequest] } private def updatePlanRunExecution(planRunExecution: PlanRunExecution, status: String, failedReason: Option[String] = None, results: Option[PlanRunResults] = None): Unit = { + LOGGER.debug(s"Update plan execution, plan-name=${planRunExecution.name}, plan-run-id=${planRunExecution.id}, status=$status") val executionFile = Path.of(s"$executionSaveFolder/${planRunExecution.id}.csv") val cleanFailReason = failedReason.map(s => s.replaceAll("\n", " ")) val generationSummary = results.map(res => res.generationResults.map(_.summarise)).getOrElse(List()) @@ -173,12 +178,14 @@ object PlanRepository extends JsonSupport { } private def getPlanRunStatus(id: String): PlanRunExecution = { + LOGGER.debug(s"Getting current plan status, plan-run-id=$id") val executionFile = Path.of(s"$executionSaveFolder/$id.csv") val latestUpdate = Files.readAllLines(executionFile).asScala.last PlanRunExecution.fromString(latestUpdate) } private def getAllPlanExecutions: PlanRunExecutionDetails = { + LOGGER.debug("Getting all plan executions") val executionPath = Path.of(executionSaveFolder) if (!executionPath.toFile.exists()) executionPath.toFile.mkdirs() val allPlanRunExecutions = Files.list(executionPath) diff --git a/misc/jpackage/jpackage-linux.cfg b/misc/jpackage/jpackage-linux.cfg index 6a9fce4a..2a158f06 100644 --- a/misc/jpackage/jpackage-linux.cfg +++ b/misc/jpackage/jpackage-linux.cfg @@ -1,2 +1,3 @@ --type deb ---icon misc/banner/data_catering_transparent.png \ No newline at end of file +--icon misc/banner/data_catering_transparent.png +--java-options "-DLOG_FOLDER=/opt/DataCaterer/log" \ No newline at end of file diff --git a/misc/jpackage/jpackage-mac.cfg b/misc/jpackage/jpackage-mac.cfg index f2b730bb..67252b66 100644 --- a/misc/jpackage/jpackage-mac.cfg +++ b/misc/jpackage/jpackage-mac.cfg @@ -1,2 +1,3 @@ --type dmg ---icon misc/banner/data_catering_transparent.icns \ No newline at end of file +--icon misc/banner/data_catering_transparent.icns +--java-options "-DLOG_FOLDER=/Applications/DataCaterer.app/Logs" \ No newline at end of file diff --git a/misc/jpackage/jpackage-windows.cfg b/misc/jpackage/jpackage-windows.cfg index 4e3387f5..d3562f2b 100644 --- a/misc/jpackage/jpackage-windows.cfg +++ b/misc/jpackage/jpackage-windows.cfg @@ -1,2 +1,3 @@ ---type exe ---icon misc/banner/data_catering_transparent.ico \ No newline at end of file +--type msi +--icon misc/banner/data_catering_transparent.ico +--java-options "-DLOG_FOLDER=/Program Files/DataCaterer/log" \ No newline at end of file diff --git a/misc/jpackage/jpackage.cfg b/misc/jpackage/jpackage.cfg index 4e8f66df..fc702a15 100644 --- a/misc/jpackage/jpackage.cfg +++ b/misc/jpackage/jpackage.cfg @@ -1,7 +1,7 @@ --name "DataCaterer" --vendor "Data Catering" --about-url https://data.catering ---description "Data generation and validation tool" +--description "Data Caterer: Data generation and validation tool" --input app/build/libs/ --app-version 1.0.0 --main-jar app-0.7.0-all.jar