clean up log files when job is complete

README.md

@@ -58,7 +58,7 @@ Note: the term container here is used to be consistent with LogDNA's terminology
 ```
 [gcp]  # Google Cloud Project settings
 ProjectID = "gcpproj" # name of GCP project
-CredentialsFile = "gcp_credentials.json" # path to the credentials file downloaded from GCP
+CredentialsFile = "gcp_credentials.json" # relative or absolute path to the credentials file downloaded from GCP
 Bucket = "logdna_to_bq" # name of bucket in Google Cloud Storage to save results for ingestion into BigQuery, bucket will need to be created before first run
 Dataset = "logdna" # BigQuery dataset
 TemplateTable = "logdna" # currently DNAQuery uses a template table. More details below.

main.go

@@ -93,13 +93,22 @@ func readConfig(path string, cfg *Config) {
 	}
 }
 
-func setupDirectories(cfg *Config) {
+func setupDirectory(cfg *Config) {
 	err := os.MkdirAll(cfg.Storage.LogDirectory, os.ModePerm)
 	if err != nil {
 		log.Fatalf("Can't create log dir (%s): %s\n", cfg.Storage.LogDirectory, err.Error())
 	}
 }
 
+func cleanupFiles(path ...string) {
+	for _, f := range path {
+		err := os.Remove(f)
+		if err != nil {
+			log.Printf("Unable to delete file (%s): %s", f, err.Error())
+		}
+	}
+}
+
 func readLine(path string, cfg *Config) chan [2]string {
 	ch := make(chan [2]string, 50)
 	go func(path string, ch chan [2]string, cfg *Config) {
@@ -366,18 +375,21 @@ func main() {
 		fmt.Printf("Usage: %s date\n", os.Args[0])
 		return
 	}
+	dateToProcess := os.Args[1]
+
+	// load config
 	cfg := &Config{}
 	readConfig("dnaquery.toml", cfg)
 	cfg.compileRegexes()
-	dateToProcess := os.Args[1]
-	outFile := "results_" + dateToProcess + ".csv"
-	outPath := filepath.Join(cfg.Storage.LogDirectory, outFile)
 
-	setupDirectories(cfg)
+	setupDirectory(cfg)
 
 	logName := getLogfile(cfg, dateToProcess)
 
 	ch := readLine(logName, cfg)
+
+	outFile := "results_" + dateToProcess + ".csv"
+	outPath := filepath.Join(cfg.Storage.LogDirectory, outFile)
 	processLine(outPath, ch, cfg)
 
 	gcsObject := dateToProcess + "_results.csv"
@@ -386,4 +398,6 @@ func main() {
 		log.Fatalln("Error uploading to GCS:", err.Error())
 	}
 	loadInBQ(gcsObject, dateToProcess, cfg)
+
+	cleanupFiles(logName, outPath)
 }