Skip to content

Commit

Permalink
Option to clear old logs based on total number or number of days
Browse files Browse the repository at this point in the history
  • Loading branch information
rcastells committed Aug 12, 2017
1 parent 5f76b45 commit ce337fe
Show file tree
Hide file tree
Showing 5 changed files with 284 additions and 20 deletions.
249 changes: 234 additions & 15 deletions FileStreamRotator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

/*!
* FileStreamRotator
* Copyright(c) 2012 Holiday Extras.
* Copyright(c) 2012-2017 Holiday Extras.
* Copyright(c) 2017 Roger C.
* MIT Licensed
*/

Expand All @@ -12,6 +13,8 @@
var fs = require('fs');
var path = require('path');
var moment = require('moment');
var crypto = require('crypto');

var EventEmitter = require('events');

/**
Expand All @@ -21,10 +24,25 @@ var EventEmitter = require('events');
*
* Options:
*
* - `filename` Filename including full path used by the stream
* - `frequency` How often to rotate. At present only 'daily' and 'test' are available. 'test' rotates every minute.
* If frequency is set to none of the above, a YYYYMMDD string will be added to the end of the filename.
* - `verbose` If set, it will log to STDOUT when it rotates files and name of log file. Default is TRUE.
* - `filename` Filename including full path used by the stream
*
* - `frequency` How often to rotate. Options are 'daily', 'custom' and 'test'. 'test' rotates every minute.
* If frequency is set to none of the above, a YYYYMMDD string will be added to the end of the filename.
*
* - `verbose` If set, it will log to STDOUT when it rotates files and name of log file. Default is TRUE.
*
* - `date_format` Format as used in moment.js http://momentjs.com/docs/#/displaying/format/. The result is used to replace
* the '%DATE%' placeholder in the filename.
* If using 'custom' frequency, it is used to trigger file change when the string representation changes.
*
* - `size` Max size of the file after which it will rotate. It can be combined with frequency or date format.
* The size units are 'k', 'm' and 'g'. Units need to directly follow a number e.g. 1g, 100m, 20k.
*
* - `max_logs` Max number of logs to keep. If not set, it won't remove past logs. It uses its own log audit file
* to keep track of the log files in a json format. It won't delete any file not contained in it.
* It can be a number of files or number of days. If using days, add 'd' as the suffix.
*
* - `audit_file` Location to store the log audit file. If not set, it will be stored in the root of the application.
*
* To use with Express / Connect, use as below.
*
Expand All @@ -42,6 +60,14 @@ module.exports = FileStreamRotator;
var staticFrequency = ['daily', 'test', 'm', 'h', 'custom'];
var DATE_FORMAT = ('YYYYMMDDHHmm');


/**
* Returns frequency metadata for minute/hour rotation
* @param type
* @param num
* @returns {*}
* @private
*/
var _checkNumAndType = function (type, num) {
if (typeof num == 'number') {
switch (type) {
Expand All @@ -60,6 +86,12 @@ var _checkNumAndType = function (type, num) {
}
}

/**
* Returns frequency metadata for defined frequency
* @param freqType
* @returns {*}
* @private
*/
var _checkDailyAndTest = function (freqType) {
switch (freqType) {
case 'custom':
Expand All @@ -72,6 +104,12 @@ var _checkDailyAndTest = function (freqType) {
return false;
}


/**
* Returns frequency metadata
* @param frequency
* @returns {*}
*/
FileStreamRotator.getFrequency = function (frequency) {
var _f = frequency.toLowerCase().match(/^(\d+)([m|h])$/)
if(_f){
Expand All @@ -86,6 +124,11 @@ FileStreamRotator.getFrequency = function (frequency) {
return false;
}

/**
* Returns a number based on the option string
* @param size
* @returns {Number}
*/
FileStreamRotator.parseFileSize = function (size) {
if(size && typeof size == "string"){
var _s = size.toLowerCase().match(/^((?:0\.)?\d+)([k|m|g])$/);
Expand All @@ -103,6 +146,12 @@ FileStreamRotator.parseFileSize = function (size) {
return null;
};

/**
* Returns date string for a given format / date_format
* @param format
* @param date_format
* @returns {string}
*/
FileStreamRotator.getDate = function (format, date_format) {
date_format = date_format || DATE_FORMAT;
if (format && staticFrequency.indexOf(format.type) !== -1) {
Expand All @@ -124,20 +173,169 @@ FileStreamRotator.getDate = function (format, date_format) {
return moment().format(date_format);
}

/**
* Read audit json object from disk or return new object or null
* @param max_logs
* @param audit_file
* @param log_file
* @returns {Object} auditLogSettings
* @property {Object} auditLogSettings.keep
* @property {Boolean} auditLogSettings.keep.days
* @property {Number} auditLogSettings.keep.amount
* @property {String} auditLogSettings.auditLog
* @property {Array} auditLogSettings.files
*/
FileStreamRotator.setAuditLog = function (max_logs, audit_file, log_file){
var _rtn = null;
if(max_logs){
var use_days = max_logs.toString().substr(-1);
var _num = max_logs.toString().match(/^(\d+)/);

if(Number(_num[1]) > 0) {
var baseLog = path.dirname(log_file.replace(/%DATE%.+/,"_filename"));

try{
if(audit_file){
var full_path = path.resolve(audit_file);
_rtn = require(full_path);
}else{
var full_path = path.resolve(baseLog + "/" + ".audit.json")
_rtn = require(full_path);
}
}catch(e){
if(e.code !== "MODULE_NOT_FOUND"){
return null;
}
_rtn = {
keep: {
days: false,
amount: Number(_num[1])
},
auditLog: audit_file || baseLog + "/" + ".audit.json",
files: []
};
}

_rtn.keep = {
days: use_days === 'd',
amount: Number(_num[1])
};

}
}
return _rtn;
};

/**
* Write audit json object to disk
* @param {Object} audit
* @param {Object} audit.keep
* @param {Boolean} audit.keep.days
* @param {Number} audit.keep.amount
* @param {String} audit.auditLog
* @param {Array} audit.files
*/
FileStreamRotator.writeAuditLog = function(audit){
try{
mkDirForFile(audit.auditLog);
fs.writeFileSync(audit.auditLog, JSON.stringify(audit,null,4));
}catch(e){
console.error(new Date(),"[FileStreamRotator] Failed to store log audit at:", audit.auditLog,"Error:", e);
}
};


/**
* Removes old log file
* @param file
* @param file.hash
* @param file.name
* @param file.date
*/
function removeFile(file){
if(file.hash === crypto.createHash('md5').update(file.name + "LOG_FILE" + file.date).digest("hex")){
try{
fs.unlinkSync(file.name);
}catch(e){
console.error(new Date(), "[FileStreamRotator] Could not remove old log file: ", file.name);
}
}
}

/**
* Write audit json object to disk
* @param {String} logfile
* @param {Object} audit
* @param {Object} audit.keep
* @param {Boolean} audit.keep.days
* @param {Number} audit.keep.amount
* @param {String} audit.auditLog
* @param {Array} audit.files
*/
FileStreamRotator.addLogToAudit = function(logfile, audit){
if(audit && audit.files){
var time = Date.now();
audit.files.push({
date: time,
name: logfile,
hash: crypto.createHash('md5').update(logfile + "LOG_FILE" + time).digest("hex")
});

if(audit.keep.days){
var oldestDate = moment().subtract(audit.keep.amount,"days").valueOf();
var recentFiles = audit.files.filter(function(file){
if(file.date > oldestDate){
return true;
}
removeFile(file);
return false;
});
audit.files = recentFiles;
}else{
var filesToKeep = audit.files.splice(-audit.keep.amount);
if(audit.files.length > 0){
audit.files.filter(function(file){
removeFile(file);
return false;
})
}
audit.files = filesToKeep;
}

FileStreamRotator.writeAuditLog(audit);
}

return audit;
}

/**
*
* @param options
* @param options.filename
* @param options.frequency
* @param options.verbose
* @param options.date_format
* @param options.size
* @param options.max_logs
* @param options.audit_file
* @returns {Object} stream
*/
FileStreamRotator.getStream = function (options) {
var frequencyMetaData = null;
var curDate = null;
var self = this;

if (!options.filename) {
console.error("No filename supplied. Defaulting to STDOUT");
console.error(new Date(),"[FileStreamRotator] No filename supplied. Defaulting to STDOUT");
return process.stdout;
}

if (options.frequency) {
frequencyMetaData = self.getFrequency(options.frequency);
}

self.auditLog = self.setAuditLog(options.max_logs, options.audit_file, options.filename);

var fileSize = null;
var fileCount = 0;
var curSize = 0;
Expand All @@ -151,7 +349,9 @@ FileStreamRotator.getStream = function (options) {
dateFormat = "YYYY-MM-DD";
}
if(moment().format(dateFormat) != moment().add(2,"hours").format(dateFormat) || moment().format(dateFormat) == moment().add(1,"day").format(dateFormat)){
console.log("Changing type to custom as date format changes more often than once a day or not every day");
if(options.verbose){
console.log(new Date(),"[FileStreamRotator] Changing type to custom as date format changes more often than once a day or not every day");
}
frequencyMetaData.type = "custom";
}
}
Expand All @@ -168,7 +368,7 @@ FileStreamRotator.getStream = function (options) {
}
var verbose = (options.verbose !== undefined ? options.verbose : true);
if (verbose) {
console.log("Logging to", logfile);
console.log(new Date(),"[FileStreamRotator] Logging to: ", logfile);
}

if(fileSize){
Expand All @@ -186,14 +386,19 @@ FileStreamRotator.getStream = function (options) {
var rotateStream = fs.createWriteStream(logfile, {flags: 'a'});
if (curDate && frequencyMetaData && (staticFrequency.indexOf(frequencyMetaData.type) > -1)) {
if (verbose) {
console.log("Rotating file", frequencyMetaData.type);
console.log(new Date(),"[FileStreamRotator] Rotating file: ", frequencyMetaData.type);
}
var stream = new EventEmitter();
stream.end = function(){
rotateStream.end.apply(rotateStream,arguments);
};
BubbleEvents(rotateStream,stream);

stream.on("new",function(newLog){
self.auditLog = self.addLogToAudit(newLog,self.auditLog);

});

stream.write = (function (str, encoding) {
var newDate = this.getDate(frequencyMetaData,dateFormat);
if (newDate != curDate || (fileSize && curSize > fileSize)) {
Expand All @@ -212,7 +417,7 @@ FileStreamRotator.getStream = function (options) {
curSize = 0;

if (verbose) {
console.log("Changing logs from %s to %s", logfile, newLogfile);
console.log(new Date(),"[FileStreamRotator] Changing logs from %s to %s", logfile, newLogfile);
}
curDate = newDate;
oldFile = logfile;
Expand All @@ -235,7 +440,7 @@ FileStreamRotator.getStream = function (options) {
return stream;
} else {
if (verbose) {
console.log("File won't be rotated", options.frequency, frequencyMetaData && frequencyMetaData.type);
console.log(new Date(),"[FileStreamRotator] File won't be rotated: ", options.frequency, frequencyMetaData && frequencyMetaData.type);
}
process.nextTick(function(){
rotateStream.emit('new',logfile);
Expand All @@ -244,25 +449,39 @@ FileStreamRotator.getStream = function (options) {
}
}

/**
* Check and make parent directory
* @param pathWithFile
*/
var mkDirForFile = function(pathWithFile){
var _path = path.dirname(pathWithFile);
// var start = Date.now();
_path.split(path.sep).reduce(
function(fullPath, folder) {
fullPath += folder + path.sep;
// console.log('current path', fullPath, 'current folder', folder);
// Option to replace existsSync as deprecated. Maybe in a future release.
// try{
// var stats = fs.statSync(fullPath);
// console.log('STATS',fullPath, stats);
// }catch(e){
// fs.mkdirSync(fullPath);
// console.log("STATS ERROR",e)
// }
if (!fs.existsSync(fullPath)) {
fs.mkdirSync(fullPath);
}
return fullPath;
},
''
);
// var end = Date.now();
// console.log("Took " + (end - start) + "ms to check folders");
};


/**
* Bubbles events to the proxy
* @param emitter
* @param proxy
* @constructor
*/
var BubbleEvents = function BubbleEvents(emitter,proxy){
emitter.on('close',function(){
proxy.emit('close');
Expand Down
Loading

0 comments on commit ce337fe

Please sign in to comment.