Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support rolling log files base on file size and time #811

Closed
wants to merge 1 commit into from
Closed

Support rolling log files base on file size and time #811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/glog/logging.h.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,13 @@ DECLARE_string(vmodule); // also in vlog_is_on.cc
// Sets the maximum log file size (in MB).
DECLARE_uint32(max_log_size);

// Set maximum log file num after rolling
DECLARE_uint32(max_logfile_num);

// Set log file rolling policy, support size-based and time-based
// The available values are size, day and hour
DECLARE_string(log_rolling_policy);

// Sets whether to avoid logging to the disk if the disk is full.
DECLARE_bool(stop_logging_if_full_disk);

Expand Down
217 changes: 178 additions & 39 deletions src/logging.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,12 @@
#ifdef HAVE_SYSLOG_H
# include <syslog.h>
#endif
#include <vector>
#include <cerrno> // for errno
#include <sstream>
#include <cctype> // for std::isspace
#include <cerrno> // for errno
#include <regex>
#include <cctype> // for std::isspace
#include <set>
#include <sstream>
#include <vector>
#ifdef GLOG_OS_WINDOWS
#include "windows/dirent.h"
#else
Expand All @@ -81,15 +82,16 @@
#include <android/log.h>
#endif

using std::string;
using std::vector;
using std::setw;
using std::setfill;
using std::hex;
using std::dec;
using std::hex;
using std::min;
using std::multiset;
using std::ostream;
using std::ostringstream;
using std::setfill;
using std::setw;
using std::string;
using std::vector;

using std::FILE;
using std::fwrite;
Expand Down Expand Up @@ -201,6 +203,15 @@ GLOG_DEFINE_uint32(max_log_size, 1800,
"approx. maximum log file size (in MB). A value of 0 will "
"be silently overridden to 1.");

GLOG_DEFINE_string(
log_rolling_policy, "size",
"log file rolling policy, support size-based and time-based."
" The available values are size, day and hour, if use time-based policy,"
" the maximum log file size also control by max_log_size");

GLOG_DEFINE_uint32(max_logfile_num, 10,
"maximum history log file num after rolling");

GLOG_DEFINE_bool(stop_logging_if_full_disk, false,
"Stop attempting to log to disk if the disk is full.");

Expand Down Expand Up @@ -433,6 +444,12 @@ void* custom_prefix_callback_data = nullptr;

namespace {

struct Filetime {
std::string name;
std::time_t time;
bool operator<(const Filetime& o) const { return time < o.time; }
};

// Encapsulates all file-system related state
class LogFileObject : public base::Logger {
public:
Expand Down Expand Up @@ -463,6 +480,10 @@ class LogFileObject : public base::Logger {
// acquiring lock_.
void FlushUnlocked();

// check the logfiles rolling by size or date, and remve the oldest ones
// according to FLAGS_max_logfile_num
void CheckHistoryFileNum();
yangzhg marked this conversation as resolved.
Show resolved Hide resolved

private:
static const uint32 kRolloverAttemptFrequency = 0x20;

Expand All @@ -476,14 +497,24 @@ class LogFileObject : public base::Logger {
uint32 bytes_since_flush_{0};
uint32 dropped_mem_length_{0};
uint32 file_length_{0};
unsigned int rollover_attempt_;
uint32 rollover_attempt_;
int64 next_flush_time_{0}; // cycle count at which to flush log
WallTime start_time_;
std::multiset<Filetime> file_list_;
Copy link
Collaborator

@sergiud sergiud Oct 7, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why a std::multiset is needed here. Can a file be associated with multiple timestamps? I don't see how this is possible. Either way, a multiset would be rather memory inefficient here, since the same file name would be potentially stored multiple times.

Please drop the Filetime structure and use an associative container. I guess a std::map is sufficient here.

bool initialized_;
std::tm tm_time_;

// Actually create a logfile using the value of base_filename_ and the
// optional argument time_pid_string
// REQUIRES: lock_ is held
bool CreateLogfile(const string& time_pid_string);

// Actually create a logfile using full file name and flags
bool CreateLogfileInternal(const char* filename, int flags);

// Check if log file is getting too big or match the date time condition, If
// so, rollover.
bool CheckNeedRollLogFiles(time_t timestamp);
};

// Encapsulate all log cleaner related states
Expand Down Expand Up @@ -989,7 +1020,8 @@ LogFileObject::LogFileObject(LogSeverity severity, const char* base_filename)

rollover_attempt_(kRolloverAttemptFrequency - 1),

start_time_(WallTime_Now()) {
start_time_(WallTime_Now()),
initialized_(false) {
assert(severity >= 0);
assert(severity < NUM_SEVERITIES);
}
Expand Down Expand Up @@ -1050,19 +1082,7 @@ void LogFileObject::FlushUnlocked(){
next_flush_time_ = CycleClock_Now() + UsecToCycles(next);
}

bool LogFileObject::CreateLogfile(const string& time_pid_string) {
string string_filename = base_filename_;
if (FLAGS_timestamp_in_logfile_name) {
string_filename += time_pid_string;
}
string_filename += filename_extension_;
const char* filename = string_filename.c_str();
//only write to files, create if non-existant.
int flags = O_WRONLY | O_CREAT;
if (FLAGS_timestamp_in_logfile_name) {
//demand that the file is unique for our timestamp (fail if it exists).
flags = flags | O_EXCL;
}
bool LogFileObject::CreateLogfileInternal(const char* filename, int flags) {
int fd = open(filename, flags, static_cast<mode_t>(FLAGS_logfile_mode));
if (fd == -1) return false;
#ifdef HAVE_FCNTL
Expand All @@ -1077,8 +1097,7 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) {
// This will work after a fork as it is not inherited (not stored in the fd).
// Lock will not be lost because the file is opened with exclusive lock
// (write) and we will never read from it inside the process.
// TODO: windows implementation of this (as flock is not available on
// mingw).
// TODO windows implementation of this (as flock is not available on mingw).
static struct flock w_lock;

w_lock.l_type = F_WRLCK;
Expand All @@ -1095,11 +1114,11 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) {

//fdopen in append mode so if the file exists it will fseek to the end
file_ = fdopen(fd, "a"); // Make a FILE*.
if (file_ == nullptr) { // Man, we're screwed!
if (file_ == nullptr) { // Man, we're screwed!
close(fd);
if (FLAGS_timestamp_in_logfile_name) {
unlink(filename); // Erase the half-baked evidence: an unusable log file, only if we just created it.
}
}
return false;
}
#ifdef GLOG_OS_WINDOWS
Expand All @@ -1111,6 +1130,29 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) {
}
}
#endif
return true;
}

bool LogFileObject::CreateLogfile(const string& time_pid_string) {
string string_filename = base_filename_;
if (FLAGS_timestamp_in_logfile_name) {
string_filename += time_pid_string;
}
string_filename += filename_extension_;
const char* filename = string_filename.c_str();
// only write to files, create if non-existant.
int flags = O_WRONLY | O_CREAT;
if (FLAGS_timestamp_in_logfile_name) {
// demand that the file is unique for our timestamp (fail if it exists).
flags = flags | O_EXCL;
}
bool success = CreateLogfileInternal(filename, flags);
if (!success) {
return false;
}
Filetime ft;
ft.name = string_filename;
file_list_.insert(ft);
// We try to create a symlink called <program_name>.<severity>,
// which is easier to use. (Every time we create a new logfile,
// we destroy the old symlink and create a new one, so it always
Expand Down Expand Up @@ -1152,6 +1194,78 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) {
return true; // Everything worked
}

void LogFileObject::CheckHistoryFileNum() {
struct dirent* entry;
DIR* dp;

const vector<string>& log_dirs = GetLoggingDirectories();
if (log_dirs.empty()) return;

// list file in log dir
dp = opendir(log_dirs[0].c_str());
if (dp == nullptr) {
fprintf(stderr, "open log dir %s fail\n", log_dirs[0].c_str());
return;
}

file_list_.clear();
while ((entry = readdir(dp)) != nullptr) {
if (DT_DIR == entry->d_type || DT_LNK == entry->d_type) {
continue;
}
std::string filename = entry->d_name;

if (filename.find(symlink_basename_ + '.' + LogSeverityNames[severity_]) ==
0) {
std::string filepath = log_dirs[0] + "/" + filename;

struct stat fstat;
if (::stat(filepath.c_str(), &fstat) < 0) {
fprintf(stderr, "state %s fail\n", filepath.c_str());
closedir(dp);
return;
}

Filetime file_time;
file_time.time = fstat.st_mtime;
file_time.name = filepath;
file_list_.insert(file_time);
}
}
closedir(dp);

while (FLAGS_max_logfile_num > 0 &&
file_list_.size() >= FLAGS_max_logfile_num) {
unlink(file_list_.begin()->name.c_str());
file_list_.erase(file_list_.begin());
}
}

bool LogFileObject::CheckNeedRollLogFiles(time_t timestamp) {
bool roll_needed = false;
struct ::tm tm_time;
if (FLAGS_log_rolling_policy == "day") {
localtime_r(&timestamp, &tm_time);
if (tm_time.tm_year != tm_time_.tm_year ||
tm_time.tm_mon != tm_time_.tm_mon ||
tm_time.tm_mday != tm_time_.tm_mday) {
roll_needed = true;
}
} else if (FLAGS_log_rolling_policy == "hour") {
localtime_r(&timestamp, &tm_time);
if (tm_time.tm_year != tm_time_.tm_year ||
tm_time.tm_mon != tm_time_.tm_mon ||
tm_time.tm_mday != tm_time_.tm_mday ||
tm_time.tm_hour != tm_time_.tm_hour) {
roll_needed = true;
}
} else if (file_length_ >> 20U >= MaxLogSize() || PidHasChanged()) {
roll_needed = true;
}

return roll_needed;
}

void LogFileObject::Write(bool force_flush,
time_t timestamp,
const char* message,
Expand All @@ -1162,13 +1276,27 @@ void LogFileObject::Write(bool force_flush,
if (base_filename_selected_ && base_filename_.empty()) {
return;
}

if (file_length_ >> 20U >= MaxLogSize() || PidHasChanged()) {
bool roll_needed = CheckNeedRollLogFiles(timestamp);
if (roll_needed) {
if (file_ != nullptr) fclose(file_);
file_ = nullptr;
file_length_ = bytes_since_flush_ = dropped_mem_length_ = 0;
rollover_attempt_ = kRolloverAttemptFrequency - 1;
}
if ((file_ == nullptr) && (!initialized_) &&
(FLAGS_log_rolling_policy == "size")) {
CheckHistoryFileNum();
if (!file_list_.empty()) {
std::multiset<Filetime>::iterator it = file_list_.end();
it--;
const char* filename = it->name.c_str();
int flags = O_WRONLY | O_CREAT | O_APPEND;
bool success = CreateLogfileInternal(filename, flags);
if (success) {
initialized_ = true;
}
}
}

// If there's no destination file, make one before outputting
if (file_ == nullptr) {
Expand All @@ -1178,6 +1306,16 @@ void LogFileObject::Write(bool force_flush,
if (++rollover_attempt_ != kRolloverAttemptFrequency) return;
rollover_attempt_ = 0;

if (!initialized_) {
CheckHistoryFileNum();
initialized_ = true;
} else {
while (FLAGS_max_logfile_num > 0 &&
file_list_.size() >= FLAGS_max_logfile_num) {
unlink(file_list_.begin()->name.c_str());
file_list_.erase(file_list_.begin());
}
}
struct ::tm tm_time;
if (FLAGS_log_utc_time) {
gmtime_r(&timestamp, &tm_time);
Expand All @@ -1188,15 +1326,16 @@ void LogFileObject::Write(bool force_flush,
// The logfile's filename will have the date/time & pid in it
ostringstream time_pid_stream;
time_pid_stream.fill('0');
time_pid_stream << 1900+tm_time.tm_year
<< setw(2) << 1+tm_time.tm_mon
<< setw(2) << tm_time.tm_mday
<< '-'
<< setw(2) << tm_time.tm_hour
<< setw(2) << tm_time.tm_min
<< setw(2) << tm_time.tm_sec
<< '.'
<< GetMainThreadPid();
time_pid_stream << 1900 + tm_time.tm_year << setw(2) << 1 + tm_time.tm_mon
<< setw(2) << tm_time.tm_mday;
if (FLAGS_log_rolling_policy == "hour") {
time_pid_stream << setw(2) << tm_time.tm_hour;
} else if (FLAGS_log_rolling_policy != "day") {
time_pid_stream << '-' << setw(2) << tm_time.tm_hour << setw(2)
<< tm_time.tm_min << setw(2) << tm_time.tm_sec;
}
time_pid_stream << '.' << GetMainThreadPid();
tm_time_ = tm_time;
const string& time_pid_string = time_pid_stream.str();

if (base_filename_selected_) {
Expand Down
Loading