Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

The plugin log name and checkpoint name support the logtail mode. #1901

Merged
merged 17 commits into from
Nov 20, 2024
42 changes: 41 additions & 1 deletion core/app_config/AppConfig.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,46 @@ string GetPipelineConfigDir() {
}
}

string GetPluginLogName() {
if (BOOL_FLAG(logtail_mode)) {
return "logtail_plugin.LOG";
} else {
return "go_plugin.LOG";
}
}

std::string GetVersionTag() {
if (BOOL_FLAG(logtail_mode)) {
return "logtail_version";
} else {
return "loongcollector_version";
}
}

std::string GetGoPluginCheckpoint() {
if (BOOL_FLAG(logtail_mode)) {
return "checkpoint";
} else {
return "go_plugin_checkpoint";
}
}

std::string GetAgentName() {
if (BOOL_FLAG(logtail_mode)) {
return "ilogtail";
} else {
return "loongcollector";
}
}

std::string GetMonitorInfoFileName() {
if (BOOL_FLAG(logtail_mode)) {
return "logtail_monitor_info";
} else {
return "loongcollector_monitor_info";
}
}

AppConfig::AppConfig() {
LOG_INFO(sLogger, ("AppConfig AppConfig", "success"));
SetIlogtailConfigJson("");
Expand Down Expand Up @@ -1667,7 +1707,7 @@ void AppConfig::SetLoongcollectorConfDir(const std::string& dirPath) {
// = AbsolutePath(STRING_FLAG(ilogtail_local_yaml_config_dir), mLogtailSysConfDir) + PATH_SEPARATOR;
quzard marked this conversation as resolved.
Show resolved Hide resolved
// mUserRemoteYamlConfigDirPath
// = AbsolutePath(STRING_FLAG(ilogtail_remote_yaml_config_dir), mLogtailSysConfDir) + PATH_SEPARATOR;
LOG_INFO(sLogger, ("set loongcollector conf dir", mLoongcollectorConfDir));
LOG_INFO(sLogger, ("set " + GetAgentName() + " conf dir", mLoongcollectorConfDir));
}

bool AppConfig::IsHostPathMatchBlacklist(const string& dirPath) const {
Expand Down
5 changes: 5 additions & 0 deletions core/app_config/AppConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ std::string GetSendBufferFileNamePrefix();
std::string GetLegacyUserLocalConfigFilePath();
std::string GetExactlyOnceCheckpoint();
std::string GetPipelineConfigDir();
std::string GetPluginLogName();
std::string GetVersionTag();
std::string GetGoPluginCheckpoint();
std::string GetAgentName();
std::string GetMonitorInfoFileName();

template <class T>
class DoubleBuffer {
Expand Down
4 changes: 2 additions & 2 deletions core/application/Application.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -174,9 +174,9 @@ void Application::Init() {
appInfoJson["UUID"] = Json::Value(Application::GetInstance()->GetUUID());
appInfoJson["instance_id"] = Json::Value(Application::GetInstance()->GetInstanceId());
#ifdef __ENTERPRISE__
quzard marked this conversation as resolved.
Show resolved Hide resolved
appInfoJson["loongcollector_version"] = Json::Value(ILOGTAIL_VERSION);
appInfoJson[GetVersionTag()] = Json::Value(ILOGTAIL_VERSION);
#else
appInfoJson["loongcollector_version"] = Json::Value(string(ILOGTAIL_VERSION) + " Community Edition");
appInfoJson[GetVersionTag()] = Json::Value(string(ILOGTAIL_VERSION) + " Community Edition");
appInfoJson["git_hash"] = Json::Value(ILOGTAIL_GIT_HASH);
appInfoJson["build_date"] = Json::Value(ILOGTAIL_BUILD_DATE);
#endif
Expand Down
2 changes: 1 addition & 1 deletion core/common/LogtailCommonFlags.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@
// app config
DEFINE_FLAG_STRING(ilogtail_config,
"set dataserver & configserver address; (optional)set cpu,mem,bufflerfile,buffermap and etc.",
"loongcollector_config.json");
"ilogtail_config.json");
DEFINE_FLAG_BOOL(enable_full_drain_mode, "", false);
DEFINE_FLAG_INT32(cpu_limit_num, "cpu violate limit num before shutdown", 10);
DEFINE_FLAG_INT32(mem_limit_num, "memory violate limit num before shutdown", 10);
Expand Down
3 changes: 3 additions & 0 deletions core/go_pipeline/LogtailPlugin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ LogtailPlugin::LogtailPlugin() {
mPluginCfg["LoongcollectorConfDir"] = AppConfig::GetInstance()->GetLoongcollectorConfDir();
mPluginCfg["LoongcollectorLogDir"] = GetAgentLogDir();
mPluginCfg["LoongcollectorDataDir"] = GetAgentDataDir();
mPluginCfg["LoongcollectorPluginLogName"] = GetPluginLogName();
mPluginCfg["LoongcollectorVersionTag"] = GetVersionTag();
mPluginCfg["LoongcollectorCheckPointFile"] = GetGoPluginCheckpoint();
mPluginCfg["LoongcollectorThirdPartyDir"] = GetAgentThirdPartyDir();
mPluginCfg["HostIP"] = LogFileProfiler::mIpAddr;
mPluginCfg["Hostname"] = LogFileProfiler::mHostname;
Expand Down
2 changes: 1 addition & 1 deletion core/monitor/LogFileProfiler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ bool LogFileProfiler::GetProfileData(LogGroup& logGroup, LogStoreStatistic* stat
contentPtr->set_value(statistic->mHostLogPath);
}
contentPtr = logPtr->add_contents();
contentPtr->set_key("loongcollector_version");
contentPtr->set_key(GetVersionTag());
contentPtr->set_value(ILOGTAIL_VERSION);
contentPtr = logPtr->add_contents();
contentPtr->set_key("source_ip");
Expand Down
2 changes: 1 addition & 1 deletion core/monitor/Monitor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,7 @@ void LogtailMonitor::DumpToLocal(const sls_logs::LogGroup& logGroup) {
}

bool LogtailMonitor::DumpMonitorInfo(time_t monitorTime) {
string path = GetAgentLogDir() + "loongcollector_monitor_info";
string path = GetAgentLogDir() + GetMonitorInfoFileName();
ofstream outfile(path.c_str(), ofstream::app);
if (!outfile)
return false;
Expand Down
5 changes: 3 additions & 2 deletions core/pipeline/Pipeline.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <cstdint>
#include <utility>

#include "app_config/AppConfig.h"
#include "common/Flags.h"
#include "common/ParamExtractor.h"
#include "go_pipeline/LogtailPlugin.h"
Expand Down Expand Up @@ -498,7 +499,7 @@ bool Pipeline::LoadGoPipelines() const {
mContext.GetRegion(),
mContext.GetLogstoreKey())) {
LOG_ERROR(mContext.GetLogger(),
("failed to init pipeline", "Go pipeline is invalid, see go_plugin.LOG for detail")(
("failed to init pipeline", "Go pipeline is invalid, see " + GetPluginLogName() + " for detail")(
"Go pipeline num", "2")("Go pipeline content", content)("config", mName));
AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM,
"Go pipeline is invalid, content: " + content + ", config: " + mName,
Expand All @@ -517,7 +518,7 @@ bool Pipeline::LoadGoPipelines() const {
mContext.GetRegion(),
mContext.GetLogstoreKey())) {
LOG_ERROR(mContext.GetLogger(),
("failed to init pipeline", "Go pipeline is invalid, see go_plugin.LOG for detail")(
("failed to init pipeline", "Go pipeline is invalid, see " + GetPluginLogName() + " for detail")(
"Go pipeline num", "1")("Go pipeline content", content)("config", mName));
AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM,
"Go pipeline is invalid, content: " + content + ", config: " + mName,
Expand Down
33 changes: 21 additions & 12 deletions pkg/config/global_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ type GlobalConfig struct {
LoongcollectorDebugDir string
// Directory to store loongcollector third party data.
LoongcollectorThirdPartyDir string
// Log name of loongcollector plugin.
LoongcollectorPluginLogName string
// Tag of loongcollector version.
LoongcollectorVersionTag string
// Checkpoint file name of loongcollector plugin.
LoongcollectorCheckPointFile string
// Network identification from loongcollector.
HostIP string
Hostname string
Expand All @@ -59,18 +65,21 @@ var UserAgent = fmt.Sprintf("ilogtail/%v (%v)", BaseVersion, runtime.GOOS) // se

func newGlobalConfig() (cfg GlobalConfig) {
cfg = GlobalConfig{
InputMaxFirstCollectDelayMs: 10000, // 10s
InputIntervalMs: 1000, // 1s
AggregatIntervalMs: 3000,
FlushIntervalMs: 3000,
DefaultLogQueueSize: 1000,
DefaultLogGroupQueueSize: 4,
LoongcollectorConfDir: "./conf/",
LoongcollectorLogDir: "./log/",
LoongcollectorDataDir: "./data/",
LoongcollectorDebugDir: "./debug/",
LoongcollectorThirdPartyDir: "./thirdparty/",
DelayStopSec: 300,
InputMaxFirstCollectDelayMs: 10000, // 10s
InputIntervalMs: 1000, // 1s
AggregatIntervalMs: 3000,
FlushIntervalMs: 3000,
DefaultLogQueueSize: 1000,
DefaultLogGroupQueueSize: 4,
LoongcollectorConfDir: "./conf/",
LoongcollectorLogDir: "./log/",
LoongcollectorPluginLogName: "go_plugin.LOG",
LoongcollectorVersionTag: "loongcollector_version",
LoongcollectorCheckPointFile: "go_plugin_checkpoint",
LoongcollectorDataDir: "./data/",
LoongcollectorDebugDir: "./debug/",
LoongcollectorThirdPartyDir: "./thirdparty/",
DelayStopSec: 300,
}
return
}
6 changes: 3 additions & 3 deletions pkg/logger/logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ const (
asyncPattern = `
<seelog type="asynctimer" asyncinterval="500000" minlevel="%s" >
<outputs formatid="common">
<rollingfile type="size" filename="%sgo_plugin.LOG" maxsize="20000000" maxrolls="10"/>
<rollingfile type="size" filename="%s%s" maxsize="20000000" maxrolls="10"/>
%s
%s
</outputs>
Expand All @@ -54,7 +54,7 @@ const (
syncPattern = `
<seelog type="sync" minlevel="%s" >
<outputs formatid="common">
<rollingfile type="size" filename="%sgo_plugin.LOG" maxsize="20000000" maxrolls="10"/>
<rollingfile type="size" filename="%s%s" maxsize="20000000" maxrolls="10"/>
%s
%s
</outputs>
Expand Down Expand Up @@ -325,7 +325,7 @@ func generateDefaultConfig() string {
if memoryReceiverFlag {
memoryReceiverFlagStr = "<custom name=\"memory\" />"
}
return fmt.Sprintf(template, levelFlag, config.LoongcollectorGlobalConfig.LoongcollectorLogDir, consoleStr, memoryReceiverFlagStr)
return fmt.Sprintf(template, levelFlag, config.LoongcollectorGlobalConfig.LoongcollectorLogDir, config.LoongcollectorGlobalConfig.LoongcollectorPluginLogName, consoleStr, memoryReceiverFlagStr)
}

// Close the logger and recover the stdout and stderr
Expand Down
4 changes: 2 additions & 2 deletions pkg/logger/logger_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ func init() {

func clean() {
_ = os.Remove(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorConfDir, "plugin_logger.xml"))
_ = os.Remove(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, "go_plugin.LOG"))
_ = os.Remove(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, config.LoongcollectorGlobalConfig.LoongcollectorPluginLogName))
}

func readLog(index int) string {
bytes, _ := os.ReadFile(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, "go_plugin.LOG"))
bytes, _ := os.ReadFile(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, config.LoongcollectorGlobalConfig.LoongcollectorPluginLogName))
logs := strings.Split(string(bytes), "\n")
if index > len(logs)-1 {
return ""
Expand Down
Binary file modified pkg/logtail/libGoPluginAdapter.so
Binary file not shown.
8 changes: 6 additions & 2 deletions pluginmanager/checkpoint_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
"github.com/alibaba/ilogtail/pkg/util"
)

var CheckPointFile = flag.String("CheckPointFile", "go_plugin_checkpoint", "checkpoint file name, base dir(binary dir)")
var CheckPointFile = flag.String("CheckPointFile", "", "checkpoint file name, base dir(binary dir)")
var CheckPointCleanInterval = flag.Int("CheckPointCleanInterval", 600, "checkpoint clean interval, second")
var MaxCleanItemPerInterval = flag.Int("MaxCleanItemPerInterval", 1000, "max clean items per interval")

Expand Down Expand Up @@ -89,7 +89,11 @@ func (p *checkPointManager) Init() error {
pathExist, err := util.PathExists(logtailDataDir)
var dbPath string
if err == nil && pathExist {
dbPath = filepath.Join(logtailDataDir, *CheckPointFile)
if *CheckPointFile != "" {
dbPath = filepath.Join(logtailDataDir, *CheckPointFile)
} else {
dbPath = filepath.Join(logtailDataDir, config.LoongcollectorGlobalConfig.LoongcollectorCheckPointFile)
}
} else {
// c++程序如果这个目录创建失败会直接exit,所以这里一般应该不会走进来
logger.Error(context.Background(), "CHECKPOINT_ALARM", "logtailDataDir not exist", logtailDataDir, "err", err)
Expand Down
4 changes: 2 additions & 2 deletions pluginmanager/plugin_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ var alarmConfigJSON = `{
"DefaultLogGroupQueueSize": 4,
"Tags" : {
"base_version" : "` + config.BaseVersion + `",
"loongcollector_version" : "` + config.BaseVersion + `"
"` + config.LoongcollectorGlobalConfig.LoongcollectorVersionTag + `" : "` + config.BaseVersion + `"
}
},
"inputs" : [
Expand All @@ -77,7 +77,7 @@ var containerConfigJSON = `{
"DefaultLogGroupQueueSize": 4,
"Tags" : {
"base_version" : "` + config.BaseVersion + `",
"loongcollector_version" : "` + config.BaseVersion + `"
"` + config.LoongcollectorGlobalConfig.LoongcollectorVersionTag + `" : "` + config.BaseVersion + `"
}
},
"inputs" : [
Expand Down