-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
8 changed files
with
218 additions
and
5,628 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package entry | ||
|
|
||
| import "regexp" | ||
|
|
||
| const ( | ||
| DateTimeLayout = `2006-01-02 15:04:05` | ||
| DateTimePattern = `\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}` | ||
| IPv4Pattern = `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)` | ||
| IPv6Pattern = `(?:(?:(?:[0-9A-Fa-f]{1,4}:){7}(?:[0-9A-Fa-f]{1,4}|:))|(?:(?:[0-9A-Fa-f]{1,4}:){6}(?::[0-9A-Fa-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(?:(?:[0-9A-Fa-f]{1,4}:){5}(?:(?:(?::[0-9A-Fa-f]{1,4}){1,2})|:(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(?:(?:[0-9A-Fa-f]{1,4}:){4}(?:(?:(?::[0-9A-Fa-f]{1,4}){1,3})|(?:(?::[0-9A-Fa-f]{1,4})?:(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(?:(?:[0-9A-Fa-f]{1,4}:){3}(?:(?:(?::[0-9A-Fa-f]{1,4}){1,4})|(?:(?::[0-9A-Fa-f]{1,4}){0,2}:(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(?:(?:[0-9A-Fa-f]{1,4}:){2}(?:(?:(?::[0-9A-Fa-f]{1,4}){1,5})|(?:(?::[0-9A-Fa-f]{1,4}){0,3}:(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(?:(?:[0-9A-Fa-f]{1,4}:){1}(?:(?:(?::[0-9A-Fa-f]{1,4}){1,6})|(?:(?::[0-9A-Fa-f]{1,4}){0,4}:(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(?::(?:(?:(?::[0-9A-Fa-f]{1,4}){1,7})|(?:(?::[0-9A-Fa-f]{1,4}){0,5}:(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(?:%.+)?\s*` | ||
| IPPattern = IPv4Pattern + `|` + IPv6Pattern | ||
| ) | ||
|
|
||
| var ( | ||
| DateTimeRegex = regexp.MustCompile(DateTimePattern) | ||
| IPv4Regex = regexp.MustCompile(IPv4Pattern) | ||
| IPv6Regex = regexp.MustCompile(IPv6Pattern) | ||
| IPRegex = regexp.MustCompile(IPPattern) | ||
| ) | ||
|
|
||
| // LogEntry log meta data | ||
| type LogEntry struct { | ||
| // log Source 127.0.0.1 | ||
| Source string `json:"__source__"` | ||
| Tag *Tag `json:"__tag__"` | ||
| Topic *Topic `json:"__topic__"` | ||
| } | ||
|
|
||
| // Tag log tag | ||
| type Tag struct { | ||
| // Hostname iunlimit | ||
| Hostname string `json:"__hostname__,omitempty"` | ||
| // log storage Path /var/log/exec/file.log | ||
| Path string `json:"__path__,omitempty"` | ||
| } | ||
|
|
||
| // Topic log topic | ||
| type Topic struct { | ||
| // log Content (with `Event Time`) | ||
| Content string `json:"content"` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package entry | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
| ) | ||
|
|
||
| func Test_Regx(t *testing.T) { | ||
| content := "[PERP] [DEBUG] [2024-03-19 18:00:20]: Wait NTQQ startup: dial tcp 127.0.0.1:8000: connect: connection refused" | ||
| fmt.Println(DateTimeRegex.FindString(content)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| package entry | ||
|
|
||
| import ( | ||
| "github.com/apache/beam/sdks/v2/go/pkg/beam" | ||
| "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime" | ||
| ) | ||
|
|
||
| func Init() { | ||
| parser = NewDateTimeRegexParser() | ||
| } | ||
|
|
||
| // ParseLogEntryDoFn parse log str to LogEntry | ||
| func ParseLogEntryDoFn(element string, emit func(*LogEntry)) { | ||
| entry := &LogEntry{Topic: &Topic{Content: element}} | ||
| emit(entry) | ||
| } | ||
|
|
||
| // AddTimestampDoFn extracts an event time from a LogEntry. | ||
| func AddTimestampDoFn(element *LogEntry, emit func(beam.EventTime, *LogEntry)) error { | ||
| et, err := parser.extractEventTime(element) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| emit(mtime.FromTime(*et), element) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| package entry | ||
|
|
||
| import ( | ||
| "regexp" | ||
| "time" | ||
| ) | ||
|
|
||
| var parser *RegexParser | ||
|
|
||
| type Parser interface { | ||
| extractEventTime(element LogEntry) time.Time | ||
| } | ||
|
|
||
| type RegexParser struct { | ||
| // time string regex | ||
| regex *regexp.Regexp | ||
| // time parse layout | ||
| layout string | ||
| } | ||
|
|
||
| func NewDateTimeRegexParser() *RegexParser { | ||
| return &RegexParser{ | ||
| regex: DateTimeRegex, | ||
| layout: DateTimeLayout, | ||
| } | ||
| } | ||
|
|
||
| func NewCustomRegexParser(regx string) *RegexParser { | ||
| return &RegexParser{ | ||
| regex: regexp.MustCompile(regx), | ||
| } | ||
| } | ||
|
|
||
| func (r *RegexParser) extractEventTime(entry *LogEntry) (*time.Time, error) { | ||
| content := entry.Topic.Content | ||
| if dateTime := r.regex.FindString(content); len(dateTime) != 0 { | ||
| timeObj, err := time.Parse(r.layout, dateTime) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return &timeObj, nil | ||
| } | ||
| return nil, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.