From c807832ea2bb3ab0a6cdee4f44f2d566b75cf0ac Mon Sep 17 00:00:00 2001 From: srstack Date: Wed, 9 Mar 2022 15:04:10 +0800 Subject: [PATCH 01/14] fix --- pkg/cluster/operation/check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/operation/check.go b/pkg/cluster/operation/check.go index 25808fb5b4..25b0881d4d 100644 --- a/pkg/cluster/operation/check.go +++ b/pkg/cluster/operation/check.go @@ -173,7 +173,7 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { osInfo.Name, osInfo.Release) return result } - case "centos", "redhat", "rhel": + case "centos", "redhat", "rhel", "ol": // check version // CentOS 8 is known to be not working, and we don't have plan to support it // as of now, we may add support for RHEL 8 based systems in the future. From 8c77d8966d9cbfc7daad2d2dc69ff40385b890ef Mon Sep 17 00:00:00 2001 From: srstack Date: Wed, 9 Mar 2022 15:09:51 +0800 Subject: [PATCH 02/14] fix --- pkg/cluster/operation/check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/operation/check.go b/pkg/cluster/operation/check.go index 25808fb5b4..25b0881d4d 100644 --- a/pkg/cluster/operation/check.go +++ b/pkg/cluster/operation/check.go @@ -173,7 +173,7 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { osInfo.Name, osInfo.Release) return result } - case "centos", "redhat", "rhel": + case "centos", "redhat", "rhel", "ol": // check version // CentOS 8 is known to be not working, and we don't have plan to support it // as of now, we may add support for RHEL 8 based systems in the future. From 36769454c075bd2ec7dcb463092674a592a73a5f Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 21 Mar 2022 00:37:23 +0800 Subject: [PATCH 03/14] init --- cmd/history.go | 69 ++++++++++++ cmd/root.go | 7 ++ components/cluster/command/audit.go | 1 - components/dm/command/audit.go | 1 - pkg/cluster/audit/audit.go | 2 - pkg/environment/history.go | 166 ++++++++++++++++++++++++++++ 6 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 cmd/history.go create mode 100644 pkg/environment/history.go diff --git a/cmd/history.go b/cmd/history.go new file mode 100644 index 0000000000..a46df17704 --- /dev/null +++ b/cmd/history.go @@ -0,0 +1,69 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/pingcap/tiup/pkg/environment" + "github.com/pingcap/tiup/pkg/tui" + "github.com/spf13/cobra" +) + +// newHistoryCmd history +func newHistoryCmd() *cobra.Command { + var rows int + var displayMode string + cmd := &cobra.Command{ + Use: "history", + Short: "Display the historical execution record of TiUP", + RunE: func(cmd *cobra.Command, args []string) error { + teleCommand = cmd.CommandPath() + env := environment.GlobalEnv() + + rows, err := env.GetHistory(rows) + if err != nil { + return err + } + + if displayMode == "json" { + for _, r := range rows { + rBytes, err := json.Marshal(r) + if err != nil { + continue + } + fmt.Println(string(rBytes)) + } + return nil + } + var table [][]string + table = append(table, []string{"Date", "Command", "Code"}) + + for _, r := range rows { + table = append(table, []string{ + r.Date.Format("2006-01-02T15:04:05"), + r.Command, + strconv.Itoa(r.Code), + }) + } + tui.PrintTable(table, true) + return nil + }, + } + cmd.Flags().StringVar(&displayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]") + cmd.Flags().IntVar(&rows, "r", 60, "If the specified version was already installed, force a reinstallation") + return cmd +} diff --git a/cmd/root.go b/cmd/root.go index c8d3596072..cefd3a0eb8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -198,6 +198,7 @@ the latest stable version will be downloaded from the repository.`, newMirrorCmd(), newTelemetryCmd(), newEnvCmd(), + newHistoryCmd(), ) originHelpFunc := rootCmd.HelpFunc() @@ -248,6 +249,12 @@ func Execute() { // us a dedicated package for that reportEnabled = false } else { + // record TiUP execution history + err := environment.HistoryRecord(env, os.Args, start, code) + if err != nil { + log.Warnf("record TiUP execution failed: %v", err) + } + teleMeta, _, err := telemetry.GetMeta(env) if err == nil { reportEnabled = teleMeta.Status == telemetry.EnableStatus diff --git a/components/cluster/command/audit.go b/components/cluster/command/audit.go index f095916a57..a0cd94adda 100644 --- a/components/cluster/command/audit.go +++ b/components/cluster/command/audit.go @@ -47,7 +47,6 @@ func newAuditCleanupCmd() *cobra.Command { Use: "cleanup", Short: "cleanup cluster audit logs", RunE: func(cmd *cobra.Command, args []string) error { - if retainDays < 0 { return errors.Errorf("retain-days cannot be less than 0") } diff --git a/components/dm/command/audit.go b/components/dm/command/audit.go index f2a996cd5f..e80035a9ba 100644 --- a/components/dm/command/audit.go +++ b/components/dm/command/audit.go @@ -47,7 +47,6 @@ func newAuditCleanupCmd() *cobra.Command { Use: "cleanup", Short: "cleanup dm audit logs", RunE: func(cmd *cobra.Command, args []string) error { - if retainDays < 0 { return errors.Errorf("retain-days cannot be less than 0") } diff --git a/pkg/cluster/audit/audit.go b/pkg/cluster/audit/audit.go index b318fb5b1d..c502a71d6d 100644 --- a/pkg/cluster/audit/audit.go +++ b/pkg/cluster/audit/audit.go @@ -146,7 +146,6 @@ func GetAuditList(dir string) ([]Item, error) { // OutputAuditLog outputs audit log. func OutputAuditLog(dir, fileSuffix string, data []byte) error { - auditID := base52.Encode(time.Now().UnixNano() + rand.Int63n(1000)) if customID := os.Getenv(EnvNameAuditID); customID != "" { auditID = fmt.Sprintf("%s_%s", auditID, customID) @@ -223,7 +222,6 @@ type deleteAuditLog struct { // DeleteAuditLog cleanup audit log func DeleteAuditLog(dir string, retainDays int, skipConfirm bool, displayMode string) error { - if retainDays < 0 { return errors.Errorf("retainDays cannot be less than 0") } diff --git a/pkg/environment/history.go b/pkg/environment/history.go new file mode 100644 index 0000000000..dbe68169b4 --- /dev/null +++ b/pkg/environment/history.go @@ -0,0 +1,166 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/pingcap/tiup/pkg/utils" +) + +const ( + historyDir = "history" + historyPrefix = "tiup-history-" +) + +// commandRow type of command history row +type historyRow struct { + Command string `json:"command"` + Date time.Time `json:"time"` + Code int `json:"code"` +} + +// historyItem record history row file item +type historyItem struct { + path string + time string +} + +// HistoryRecord record tiup exec cmd +func HistoryRecord(env *Environment, command []string, date time.Time, code int) error { + if env == nil { + return nil + } + + historyPath := env.LocalPath(historyDir) + if utils.IsNotExist(historyPath) { + err := os.MkdirAll(historyPath, 0755) + if err != nil { + return err + } + } + + h := &historyRow{ + Command: strings.Join(command, " "), + Date: date, + Code: code, + } + + return h.save(historyPath) +} + +// save save commandRow to file +func (h *historyRow) save(dir string) error { + hBytes, err := json.Marshal(h) + if err != nil { + return err + } + + historyFile := filepath.Join(dir, + fmt.Sprintf("%s%s", historyPrefix, h.Date.Format("2006-01-02")), + ) + + f, err := os.OpenFile(historyFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0755) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(append(hBytes, []byte("\n")...)) + return err +} + +// GetHistory get tiup history +func (env *Environment) GetHistory(count int) ([]*historyRow, error) { + fList, err := getHistoryFileList(env.LocalPath(historyDir)) + if err != nil { + return nil, err + } + rows := []*historyRow{} + for _, f := range fList { + rs, err := getHistoryRows(f.path) + if err != nil { + return rows, err + } + if (len(rows) + len(rs)) > count { + i := len(rows) + len(rs) - count + rows = append(rs[i:], rows...) + break + } + + rows = append(rs, rows...) + } + return rows, nil +} + +// getHistoryRows get tiup history execution row +func getHistoryRows(path string) ([]*historyRow, error) { + rows := []*historyRow{} + + fi, err := os.Open(path) + if err != nil { + return rows, err + } + defer fi.Close() + + br := bufio.NewReader(fi) + for { + a, _, c := br.ReadLine() + if c == io.EOF { + break + } + r := &historyRow{} + // ignore + err := json.Unmarshal(a, r) + if err != nil { + continue + } + rows = append(rows, r) + } + + return rows, nil +} + +// getHistoryFileList get the history file list +func getHistoryFileList(dir string) ([]historyItem, error) { + fileInfos, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + hfileList := []historyItem{} + for _, fi := range fileInfos { + if fi.IsDir() { + continue + } + + hfileList = append(hfileList, historyItem{ + path: filepath.Join(dir, fi.Name()), + time: strings.TrimPrefix(fi.Name(), historyPrefix), + }) + } + + sort.Slice(hfileList, func(i, j int) bool { + return hfileList[i].time > hfileList[j].time + }) + + return hfileList, nil +} From 30d519c88bcc493f44de06c7ba3898c44f4c2394 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 21 Mar 2022 00:54:36 +0800 Subject: [PATCH 04/14] fix rows flag --- cmd/history.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/cmd/history.go b/cmd/history.go index a46df17704..702564e557 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -18,6 +18,7 @@ import ( "fmt" "strconv" + "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/tui" "github.com/spf13/cobra" @@ -31,9 +32,7 @@ func newHistoryCmd() *cobra.Command { Use: "history", Short: "Display the historical execution record of TiUP", RunE: func(cmd *cobra.Command, args []string) error { - teleCommand = cmd.CommandPath() env := environment.GlobalEnv() - rows, err := env.GetHistory(rows) if err != nil { return err @@ -64,6 +63,28 @@ func newHistoryCmd() *cobra.Command { }, } cmd.Flags().StringVar(&displayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]") - cmd.Flags().IntVar(&rows, "r", 60, "If the specified version was already installed, force a reinstallation") + cmd.Flags().IntVarP(&rows, "rows", "r", 60, "If the specified version was already installed, force a reinstallation") + + cmd.AddCommand(newHistoryCleanupCmd()) + return cmd +} + +func newHistoryCleanupCmd() *cobra.Command { + var retainDays int + cmd := &cobra.Command{ + Use: "cleanup", + Short: "cleanup cluster audit logs", + RunE: func(cmd *cobra.Command, args []string) error { + if retainDays < 0 { + return errors.Errorf("retain-days cannot be less than 0") + } + + // env := environment.GlobalEnv() + + return nil + }, + } + + cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep audit logs for deletion") return cmd } From 8ad904ffde16045afcb17c5bbac678ff5504e518 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 21 Mar 2022 17:30:14 +0800 Subject: [PATCH 05/14] add cleanup --- cmd/history.go | 16 ++++----- cmd/root.go | 2 +- pkg/environment/history.go | 67 +++++++++++++++++++++++++++++++------- 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/cmd/history.go b/cmd/history.go index 702564e557..08515c2073 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -18,7 +18,6 @@ import ( "fmt" "strconv" - "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/tui" "github.com/spf13/cobra" @@ -70,21 +69,20 @@ func newHistoryCmd() *cobra.Command { } func newHistoryCleanupCmd() *cobra.Command { - var retainDays int + // var retainDays int cmd := &cobra.Command{ Use: "cleanup", Short: "cleanup cluster audit logs", RunE: func(cmd *cobra.Command, args []string) error { - if retainDays < 0 { - return errors.Errorf("retain-days cannot be less than 0") - } - - // env := environment.GlobalEnv() + // if retainDays < 0 { + // return errors.Errorf("retain-days cannot be less than 0") + // } - return nil + env := environment.GlobalEnv() + return env.DeleteHistory() }, } - cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep audit logs for deletion") + // cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep audit logs for deletion") return cmd } diff --git a/cmd/root.go b/cmd/root.go index cefd3a0eb8..c3afc690a8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -252,7 +252,7 @@ func Execute() { // record TiUP execution history err := environment.HistoryRecord(env, os.Args, start, code) if err != nil { - log.Warnf("record TiUP execution failed: %v", err) + log.Warnf("Record TiUP execution history log failed: %v", err) } teleMeta, _, err := telemetry.GetMeta(env) diff --git a/pkg/environment/history.go b/pkg/environment/history.go index dbe68169b4..774263796a 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -28,21 +29,23 @@ import ( ) const ( - historyDir = "history" - historyPrefix = "tiup-history-" + historyDir = "history" + historyPrefix = "tiup-history-" + historyFileSize int64 = 1024 * 1024 // defaule 1M, about 1w records ) // commandRow type of command history row type historyRow struct { - Command string `json:"command"` Date time.Time `json:"time"` + Command string `json:"command"` Code int `json:"code"` } // historyItem record history row file item type historyItem struct { - path string - time string + path string + size int64 + index int } // HistoryRecord record tiup exec cmd @@ -75,11 +78,9 @@ func (h *historyRow) save(dir string) error { return err } - historyFile := filepath.Join(dir, - fmt.Sprintf("%s%s", historyPrefix, h.Date.Format("2006-01-02")), - ) + historyFile := getLatestHistoryFile(dir) - f, err := os.OpenFile(historyFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0755) + f, err := os.OpenFile(historyFile.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0755) if err != nil { return err } @@ -111,6 +112,18 @@ func (env *Environment) GetHistory(count int) ([]*historyRow, error) { return rows, nil } +// DeleteHistory delete history file +func (env *Environment) DeleteHistory() error { + fList, err := getHistoryFileList(env.LocalPath(historyDir)) + if err != nil { + return err + } + for _, f := range fList { + os.Remove(f.path) + } + return nil +} + // getHistoryRows get tiup history execution row func getHistoryRows(path string) ([]*historyRow, error) { rows := []*historyRow{} @@ -152,15 +165,45 @@ func getHistoryFileList(dir string) ([]historyItem, error) { continue } + // another suffix + // ex: tiup-history-0.bak + i, err := strconv.Atoi((strings.TrimPrefix(fi.Name(), historyPrefix))) + if err != nil { + continue + } + + fInfo, _ := fi.Info() hfileList = append(hfileList, historyItem{ - path: filepath.Join(dir, fi.Name()), - time: strings.TrimPrefix(fi.Name(), historyPrefix), + path: filepath.Join(dir, fi.Name()), + index: i, + size: fInfo.Size(), }) } sort.Slice(hfileList, func(i, j int) bool { - return hfileList[i].time > hfileList[j].time + return hfileList[i].index > hfileList[j].index }) return hfileList, nil } + +func getLatestHistoryFile(dir string) (item historyItem) { + fileList, err := getHistoryFileList(dir) + // start from 0 + if len(fileList) == 0 || err != nil { + item.index = 0 + item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) + return + } + + latestItem := fileList[0] + + if latestItem.size > historyFileSize { + item.index = latestItem.index + 1 + item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) + } else { + item = latestItem + } + + return item +} From a9e134066d5e95ba1dda7d6c0291634f50e5f986 Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 21 Mar 2022 18:04:47 +0800 Subject: [PATCH 06/14] add cleanup --- cmd/history.go | 13 +++++++------ go.mod | 1 + pkg/environment/history.go | 24 ++++++++++++++++++------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/cmd/history.go b/cmd/history.go index 08515c2073..fdca94866a 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -18,6 +18,7 @@ import ( "fmt" "strconv" + "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/tui" "github.com/spf13/cobra" @@ -69,20 +70,20 @@ func newHistoryCmd() *cobra.Command { } func newHistoryCleanupCmd() *cobra.Command { - // var retainDays int + var retainDays int cmd := &cobra.Command{ Use: "cleanup", Short: "cleanup cluster audit logs", RunE: func(cmd *cobra.Command, args []string) error { - // if retainDays < 0 { - // return errors.Errorf("retain-days cannot be less than 0") - // } + if retainDays < 0 { + return errors.Errorf("retain-days cannot be less than 0") + } env := environment.GlobalEnv() - return env.DeleteHistory() + return env.DeleteHistory(retainDays) }, } - // cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep audit logs for deletion") + cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep audit logs for deletion") return cmd } diff --git a/go.mod b/go.mod index 0df7086820..0bf3c7d52b 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20220125073028-58f2ac94aa38 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect github.com/pingcap/tidb-insight/collector v0.0.0-20220111101533-227008e9835b + github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.32.1 github.com/prometheus/prom2json v1.3.0 diff --git a/pkg/environment/history.go b/pkg/environment/history.go index 774263796a..4a8e82a134 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "os" "path/filepath" "sort" @@ -26,6 +27,7 @@ import ( "time" "github.com/pingcap/tiup/pkg/utils" + "github.com/pkg/errors" ) const ( @@ -44,7 +46,7 @@ type historyRow struct { // historyItem record history row file item type historyItem struct { path string - size int64 + info fs.FileInfo index int } @@ -80,7 +82,7 @@ func (h *historyRow) save(dir string) error { historyFile := getLatestHistoryFile(dir) - f, err := os.OpenFile(historyFile.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0755) + f, err := os.OpenFile(historyFile.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { return err } @@ -113,13 +115,23 @@ func (env *Environment) GetHistory(count int) ([]*historyRow, error) { } // DeleteHistory delete history file -func (env *Environment) DeleteHistory() error { +func (env *Environment) DeleteHistory(retainDays int) error { + if retainDays < 0 { + return errors.Errorf("retainDays cannot be less than 0") + } + + // history file before `DelBeforeTime` will be deleted + oneDayDuration, _ := time.ParseDuration("-24h") + delBeforeTime := time.Now().Add(oneDayDuration * time.Duration(retainDays)) + fList, err := getHistoryFileList(env.LocalPath(historyDir)) if err != nil { return err } for _, f := range fList { - os.Remove(f.path) + if f.info.ModTime().Before(delBeforeTime) { + os.Remove(f.path) + } } return nil } @@ -176,7 +188,7 @@ func getHistoryFileList(dir string) ([]historyItem, error) { hfileList = append(hfileList, historyItem{ path: filepath.Join(dir, fi.Name()), index: i, - size: fInfo.Size(), + info: fInfo, }) } @@ -198,7 +210,7 @@ func getLatestHistoryFile(dir string) (item historyItem) { latestItem := fileList[0] - if latestItem.size > historyFileSize { + if latestItem.info.Size() > historyFileSize { item.index = latestItem.index + 1 item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) } else { From d68c3c86ed48d5d58b28b71fc754832e24b70808 Mon Sep 17 00:00:00 2001 From: srstack Date: Wed, 23 Mar 2022 10:56:49 +0800 Subject: [PATCH 07/14] fix --- cmd/history.go | 27 ++++++++++--- pkg/environment/history.go | 80 +++++++++++++++++++++++++++++++------- 2 files changed, 87 insertions(+), 20 deletions(-) diff --git a/cmd/history.go b/cmd/history.go index fdca94866a..0b87bf2562 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -26,14 +26,24 @@ import ( // newHistoryCmd history func newHistoryCmd() *cobra.Command { - var rows int + rows := 100 var displayMode string + var all bool cmd := &cobra.Command{ - Use: "history", - Short: "Display the historical execution record of TiUP", + Use: "history ", + Short: "Display the historical execution record of TiUP, displays 100 lines by default", RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + r, err := strconv.Atoi(args[0]) + if err == nil { + rows = r + } else { + return fmt.Errorf("%s: numeric argument required", args[0]) + } + } + env := environment.GlobalEnv() - rows, err := env.GetHistory(rows) + rows, err := env.GetHistory(rows, all) if err != nil { return err } @@ -63,14 +73,14 @@ func newHistoryCmd() *cobra.Command { }, } cmd.Flags().StringVar(&displayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]") - cmd.Flags().IntVarP(&rows, "rows", "r", 60, "If the specified version was already installed, force a reinstallation") - + cmd.Flags().BoolVar(&all, "all", false, "Delete all history") cmd.AddCommand(newHistoryCleanupCmd()) return cmd } func newHistoryCleanupCmd() *cobra.Command { var retainDays int + var all bool cmd := &cobra.Command{ Use: "cleanup", Short: "cleanup cluster audit logs", @@ -79,11 +89,16 @@ func newHistoryCleanupCmd() *cobra.Command { return errors.Errorf("retain-days cannot be less than 0") } + if all { + retainDays = 0 + } + env := environment.GlobalEnv() return env.DeleteHistory(retainDays) }, } cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep audit logs for deletion") + cmd.Flags().BoolVar(&all, "all", false, "Delete all history") return cmd } diff --git a/pkg/environment/history.go b/pkg/environment/history.go index 4a8e82a134..4909e8af0d 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -31,9 +31,9 @@ import ( ) const ( - historyDir = "history" - historyPrefix = "tiup-history-" - historyFileSize int64 = 1024 * 1024 // defaule 1M, about 1w records + historyDir = "history" + historyPrefix = "tiup-history-" + historySize int64 = 1024 * 64 // history file default size is 64k ) // commandRow type of command history row @@ -74,8 +74,8 @@ func HistoryRecord(env *Environment, command []string, date time.Time, code int) } // save save commandRow to file -func (h *historyRow) save(dir string) error { - hBytes, err := json.Marshal(h) +func (r *historyRow) save(dir string) error { + rBytes, err := json.Marshal(r) if err != nil { return err } @@ -87,23 +87,23 @@ func (h *historyRow) save(dir string) error { return err } defer f.Close() - _, err = f.Write(append(hBytes, []byte("\n")...)) + _, err = f.Write(append(rBytes, []byte("\n")...)) return err } // GetHistory get tiup history -func (env *Environment) GetHistory(count int) ([]*historyRow, error) { +func (env *Environment) GetHistory(count int, all bool) ([]*historyRow, error) { fList, err := getHistoryFileList(env.LocalPath(historyDir)) if err != nil { return nil, err } rows := []*historyRow{} for _, f := range fList { - rs, err := getHistoryRows(f.path) + rs, err := f.getHistory() if err != nil { return rows, err } - if (len(rows) + len(rs)) > count { + if (len(rows)+len(rs)) > count && !all { i := len(rows) + len(rs) - count rows = append(rs[i:], rows...) break @@ -128,19 +128,70 @@ func (env *Environment) DeleteHistory(retainDays int) error { if err != nil { return err } + + if len(fList) == 0 { + return nil + } + for _, f := range fList { if f.info.ModTime().Before(delBeforeTime) { - os.Remove(f.path) + err := os.Remove(f.path) + if err != nil { + return err + } + continue + } + + rows, err := f.getHistory() + if err != nil { + continue + } + // create time before deltime + if rows[0].Date.Before(delBeforeTime) { + if rows[len(rows)-1].Date.Before(delBeforeTime) { + err := os.Remove(f.path) + if err != nil { + return err + } + continue + } + + tmp := f.path + "_tmp" + err := os.Rename(f.path, tmp) + if err != nil { + return err + } + + fi, err := os.OpenFile(f.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return err + } + defer fi.Close() + + for _, row := range rows { + if row.Date.After(delBeforeTime) { + rBytes, err := json.Marshal(row) + if err != nil { + continue + } + + _, err = fi.Write(append(rBytes, []byte("\n")...)) + if err != nil { + continue + } + } + } + os.Remove(tmp) } } return nil } -// getHistoryRows get tiup history execution row -func getHistoryRows(path string) ([]*historyRow, error) { +// getHistory get tiup history execution row +func (i *historyItem) getHistory() ([]*historyRow, error) { rows := []*historyRow{} - fi, err := os.Open(path) + fi, err := os.Open(i.path) if err != nil { return rows, err } @@ -199,6 +250,7 @@ func getHistoryFileList(dir string) ([]historyItem, error) { return hfileList, nil } +// getLatestHistoryFile get the latest history file, use index 0 if it doesn't exist func getLatestHistoryFile(dir string) (item historyItem) { fileList, err := getHistoryFileList(dir) // start from 0 @@ -210,7 +262,7 @@ func getLatestHistoryFile(dir string) (item historyItem) { latestItem := fileList[0] - if latestItem.info.Size() > historyFileSize { + if latestItem.info.Size() >= historySize { item.index = latestItem.index + 1 item.path = filepath.Join(dir, fmt.Sprintf("%s%s", historyPrefix, strconv.Itoa(item.index))) } else { From 933f4cf20f7c3b9b456e6b0d8c1e845d34506a23 Mon Sep 17 00:00:00 2001 From: srstack Date: Wed, 23 Mar 2022 10:59:40 +0800 Subject: [PATCH 08/14] fix --- cmd/history.go | 2 +- pkg/environment/history.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/history.go b/cmd/history.go index 0b87bf2562..be7cf69e34 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -73,7 +73,7 @@ func newHistoryCmd() *cobra.Command { }, } cmd.Flags().StringVar(&displayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]") - cmd.Flags().BoolVar(&all, "all", false, "Delete all history") + cmd.Flags().BoolVar(&all, "all", false, "Display all history") cmd.AddCommand(newHistoryCleanupCmd()) return cmd } diff --git a/pkg/environment/history.go b/pkg/environment/history.go index 4909e8af0d..bf569341ff 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -269,5 +269,5 @@ func getLatestHistoryFile(dir string) (item historyItem) { item = latestItem } - return item + return } From 198021e97f32435f2f177f85552428bdde09a49d Mon Sep 17 00:00:00 2001 From: srstack Date: Wed, 23 Mar 2022 11:01:07 +0800 Subject: [PATCH 09/14] fix --- cmd/history.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/history.go b/cmd/history.go index be7cf69e34..c57c829f8b 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -73,7 +73,7 @@ func newHistoryCmd() *cobra.Command { }, } cmd.Flags().StringVar(&displayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]") - cmd.Flags().BoolVar(&all, "all", false, "Display all history") + cmd.Flags().BoolVar(&all, "all", false, "Display all execution history") cmd.AddCommand(newHistoryCleanupCmd()) return cmd } @@ -83,7 +83,7 @@ func newHistoryCleanupCmd() *cobra.Command { var all bool cmd := &cobra.Command{ Use: "cleanup", - Short: "cleanup cluster audit logs", + Short: "delete all execution history", RunE: func(cmd *cobra.Command, args []string) error { if retainDays < 0 { return errors.Errorf("retain-days cannot be less than 0") From 2cda001a8f9ca44dd634b4da4cc642cdc09cdd1f Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 24 Mar 2022 14:46:24 +0800 Subject: [PATCH 10/14] fix --- cmd/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/history.go b/cmd/history.go index c57c829f8b..e0f475c0aa 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -98,7 +98,7 @@ func newHistoryCleanupCmd() *cobra.Command { }, } - cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep audit logs for deletion") + cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep history for deletion") cmd.Flags().BoolVar(&all, "all", false, "Delete all history") return cmd } From c70960833403eb8375f95903ebce51d1f0a6a422 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 24 Mar 2022 17:09:47 +0800 Subject: [PATCH 11/14] add some hints --- cmd/history.go | 5 ++++- pkg/environment/history.go | 23 ++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cmd/history.go b/cmd/history.go index e0f475c0aa..521a55e155 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -69,6 +69,7 @@ func newHistoryCmd() *cobra.Command { }) } tui.PrintTable(table, true) + fmt.Printf("history log save path: %s\n", env.LocalPath(environment.HistoryDir)) return nil }, } @@ -81,6 +82,7 @@ func newHistoryCmd() *cobra.Command { func newHistoryCleanupCmd() *cobra.Command { var retainDays int var all bool + var skipConfirm bool cmd := &cobra.Command{ Use: "cleanup", Short: "delete all execution history", @@ -94,11 +96,12 @@ func newHistoryCleanupCmd() *cobra.Command { } env := environment.GlobalEnv() - return env.DeleteHistory(retainDays) + return env.DeleteHistory(retainDays, skipConfirm) }, } cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep history for deletion") cmd.Flags().BoolVar(&all, "all", false, "Delete all history") + cmd.Flags().BoolVarP(&skipConfirm, "yes", "y", false, "Skip all confirmations and assumes 'yes'") return cmd } diff --git a/pkg/environment/history.go b/pkg/environment/history.go index bf569341ff..e7ab8fb321 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -26,12 +26,15 @@ import ( "strings" "time" + "github.com/fatih/color" + "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" "github.com/pkg/errors" ) const ( - historyDir = "history" + // HistoryDir history save path + HistoryDir = "history" historyPrefix = "tiup-history-" historySize int64 = 1024 * 64 // history file default size is 64k ) @@ -56,7 +59,7 @@ func HistoryRecord(env *Environment, command []string, date time.Time, code int) return nil } - historyPath := env.LocalPath(historyDir) + historyPath := env.LocalPath(HistoryDir) if utils.IsNotExist(historyPath) { err := os.MkdirAll(historyPath, 0755) if err != nil { @@ -93,7 +96,7 @@ func (r *historyRow) save(dir string) error { // GetHistory get tiup history func (env *Environment) GetHistory(count int, all bool) ([]*historyRow, error) { - fList, err := getHistoryFileList(env.LocalPath(historyDir)) + fList, err := getHistoryFileList(env.LocalPath(HistoryDir)) if err != nil { return nil, err } @@ -115,7 +118,7 @@ func (env *Environment) GetHistory(count int, all bool) ([]*historyRow, error) { } // DeleteHistory delete history file -func (env *Environment) DeleteHistory(retainDays int) error { +func (env *Environment) DeleteHistory(retainDays int, skipConfirm bool) error { if retainDays < 0 { return errors.Errorf("retainDays cannot be less than 0") } @@ -124,7 +127,17 @@ func (env *Environment) DeleteHistory(retainDays int) error { oneDayDuration, _ := time.ParseDuration("-24h") delBeforeTime := time.Now().Add(oneDayDuration * time.Duration(retainDays)) - fList, err := getHistoryFileList(env.LocalPath(historyDir)) + if !skipConfirm { + fmt.Printf("History logs before %s will be %s!\n", + color.HiYellowString(delBeforeTime.Format("2006-01-02T15:04:05")), + color.HiYellowString("deleted"), + ) + if err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]:"); err != nil { + return err + } + } + + fList, err := getHistoryFileList(env.LocalPath(HistoryDir)) if err != nil { return err } From 9c6de1f94f753391f73c54289826ed501a0c4dc3 Mon Sep 17 00:00:00 2001 From: srstack Date: Sun, 27 Mar 2022 22:34:35 +0800 Subject: [PATCH 12/14] fix --- pkg/environment/history.go | 42 -------------------------------------- 1 file changed, 42 deletions(-) diff --git a/pkg/environment/history.go b/pkg/environment/history.go index e7ab8fb321..d3ca8f437d 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -154,48 +154,6 @@ func (env *Environment) DeleteHistory(retainDays int, skipConfirm bool) error { } continue } - - rows, err := f.getHistory() - if err != nil { - continue - } - // create time before deltime - if rows[0].Date.Before(delBeforeTime) { - if rows[len(rows)-1].Date.Before(delBeforeTime) { - err := os.Remove(f.path) - if err != nil { - return err - } - continue - } - - tmp := f.path + "_tmp" - err := os.Rename(f.path, tmp) - if err != nil { - return err - } - - fi, err := os.OpenFile(f.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return err - } - defer fi.Close() - - for _, row := range rows { - if row.Date.After(delBeforeTime) { - rBytes, err := json.Marshal(row) - if err != nil { - continue - } - - _, err = fi.Write(append(rBytes, []byte("\n")...)) - if err != nil { - continue - } - } - } - os.Remove(tmp) - } } return nil } From d41d7c228483e4c99e412cb7e555b60330cf567d Mon Sep 17 00:00:00 2001 From: srstack Date: Mon, 28 Mar 2022 11:52:04 +0800 Subject: [PATCH 13/14] fix --- cmd/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/history.go b/cmd/history.go index 521a55e155..8adc2b18c9 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -73,7 +73,7 @@ func newHistoryCmd() *cobra.Command { return nil }, } - cmd.Flags().StringVar(&displayMode, "format", "default", "(EXPERIMENTAL) The format of output, available values are [default, json]") + cmd.Flags().StringVar(&displayMode, "format", "default", "The format of output, available values are [default, json]") cmd.Flags().BoolVar(&all, "all", false, "Display all execution history") cmd.AddCommand(newHistoryCleanupCmd()) return cmd From f9c58a44fdc63ae3abac0e888f97bb4a7cdca2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Qi=CE=BC=24hi=D0=AFu=C3=AD?= <39378935+srstack@users.noreply.github.com> Date: Thu, 14 Apr 2022 16:06:08 +0800 Subject: [PATCH 14/14] Update pkg/environment/history.go Co-authored-by: Allen Zhong --- pkg/environment/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/environment/history.go b/pkg/environment/history.go index d3ca8f437d..a71e303fb7 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -43,7 +43,7 @@ const ( type historyRow struct { Date time.Time `json:"time"` Command string `json:"command"` - Code int `json:"code"` + Code int `json:"exit_code"` } // historyItem record history row file item