rkt: Add --cpuprofile --memprofile for profiling the rkt

Documentation/commands.md

@@ -72,10 +72,12 @@ In addition to the flags used by individual `rkt` commands, `rkt` has a set of g
 
 | Flag | Default | Options | Description |
 | --- | --- | --- | --- |
+| `--cpuprofile (hidden flag)` | `` | A file path | Write CPU profile to the file |
 | `--debug` |  `false` | `true` or `false` | Prints out more debug information to `stderr` |
 | `--dir` | `/var/lib/rkt` | A directory path | Path to the `rkt` data directory |
 | `--insecure-options` |  none | **none**: All security features are enabled<br/>**http**: Allow HTTP connections. Be warned that this will send any credentials as clear text.<br/>**image**: Disables verifying image signatures<br/>**tls**: Accept any certificate from the server and any host name in that certificate<br/>**ondisk**: Disables verifying the integrity of the on-disk, rendered image before running. This significantly speeds up start time.<br/>**pubkey**: Allow fetching pubkeys via insecure connections (via HTTP connections or from servers with unverified certificates). This slightly extends the meaning of the `--trust-keys-from-https` flag.<br/>**all**: Disables all security checks | Comma-separated list of security features to disable |
 | `--local-config` |  `/etc/rkt` | A directory path | Path to the local configuration directory |
+| `--memprofile (hidden flag)` | `` | A file path | Write memory profile to the file |
 | `--system-config` |  `/usr/lib/rkt` | A directory path | Path to the system configuration directory |
 | `--trust-keys-from-https` |  `false` | `true` or `false` | Automatically trust gpg keys fetched from HTTPS (or HTTP if the insecure `pubkey` option is also specified) |
 | `--user-config` |  `` | A directory path | Path to the user configuration directory |

Documentation/benchmarks/README.md → Documentation/performance/README.md

@@ -29,3 +29,21 @@ temporary directory with `rkt_benchmark` prefix, `-v` will print out the current
 resource usage of each process every second. `-d` can be used to specify a
 duration to run the tests for (default of 10s). For example, `-d 30s` will run
 the tests for 30 seconds.
+
+# Profiling
+
+rkt will provide two hidden global flags `--cpuprofile` and `--memprofile` that can be used for performance profiling.
+Setting `--cpuprofile=$FILE` will make rkt write down the CPU profiles to the `$FILE`.
+Setting `--memprofile=$FILE` will make rkt write down the Memory profile to the `$FILE`.
+Note that memory profile will only be written down before rkt exits, so during the execution
+of rkt, the memory profile will be empty.
+
+The profile result can be viewed by go's profiling tool, for example:
+
+```shell
+$ sudo /usr/bin/rkt --cpuprofile=/tmp/cpu.profile --memprofile=/tmp/mem.profile gc --grace-period=0
+$ go tool pprof /usr/bin/rkt /tmp/cpu.profile
+$ go tool pprof /usr/bin/rkt /tmp/mem.profile
+```
+
+For more profiling tips, please see https://blog.golang.org/profiling-go-programs

Documentation/benchmarks/rkt-1-4-0-benchmarks.md → Documentation/performance/rkt-1-4-0-benchmarks.md

@@ -4,7 +4,7 @@ model name  : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz
 model name  : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz
 model name  : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz
 model name  : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz
-derek@proton ~> uname -a                             
+derek@proton ~> uname -a
 Linux proton 4.4.6 #1-NixOS SMP Wed Mar 16 15:43:17 UTC 2016 x86_64 GNU/Linux
 ```
 
@@ -25,7 +25,7 @@ container stop time: 17332926ns
 
 ### mem-stresser.aci
 ```
-derek@proton ~/go/src/github.com/coreos/rkt/tests/rkt-monitor> sudo ./rkt-monitor mem-stresser.aci 
+derek@proton ~/go/src/github.com/coreos/rkt/tests/rkt-monitor> sudo ./rkt-monitor mem-stresser.aci
 worker(18634): seconds alive: 9  avg CPU: 98.550401%  avg Mem: 318 mB  peak Mem: 555 mB
 rkt(18599): seconds alive: 10  avg CPU: 3.583814%  avg Mem: 2 mB  peak Mem: 2 mB
 systemd(18628): seconds alive: 9  avg CPU: 0.000000%  avg Mem: 4 mB  peak Mem: 4 mB
@@ -37,7 +37,7 @@ container stop time: 17593446ns
 
 ### cpu-stresser.aci
 ```
-derek@proton ~/go/src/github.com/coreos/rkt/tests/rkt-monitor> sudo ./rkt-monitor cpu-stresser.aci 
+derek@proton ~/go/src/github.com/coreos/rkt/tests/rkt-monitor> sudo ./rkt-monitor cpu-stresser.aci
 rkt(18706): seconds alive: 10  avg CPU: 3.587050%  avg Mem: 2 mB  peak Mem: 2 mB
 systemd(18736): seconds alive: 9  avg CPU: 0.000000%  avg Mem: 4 mB  peak Mem: 4 mB
 systemd-journal(18740): seconds alive: 9  avg CPU: 0.000000%  avg Mem: 6 mB  peak Mem: 6 mB

rkt/rkt.go

@@ -19,6 +19,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime/pprof"
 	"text/tabwriter"
 
 	"github.com/coreos/rkt/common"
@@ -128,6 +129,10 @@ var (
 		Help               bool
 		InsecureFlags      *rktflag.SecFlags
 		TrustKeysFromHTTPS bool
+
+		// Hidden flags for profiling.
+		CPUProfile string
+		MemProfile string
 	}{
 		Dir:             defaultDataDir,
 		SystemConfigDir: common.DefaultSystemConfigDir,
@@ -171,6 +176,10 @@ func init() {
 			globalFlags.InsecureFlags.PermissibleString()))
 	cmdRkt.PersistentFlags().BoolVar(&globalFlags.TrustKeysFromHTTPS, "trust-keys-from-https",
 		false, "automatically trust gpg keys fetched from https")
+	cmdRkt.PersistentFlags().StringVar(&globalFlags.CPUProfile, "cpuprofile", "", "write CPU profile to the file")
+	cmdRkt.PersistentFlags().MarkHidden("cpuprofile")
+	cmdRkt.PersistentFlags().StringVar(&globalFlags.MemProfile, "memprofile", "", "write memory profile to the file")
+	cmdRkt.PersistentFlags().MarkHidden("memprofile")
 
 	// Run this before the execution of each subcommand to set up output
 	cmdRkt.PersistentPreRun = func(cmd *cobra.Command, args []string) {
@@ -189,11 +198,47 @@ func getTabOutWithWriter(writer io.Writer) *tabwriter.Writer {
 	return aTabOut
 }
 
+func startProfile() (cpufile *os.File, memfile *os.File, err error) {
+	if globalFlags.CPUProfile != "" {
+		cpufile, err = os.Create(globalFlags.CPUProfile)
+		if err != nil {
+			return nil, nil, fmt.Errorf("cannot create cpu profile file %q: %v", globalFlags.CPUProfile, err)
+		}
+		pprof.StartCPUProfile(cpufile)
+	}
+	if globalFlags.MemProfile != "" {
+		memfile, err = os.Create(globalFlags.MemProfile)
+		if err != nil {
+			return nil, nil, fmt.Errorf("cannot create memory profile file %q: %v", globalFlags.MemProfile, err)
+		}
+	}
+	return cpufile, memfile, nil
+}
+
+func stopProfile(cpuprofile, memprofile *os.File) {
+	if globalFlags.CPUProfile != "" {
+		pprof.StopCPUProfile()
+		cpuprofile.Close()
+	}
+	if globalFlags.MemProfile != "" {
+		pprof.WriteHeapProfile(memprofile)
+		memprofile.Close()
+	}
+}
+
 // runWrapper returns a func(cmd *cobra.Command, args []string) that internally
 // will add command function return code and the reinsertion of the "--" flag
 // terminator.
 func runWrapper(cf func(cmd *cobra.Command, args []string) (exit int)) func(cmd *cobra.Command, args []string) {
 	return func(cmd *cobra.Command, args []string) {
+		cpufile, memfile, err := startProfile()
+		if err != nil {
+			stderr.PrintE("cannot setup profiling", err)
+			cmdExitCode = 1
+			return
+		}
+		defer stopProfile(cpufile, memfile)
+
 		cmdExitCode = cf(cmd, args)
 	}
 }