Skip to content

Commit

Permalink
[POA-2129] Separate out common apidump flags (#49)
Browse files Browse the repository at this point in the history
* In this PR, we have moved out common `apidump` flags from `apidump.go`
to `common_flags.go`.
* `common_flags.go` have two functions:
* `AddCommonApiDumpFlags()`: Add the flags to cobra.Command object. It
returns the objects to which flag values will be assigned.
* `ConvertCommonApiDumpFlagsToArgs()`: Returns an array of strings (like
`--flag value`). It is added to the apidump command in different
platform installation configurations.
* The `ecs` and `ec2` commands will read these values and then convert
them to flags for `apidump` commands and add them to the definition file
(`ecs`) or systems service file (`ec2`)
* The `apidump` command will use these flag values to set the main
`apidump` package variables.
  • Loading branch information
mudit-postman authored Dec 4, 2024
1 parent f2cc043 commit ac38940
Show file tree
Hide file tree
Showing 7 changed files with 170 additions and 126 deletions.
96 changes: 14 additions & 82 deletions cmd/internal/apidump/apidump.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,9 @@ var (
outFlag location.Location
projectID string
postmanCollectionID string
interfacesFlag []string
filterFlag string
sampleRateFlag float64
rateLimitFlag float64
tagsFlag []string
appendByTagFlag bool
pathExclusionsFlag []string
hostExclusionsFlag []string
pathAllowlistFlag []string
hostAllowlistFlag []string
execCommandFlag string
execCommandUserFlag string
pluginsFlag []string
Expand All @@ -47,8 +40,8 @@ var (
maxWitnessSize_bytes int
dockerExtensionMode bool
healthCheckPort int
randomizedStart int
sendWitnessPayloads bool

commonApidumpFlags CommonApidumpFlags
)

// This function will either startup apidump normally, or never return, with probability
Expand All @@ -59,7 +52,7 @@ var (
//
// Negative values are effectively treated as 0 probability, instead of being validated.
func applyRandomizedStart() {
prob := randomizedStart
prob := commonApidumpFlags.RandomizedStart

if env := os.Getenv("POSTMAN_AGENT_RANDOM_START"); env != "" {
override, err := strconv.Atoi(env)
Expand Down Expand Up @@ -183,8 +176,8 @@ func apidumpRunInternal(cmd *cobra.Command, _ []string) error {
}

// Rate limit must be greater than zero.
if rateLimitFlag <= 0.0 {
rateLimitFlag = 1000.0
if commonApidumpFlags.RateLimit <= 0.0 {
commonApidumpFlags.RateLimit = 1000.0
}

// If we collect TLS information, we have to parse it
Expand All @@ -203,13 +196,13 @@ func apidumpRunInternal(cmd *cobra.Command, _ []string) error {
ServiceID: serviceID,
Tags: traceTags,
SampleRate: sampleRateFlag,
WitnessesPerMinute: rateLimitFlag,
Interfaces: interfacesFlag,
Filter: filterFlag,
PathExclusions: pathExclusionsFlag,
HostExclusions: hostExclusionsFlag,
PathAllowlist: pathAllowlistFlag,
HostAllowlist: hostAllowlistFlag,
WitnessesPerMinute: commonApidumpFlags.RateLimit,
Interfaces: commonApidumpFlags.Interfaces,
Filter: commonApidumpFlags.Filter,
PathExclusions: commonApidumpFlags.PathExclusions,
HostExclusions: commonApidumpFlags.HostExclusions,
PathAllowlist: commonApidumpFlags.PathAllowlist,
HostAllowlist: commonApidumpFlags.HostAllowlist,
ExecCommand: execCommandFlag,
ExecCommandUser: execCommandUserFlag,
Plugins: plugins,
Expand All @@ -222,7 +215,7 @@ func apidumpRunInternal(cmd *cobra.Command, _ []string) error {
MaxWitnessSize_bytes: maxWitnessSize_bytes,
DockerExtensionMode: dockerExtensionMode,
HealthCheckPort: healthCheckPort,
SendWitnessPayloads: sendWitnessPayloads,
SendWitnessPayloads: commonApidumpFlags.SendWitnessPayloads,
}
if err := apidump.Run(args); err != nil {
return cmderr.AkitaErr{Err: err}
Expand Down Expand Up @@ -255,18 +248,6 @@ func init() {

Cmd.MarkFlagsMutuallyExclusive("project", "collection")

Cmd.Flags().StringVar(
&filterFlag,
"filter",
"",
"Used to match packets going to and coming from your API service.")

Cmd.Flags().StringSliceVar(
&interfacesFlag,
"interfaces",
nil,
"List of network interfaces to listen on. Defaults to all interfaces on host.")

Cmd.Flags().Float64Var(
&sampleRateFlag,
"sample-rate",
Expand All @@ -275,13 +256,6 @@ func init() {
)
Cmd.Flags().MarkDeprecated("sample-rate", "use --rate-limit instead.")

Cmd.Flags().Float64Var(
&rateLimitFlag,
"rate-limit",
apispec.DefaultRateLimit,
"Number of requests per minute to capture.",
)

Cmd.Flags().StringSliceVar(
&tagsFlag,
"tags",
Expand All @@ -296,34 +270,6 @@ func init() {
"Add to the most recent trace with matching tag.")
Cmd.Flags().MarkDeprecated("append-by-tag", "and is no longer necessary. All traces in a project are now combined into a single model. Please remove this flag.")

Cmd.Flags().StringSliceVar(
&pathExclusionsFlag,
"path-exclusions",
nil,
"Removes HTTP paths matching regular expressions.",
)

Cmd.Flags().StringSliceVar(
&hostExclusionsFlag,
"host-exclusions",
nil,
"Removes HTTP hosts matching regular expressions.",
)

Cmd.Flags().StringSliceVar(
&pathAllowlistFlag,
"path-allow",
nil,
"Allows only HTTP paths matching regular expressions.",
)

Cmd.Flags().StringSliceVar(
&hostAllowlistFlag,
"host-allow",
nil,
"Allows only HTTP hosts matching regular expressions.",
)

Cmd.Flags().StringVarP(
&execCommandFlag,
"command",
Expand Down Expand Up @@ -419,19 +365,5 @@ func init() {
)
_ = Cmd.Flags().MarkHidden("health-check-port")

Cmd.Flags().IntVar(
&randomizedStart,
"randomized-start",
100,
"Probability that the apidump command will start intercepting traffic.",
)
_ = Cmd.Flags().MarkHidden("randomized-start")

Cmd.Flags().BoolVar(
&sendWitnessPayloads,
"send-witness-payloads",
false,
"Send request and response payloads to Postman",
)
_ = Cmd.Flags().MarkHidden("send-witness-payloads")
commonApidumpFlags = AddCommonApiDumpFlags(Cmd)
}
130 changes: 130 additions & 0 deletions cmd/internal/apidump/common_flags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package apidump

import (
"strconv"

"github.com/postmanlabs/postman-insights-agent/apispec"
"github.com/spf13/cobra"
)

type CommonApidumpFlags struct {
Filter string
HostAllowlist []string
HostExclusions []string
Interfaces []string
PathAllowlist []string
PathExclusions []string
RandomizedStart int
RateLimit float64
SendWitnessPayloads bool
}

func AddCommonApiDumpFlags(cmd *cobra.Command) (flags CommonApidumpFlags) {
cmd.Flags().StringVar(
&flags.Filter,
"filter",
"",
"Used to match packets going to and coming from your API service.",
)

cmd.Flags().StringSliceVar(
&flags.HostAllowlist,
"host-allow",
nil,
"Allows only HTTP hosts matching regular expressions.",
)

cmd.Flags().StringSliceVar(
&flags.HostExclusions,
"host-exclusions",
nil,
"Removes HTTP hosts matching regular expressions.",
)

cmd.Flags().StringSliceVar(
&flags.Interfaces,
"interfaces",
nil,
"List of network interfaces to listen on. Defaults to all interfaces on host.",
)

cmd.Flags().StringSliceVar(
&flags.PathAllowlist,
"path-allow",
nil,
"Allows only HTTP paths matching regular expressions.",
)

cmd.Flags().StringSliceVar(
&flags.PathExclusions,
"path-exclusions",
nil,
"Removes HTTP paths matching regular expressions.",
)

cmd.Flags().IntVar(
&flags.RandomizedStart,
"randomized-start",
100,
"Probability that the apidump command will start intercepting traffic.",
)
_ = cmd.Flags().MarkHidden("randomized-start")

cmd.Flags().Float64Var(
&flags.RateLimit,
"rate-limit",
apispec.DefaultRateLimit,
"Number of requests per minute to capture.",
)

cmd.Flags().BoolVar(
&flags.SendWitnessPayloads,
"send-witness-payloads",
false,
"Send request and response payloads to Postman",
)
_ = cmd.Flags().MarkHidden("send-witness-payloads")

return flags
}

func ConvertCommonApiDumpFlagsToArgs(flags CommonApidumpFlags) []string {
commonApidumpArgs := []string{}

if flags.Filter != "" {
commonApidumpArgs = append(commonApidumpArgs, "--filter", flags.Filter)
}

if flags.RandomizedStart != 100 {
commonApidumpArgs = append(commonApidumpArgs, "--randomized-start", strconv.Itoa(flags.RandomizedStart))
}

if flags.RateLimit != apispec.DefaultRateLimit {
commonApidumpArgs = append(commonApidumpArgs, "--rate-limit", strconv.FormatFloat(flags.RateLimit, 'f', -1, 64))
}

if flags.SendWitnessPayloads {
commonApidumpArgs = append(commonApidumpArgs, "--send-witness-payloads")
}

// Add slice type flags to the entry point.
// Flags: --host-allow, --host-exclusions, --interfaces, --path-allow, --path-exclusions
// Added them separately instead of joining with comma(,) to avoid any regex parsing issues.
for _, host := range flags.HostAllowlist {
commonApidumpArgs = append(commonApidumpArgs, "--host-allow", host)
}
for _, host := range flags.HostExclusions {
commonApidumpArgs = append(commonApidumpArgs, "--host-exclusions", host)
}
for _, interfaceFlag := range flags.Interfaces {
commonApidumpArgs = append(commonApidumpArgs, "--interfaces", interfaceFlag)
}
for _, path := range flags.PathAllowlist {
commonApidumpArgs = append(commonApidumpArgs, "--path-allow", path)
}
for _, path := range flags.PathExclusions {
commonApidumpArgs = append(commonApidumpArgs, "--path-exclusions", path)
}

return commonApidumpArgs
}
11 changes: 10 additions & 1 deletion cmd/internal/ec2/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/AlecAivazis/survey/v2"
"github.com/pkg/errors"
"github.com/postmanlabs/postman-insights-agent/cfg"
"github.com/postmanlabs/postman-insights-agent/cmd/internal/apidump"
"github.com/postmanlabs/postman-insights-agent/consts"
"github.com/postmanlabs/postman-insights-agent/printer"
"github.com/postmanlabs/postman-insights-agent/telemetry"
Expand Down Expand Up @@ -61,7 +62,7 @@ var (

// Helper function for reporting telemetry
func reportStep(stepName string) {
telemetry.WorkflowStep("Starting systemd conguration", stepName)
telemetry.WorkflowStep("Starting systemd configuration", stepName)
}

func setupAgentForServer(projectID string) error {
Expand Down Expand Up @@ -228,10 +229,18 @@ func configureSystemdFiles(projectID string) error {
return err
}

// Get the common apidump args
apidumpArgs := apidump.ConvertCommonApiDumpFlagsToArgs(apidumpFlags)

// Join the extra apidump args to a single string
apidumpArgsStr := strings.Join(apidumpArgs, " ")

serviceFileData := struct {
AgentInstallPath string
ExtraApidumpArgs string
}{
AgentInstallPath: agentInstallPath,
ExtraApidumpArgs: apidumpArgsStr,
}

// Generate and write the service file, with permissions 0600 (read/write for owner only)
Expand Down
7 changes: 6 additions & 1 deletion cmd/internal/ec2/ec2.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ package ec2
import (
"fmt"

"github.com/postmanlabs/postman-insights-agent/cmd/internal/apidump"
"github.com/postmanlabs/postman-insights-agent/cmd/internal/cmderr"
"github.com/spf13/cobra"
)

var (
// Postman Insights project id
projectID string
projectID string
apidumpFlags apidump.CommonApidumpFlags
)

var Cmd = &cobra.Command{
Expand Down Expand Up @@ -44,6 +46,9 @@ func init() {
Cmd.PersistentFlags().StringVar(&projectID, "project", "", "Your Insights Project ID")
Cmd.MarkPersistentFlagRequired("project")

// initialize common apidump flags as flags for the ecs add command
apidumpFlags = apidump.AddCommonApiDumpFlags(Cmd)

Cmd.AddCommand(SetupInEC2Cmd)
Cmd.AddCommand(RemoveFromEC2Cmd)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/internal/ec2/postman-insights-agent.service.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ EnvironmentFile=/etc/default/postman-insights-agent
# DO NOT CHANGE
# "${FOO}" uses the arguement as is, while "$FOO" splits the string on white space
# Reference: https://www.freedesktop.org/software/systemd/man/systemd.service.html#Command%20lines
ExecStart={{.AgentInstallPath}} apidump --project "${PROJECT_ID}" --interfaces "${INTERFACES}" --filter "${FILTER}" "$EXTRA_APIDUMP_ARGS"
ExecStart={{.AgentInstallPath}} apidump --project "${PROJECT_ID}" --interfaces "${INTERFACES}" --filter "${FILTER}" {{.ExtraApidumpArgs}} "${EXTRA_APIDUMP_ARGS}"

[Install]
WantedBy=multi-user.target
Loading

0 comments on commit ac38940

Please sign in to comment.