chore: Update go pot config

cdk/go.sum

@@ -12,14 +12,19 @@ github.com/cdklabs/awscdk-asset-kubectl-go/kubectlv20/v2 v2.1.2 h1:k+WD+6cERd59M
 github.com/cdklabs/awscdk-asset-kubectl-go/kubectlv20/v2 v2.1.2/go.mod h1:CvFHBo0qcg8LUkJqIxQtP1rD/sNGv9bX3L2vHT2FUAo=
 github.com/cdklabs/awscdk-asset-node-proxy-agent-go/nodeproxyagentv6/v2 v2.0.1 h1:MBBQNKKPJ5GArbctgwpiCy7KmwGjHDjUUH5wEzwIq8w=
 github.com/cdklabs/awscdk-asset-node-proxy-agent-go/nodeproxyagentv6/v2 v2.0.1/go.mod h1:/2WiXEft9s8ViJjD01CJqDuyJ8HXBjhBLtK5OvJfdSc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -32,6 +37,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -43,3 +50,5 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

cdk/stack.go

@@ -45,6 +45,142 @@ func NewPotStackStack(scope constructs.Construct, id string, props *TestStackPro
 		},
 	})
 
+	// EC2 Prometheus Push Gateway
+	pushGatewaySg := awsec2.NewSecurityGroup(stack, jsii.String("PushGatewaySecurityGroup"), &awsec2.SecurityGroupProps{
+		Vpc: vpc,
+	})
+	pushGatewaySg.AddIngressRule(awsec2.Peer_AnyIpv4(), awsec2.Port_Tcp(jsii.Number(9092)), jsii.String("Ingress from prometheus (Internet)"), jsii.Bool(false))
+	pushGatewaySg.AddIngressRule(awsec2.Peer_AnyIpv4(), awsec2.Port_Tcp(jsii.Number(9093)), jsii.String("Ingress from prometheus (Internet)"), jsii.Bool(false))
+
+	pushGateway := awsec2.NewInstance(stack, jsii.String("PrometheusMetricsNode"), &awsec2.InstanceProps{
+		InstanceType: awsec2.NewInstanceType(jsii.String("t3.micro")),
+		MachineImage: awsec2.NewAmazonLinuxImage(&awsec2.AmazonLinuxImageProps{
+			Generation: awsec2.AmazonLinuxGeneration_AMAZON_LINUX_2,
+		}),
+		Vpc: vpc,
+		VpcSubnets: &awsec2.SubnetSelection{
+			SubnetType: awsec2.SubnetType_PUBLIC,
+		},
+		SecurityGroup: pushGatewaySg,
+	})
+
+	//ssmData := awsssm.StringParameter_FromStringParameterName(stack, jsii.String("SsmGrafanaCloud"), jsii.String("/ryan-pot/grafana-cloud-key"))
+	//ssmData.GrantRead(pushGateway.Role())
+	pushGateway.UserData().AddCommands(
+		// Install SSM Agent
+		jsii.String("sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm"),
+		jsii.String("sudo systemctl enable amazon-ssm-agent"),
+		jsii.String("sudo systemctl start amazon-ssm-agent"),
+
+		// Install utils
+		jsii.String("sudo yum install -y gettext envsubst"),
+
+		// Setup Prometheus Push Gateway
+		jsii.String("sudo useradd -M -r -s /bin/false pushgateway"),
+		jsii.String("wget https://github.com/prometheus/pushgateway/releases/download/v1.2.0/pushgateway-1.2.0.linux-amd64.tar.gz"),
+		jsii.String("tar xvfz pushgateway-1.2.0.linux-amd64.tar.gz"),
+		jsii.String("sudo cp pushgateway-1.2.0.linux-amd64/pushgateway /usr/local/bin/"),
+		jsii.String("sudo chown pushgateway:pushgateway /usr/local/bin/pushgateway"),
+		jsii.String(`echo "[Unit]
+Description=Prometheus Pushgateway
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=pushgateway
+Group=pushgateway
+Type=simple
+ExecStart=/usr/local/bin/pushgateway
+[Install]
+WantedBy=multi-user.target" > /etc/systemd/system/pushgateway.service`),
+		jsii.String("sudo systemctl enable pushgateway"),
+		jsii.String("sudo systemctl start pushgateway"),
+
+		// Install prometheus
+		jsii.String("sudo useradd --no-create-home --shell /bin/false prometheus"),
+		jsii.String("sudo mkdir /etc/prometheus /var/lib/prometheus"),
+		jsii.String("sudo chown prometheus:prometheus /etc/prometheus /var/lib/prometheus"),
+		jsii.String("cd ~"),
+		jsii.String("curl -LO https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz"),
+		jsii.String("tar -xvf prometheus-2.45.1.linux-amd64.tar.gz"),
+		jsii.String("sudo cp -p ./prometheus-2.45.1.linux-amd64/prometheus /usr/local/bin"),
+		jsii.String("sudo chown prometheus:prometheus /usr/local/bin/prom*"),
+		jsii.String("sudo cp -r ./prometheus-2.45.1.linux-amd64/consoles /etc/prometheus"),
+		jsii.String("sudo cp -r ./prometheus-2.45.1.linux-amd64/console_libraries /etc/prometheus"),
+		jsii.String("sudo chown -R prometheus:prometheus /etc/prometheus/consoles /etc/prometheus/console_libraries"),
+		jsii.String(`echo "global:
+  scrape_interval: 1m
+  evaluation_interval: 1m
+  scrape_timeout: 2s
+scrape_configs:
+- job_name: push_gateway
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets: ['localhost:9091']
+    labels:
+      service: 'prom-pushgateway'
+" > /etc/prometheus/prometheus.yml`),
+		// Pull config from SSM
+		//awscdk.Fn_Sub(jsii.String("aws ssm get-parameter --region ${REGION} --name ${NAME} --with-decryption --query Parameter.Value --output text >> /etc/prometheus/prometheus.yml"), &map[string]*string{
+		//	"REGION": props.Env.Region,
+		//	"NAME":   ssmData.ParameterName(),
+		//}),
+
+		jsii.String(`echo "[Unit]
+Description=PromServer
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=prometheus
+Group=prometheus
+Type=simple
+ExecStart=/usr/local/bin/prometheus \
+--config.file /etc/prometheus/prometheus.yml \
+--storage.tsdb.path /var/lib/prometheus/ \
+--web.console.templates=/etc/prometheus/consoles \
+--web.console.libraries=/etc/prometheus/console_libraries
+
+[Install]
+WantedBy=multi-user.target" > /etc/systemd/system/prometheus.service`),
+		jsii.String("sudo systemctl daemon-reload"),
+		jsii.String("sudo systemctl enable prometheus"),
+		jsii.String("sudo systemctl start prometheus"),
+
+		// Setup and install nginx
+		jsii.String("sudo amazon-linux-extras install nginx1 -y"),
+		jsii.String("sudo chkconfig nginx on"),
+		jsii.String("sudo service nginx start"),
+		jsii.String(`sudo echo "server {
+	listen *:9092;
+		location / {
+			auth_basic             "Restricted";
+			auth_basic_user_file   .htpasswd;
+
+			proxy_pass              http://localhost:9090;
+		}
+	}
+	server {
+	listen *:9093;
+		location / {
+			auth_basic             "Restricted";
+			auth_basic_user_file   .htpasswd;
+
+			proxy_pass              http://localhost:9091;
+		}
+	}" > /etc/nginx/conf.d/pushgateway.conf`),
+		jsii.String(`sudo yum install httpd-tools -y`),
+		awscdk.Fn_Sub(jsii.String("sudo htpasswd -c -b /etc/nginx/.htpasswd ${USERNAME} ${PASSWORD}"), &map[string]*string{
+			"USERNAME": jsii.String("ryan-pot"),
+			// @todo Pull this from Secrets Manager
+			"PASSWORD": jsii.String("SOME_SECRET"),
+		}),
+		jsii.String("sudo service nginx restart"),
+	)
+
+	pushGateway.Role().AddManagedPolicy(awsiam.ManagedPolicy_FromAwsManagedPolicyName(jsii.String("AmazonSSMManagedInstanceCore")))
+
 	appImage := awsecrassets.NewDockerImageAsset(stack, jsii.String("EcrAsset"), &awsecrassets.DockerImageAssetProps{
 		Directory: jsii.String(path.Join(filepath.Dir(filename), "..")),
 		Exclude:   jsii.Strings("./cdk"),
@@ -73,6 +209,12 @@ func NewPotStackStack(scope constructs.Construct, id string, props *TestStackPro
 			StreamPrefix: jsii.String("/ryan-pot"),
 		}),
 
+		Environment: &map[string]*string{
+			"PUSH_GATEWAY_ADDRESS": awscdk.Fn_Sub(jsii.String("${PRIVATE_IP}:9091"), &map[string]*string{
+				"PRIVATE_IP": pushGateway.InstancePrivateIp(),
+			}),
+		},
+
 		PortMappings: &[]*awsecs.PortMapping{
 			{
 				ContainerPort: jsii.Number(80),
@@ -94,6 +236,8 @@ func NewPotStackStack(scope constructs.Construct, id string, props *TestStackPro
 		Vpc: vpc,
 	})
 
+	pushGatewaySg.AddIngressRule(serviceSg, awsec2.Port_Tcp(jsii.Number(9091)), jsii.String("Allow Prometheus Push Gateway traffic"), jsii.Bool(false))
+
 	serviceSg.AddIngressRule(awsec2.Peer_AnyIpv4(), awsec2.Port_Tcp(jsii.Number(80)), jsii.String("Allow HTTP traffic from anywhere"), jsii.Bool(false))
 	serviceSg.AddIngressRule(awsec2.Peer_Ipv4(jsii.String("172.31.0.0/24")), awsec2.Port_AllTraffic(), jsii.String("Allow internal traffic"), jsii.Bool(false))
 	awsecs.NewFargateService(stack, jsii.String("EcsService"), &awsecs.FargateServiceProps{
@@ -106,7 +250,7 @@ func NewPotStackStack(scope constructs.Construct, id string, props *TestStackPro
 		},
 		VpcSubnets:        &awsec2.SubnetSelection{SubnetType: awsec2.SubnetType_PUBLIC},
 		TaskDefinition:    taskDefinition,
-		DesiredCount:      jsii.Number(4),
+		DesiredCount:      jsii.Number(8),
 		AssignPublicIp:    jsii.Bool(true),
 		MaxHealthyPercent: jsii.Number(200),
 		MinHealthyPercent: jsii.Number(0),
@@ -141,6 +285,15 @@ func main() {
 		},
 	})
 
+	NewPotStackStack(app, "GoPotStack-US", &TestStackProps{
+		awscdk.StackProps{
+			Env: &awscdk.Environment{
+				Account: jsii.String("849652302708"),
+				Region:  jsii.String("us-east-1"),
+			},
+		},
+	})
+
 	app.Synth(nil)
 }
 
@@ -149,6 +302,6 @@ func main() {
 func env() *awscdk.Environment {
 	return &awscdk.Environment{
 		Account: jsii.String("849652302708"),
-		Region:  jsii.String("eu-west-1"),
+		Region:  jsii.String("us-east-1"),
 	}
 }

docker-compose.yml

@@ -13,6 +13,7 @@ services:
     environment:
       - DOCKER_IP_ADDRESS=10.5.0.4
       - DOCKER_KNOWN_PEERS=10.5.0.5
+      - PUSH_GATEWAY_ADDRESS=http://10.5.0.9:9091
     networks:
       ryan-pot-network:
         ipv4_address: 10.5.0.4
@@ -30,6 +31,7 @@ services:
     environment:
       - DOCKER_IP_ADDRESS=10.5.0.5
       - DOCKER_KNOWN_PEERS=10.5.0.6
+      - PUSH_GATEWAY_ADDRESS=http://10.5.0.9:9091
     networks:
       ryan-pot-network:
         ipv4_address: 10.5.0.5
@@ -47,9 +49,48 @@ services:
     environment:
       - DOCKER_IP_ADDRESS=10.5.0.6
       - DOCKER_KNOWN_PEERS=10.5.0.4
+      - PUSH_GATEWAY_ADDRESS=http://10.5.0.9:9091
     networks:
       ryan-pot-network:
         ipv4_address: 10.5.0.6
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - 9090:9090
+    volumes:
+      - ./docker/prometheus:/etc/prometheus
+    networks:
+      ryan-pot-network:
+        ipv4_address: 10.5.0.7
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=grafana
+    volumes:
+      - ./docker/grafana:/etc/grafana/provisioning/datasources
+    networks:
+      ryan-pot-network:
+        ipv4_address: 10.5.0.8
+
+  prom_pushgateway:
+    image: prom/pushgateway
+    container_name: prom_pushgateway
+    ports: 
+      - 9091:9091
+
+    networks:
+      ryan-pot-network:
+        ipv4_address: 10.5.0.9
+
 networks:
   ryan-pot-network:
     driver: bridge

docker/grafana/datasource.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  url: http://10.5.0.7:9090 
+  isDefault: true
+  access: proxy
+  editable: true

docker/prometheus/prometheus.yml

@@ -0,0 +1,12 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 2s
+  evaluation_interval: 15s
+scrape_configs:
+- job_name: push_gateway
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets: ['10.5.0.9:9091']
+    labels:
+      service: 'prom-pushgateway'

go.mod

@@ -4,11 +4,15 @@ go 1.21.0
 
 require (
 	github.com/BurntSushi/toml v1.3.2
+	github.com/aws/aws-sdk-go-v2/config v1.22.0
+	github.com/aws/aws-sdk-go-v2/service/ecs v1.32.0
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20221221133751-67e37ae746cd
 	github.com/gofiber/contrib/fiberzap/v2 v2.1.1
 	github.com/gofiber/fiber/v2 v2.49.2
 	github.com/hashicorp/hcl/v2 v2.19.1
+	github.com/hashicorp/memberlist v0.5.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/prometheus/client_golang v1.17.0
 	github.com/ryanolee/go-chaff v0.0.1
 	github.com/spf13/cobra v1.7.0
 	github.com/thoas/go-funk v0.9.3
@@ -26,46 +30,51 @@ require (
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
 	github.com/aws/aws-sdk-go-v2 v1.22.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.22.0 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.15.1 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.5.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ecs v1.32.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.17.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.25.0 // indirect
 	github.com/aws/smithy-go v1.16.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/go-faker/faker/v4 v4.2.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
 	github.com/hashicorp/go-msgpack v0.5.3 // indirect
 	github.com/hashicorp/go-multierror v1.0.0 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.0 // indirect
-	github.com/hashicorp/memberlist v0.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/miekg/dns v1.1.26 // indirect
 	github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.50.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 )