1.8.3

* Fix request timeout issue
pambrose · Sep 1, 2020 · ea2cee8 · ea2cee8
1 parent 38ea83e
commit ea2cee8
Show file tree

Hide file tree

Showing 14 changed files with 463 additions and 415 deletions.
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-VERSION=1.8.1
+VERSION=1.8.3
 
 default: compile
 

diff --git a/README.md b/README.md
@@ -97,8 +97,8 @@ scrape_configs:
 
 The docker images are available via:
 ```bash
-docker pull pambrose/prometheus-proxy:1.8.1
-docker pull pambrose/prometheus-agent:1.8.1
+docker pull pambrose/prometheus-proxy:1.8.3
+docker pull pambrose/prometheus-agent:1.8.3
 ```
 
 Start a proxy container with:
@@ -107,15 +107,15 @@ Start a proxy container with:
 docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \
         --env ADMIN_ENABLED=true \
         --env METRICS_ENABLED=true \
-        pambrose/prometheus-proxy:1.8.1
+        pambrose/prometheus-proxy:1.8.3
 ```
 
 Start an agent container with:
 
 ```bash
 docker run --rm -p 8083:8083 -p 8093:8093 \
         --env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
-        pambrose/prometheus-agent:1.8.1
+        pambrose/prometheus-agent:1.8.3
 ```
 
 Using the config file [simple.conf](https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf),
@@ -131,7 +131,7 @@ is in your current directory, run an agent container with:
 docker run --rm -p 8083:8083 -p 8093:8093 \
     --mount type=bind,source="$(pwd)"/prom-agent.conf,target=/app/prom-agent.conf \
     --env AGENT_CONFIG=prom-agent.conf \
-    pambrose/prometheus-agent:1.8.1
+    pambrose/prometheus-agent:1.8.3
 ```
 
 **Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure 
@@ -252,15 +252,15 @@ docker run --rm -p 8082:8082 -p 8092:8092 -p 50440:50440 -p 8080:8080 \
     --env PROXY_CONFIG=tls-no-mutual-auth.conf \
     --env ADMIN_ENABLED=true \
     --env METRICS_ENABLED=true \
-    pambrose/prometheus-proxy:1.8.1
+    pambrose/prometheus-proxy:1.8.3
 
 docker run --rm -p 8083:8083 -p 8093:8093 \
     --mount type=bind,source="$(pwd)"/testing/certs,target=/app/testing/certs \
     --mount type=bind,source="$(pwd)"/examples/tls-no-mutual-auth.conf,target=/app/tls-no-mutual-auth.conf \
     --env AGENT_CONFIG=tls-no-mutual-auth.conf \
     --env PROXY_HOSTNAME=mymachine.lan:50440 \
     --name docker-agent \
-    pambrose/prometheus-agent:1.8.1
+    pambrose/prometheus-agent:1.8.3
 ```
 
 **Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure 

diff --git a/bin/docker-agent.sh b/bin/docker-agent.sh
@@ -3,4 +3,4 @@
 docker run --rm -p 8083:8083 -p 8093:8093 \
   --env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
   --env PROXY_HOSTNAME=mymachine.lan \
-  pambrose/prometheus-agent:1.8.1
+  pambrose/prometheus-agent:1.8.3
diff --git a/bin/docker-proxy.sh b/bin/docker-proxy.sh
@@ -2,4 +2,4 @@
 
 docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \
   --env PROXY_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
-  pambrose/prometheus-proxy:1.8.1
+  pambrose/prometheus-proxy:1.8.3
diff --git a/build.gradle b/build.gradle
@@ -11,7 +11,7 @@ plugins {
 }
 
 group = 'io.prometheus'
-version = '1.8.1'
+version = '1.8.3'
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

diff --git a/etc/compose/proxy.yml b/etc/compose/proxy.yml
@@ -1,6 +1,6 @@
 prometheus-proxy:
   autoredeploy: true
-  image: 'pambrose/prometheus-proxy:1.8.1'
+  image: 'pambrose/prometheus-proxy:1.8.3'
   ports:
     - '8080:8080'
     - '8082:8082'

diff --git a/gradle.properties b/gradle.properties
@@ -21,5 +21,5 @@ prometheus_version=0.9.0
 protoc_version=3.12.4
 slf4j_version=1.7.30
 typesafe_version=1.4.0
-utils_version=1.3.0
+utils_version=35fcd6c
 zipkin_version=5.12.5
diff --git a/src/main/java/io/prometheus/common/ConfigVals.java b/src/main/java/io/prometheus/common/ConfigVals.java
diff --git a/src/main/kotlin/io/prometheus/Agent.kt b/src/main/kotlin/io/prometheus/Agent.kt
@@ -50,7 +50,7 @@ import kotlin.time.TimeSource.Monotonic
 import kotlin.time.milliseconds
 import kotlin.time.seconds
 
-@Version(version = "1.8.1", date = "8/31/20")
+@Version(version = "1.8.3", date = "9/1/20")
 class Agent(val options: AgentOptions,
             inProcessServerName: String = "",
             testMode: Boolean = false,
@@ -100,7 +100,7 @@ class Agent(val options: AgentOptions,
 
     logger.info { "Agent name: $agentName" }
     logger.info { "Proxy reconnect pause time: ${agentConfigVals.reconnectPauseSecs.seconds}" }
-    logger.info { "Scrape timeout time: ${configVals.agent.scrapeTimeoutSecs.seconds}" }
+    logger.info { "Scrape timeout time: ${options.scrapeTimeoutSecs.seconds}" }
 
     initService {
       if (options.debugEnabled) {

diff --git a/src/main/kotlin/io/prometheus/Proxy.kt b/src/main/kotlin/io/prometheus/Proxy.kt
@@ -45,7 +45,7 @@ import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import kotlin.time.milliseconds
 
-@Version(version = "1.8.1", date = "8/31/20")
+@Version(version = "1.8.3", date = "9/1/20")
 class Proxy(val options: ProxyOptions,
             proxyHttpPort: Int = options.proxyHttpPort,
             inProcessServerName: String = "",

diff --git a/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt b/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt
@@ -19,6 +19,7 @@
 package io.prometheus.agent
 
 import com.github.pambrose.common.dsl.KtorDsl.get
+import com.github.pambrose.common.dsl.KtorDsl.newHttpClient
 import com.github.pambrose.common.dsl.KtorDsl.withHttpClient
 import com.github.pambrose.common.util.isNull
 import com.github.pambrose.common.util.simpleClassName
@@ -28,12 +29,15 @@ import io.ktor.client.features.*
 import io.ktor.client.request.*
 import io.ktor.client.statement.*
 import io.ktor.http.*
+import io.ktor.network.sockets.*
 import io.prometheus.Agent
 import io.prometheus.common.ScrapeResults
 import io.prometheus.grpc.ScrapeRequest
+import kotlinx.coroutines.TimeoutCancellationException
 import mu.KLogging
 import java.io.IOException
 import java.net.URLDecoder
+import java.net.http.HttpConnectTimeoutException
 import java.util.concurrent.atomic.AtomicReference
 import kotlin.time.seconds
 
@@ -65,20 +69,49 @@ internal class AgentHttpService(val agent: Agent) {
         if (encodedQueryParams.isNotEmpty())
           logger.debug { "URL: $url" }
 
-
         // Content is fetched here
         try {
-          withHttpClient {
-            get(url,
-                {
-                  val accept: String? = request.accept
-                  if (accept?.isNotEmpty() == true) header(HttpHeaders.ACCEPT, accept)
-                  val scrapeTimeout = agent.options.scrapeTimeoutSecs.seconds
-                  timeout { requestTimeoutMillis = scrapeTimeout.toLongMilliseconds() }
-                },
-                getBlock(url, scrapeResults, scrapeMsg, request.debugEnabled))
+          newHttpClient {
+            requestTimeout = 90.seconds.toLongMilliseconds()
+          }.use { client ->
+            withHttpClient(client) {
+              get(url,
+                  {
+                    val accept: String? = request.accept
+                    if (accept?.isNotEmpty() == true)
+                      header(HttpHeaders.ACCEPT, accept)
+                    val scrapeTimeout = agent.options.scrapeTimeoutSecs.seconds
+                    logger.debug { "Setting scrapeTimeoutSecs = $scrapeTimeout" }
+                    timeout { requestTimeoutMillis = scrapeTimeout.toLongMilliseconds() }
+                  },
+                  getBlock(url, scrapeResults, scrapeMsg, request.debugEnabled))
+            }
           }
         }
+        catch (e: TimeoutCancellationException) {
+          logger.warn(e) { "fetchScrapeUrl() $e - $url" }
+          scrapeResults.statusCode = HttpStatusCode.RequestTimeout.value
+          scrapeResults.failureReason = e.message ?: e.simpleClassName
+
+          if (request.debugEnabled)
+            scrapeResults.setDebugInfo(url, "${e.simpleClassName} - ${e.message}")
+        }
+        catch (e: HttpConnectTimeoutException) {
+          logger.warn(e) { "fetchScrapeUrl() $e - $url" }
+          scrapeResults.statusCode = HttpStatusCode.RequestTimeout.value
+          scrapeResults.failureReason = e.message ?: e.simpleClassName
+
+          if (request.debugEnabled)
+            scrapeResults.setDebugInfo(url, "${e.simpleClassName} - ${e.message}")
+        }
+        catch (e: SocketTimeoutException) {
+          logger.warn(e) { "fetchScrapeUrl() $e - $url" }
+          scrapeResults.statusCode = HttpStatusCode.RequestTimeout.value
+          scrapeResults.failureReason = e.message ?: e.simpleClassName
+
+          if (request.debugEnabled)
+            scrapeResults.setDebugInfo(url, "${e.simpleClassName} - ${e.message}")
+        }
         catch (e: HttpRequestTimeoutException) {
           logger.warn(e) { "fetchScrapeUrl() $e - $url" }
           scrapeResults.statusCode = HttpStatusCode.RequestTimeout.value

diff --git a/src/main/kotlin/io/prometheus/agent/AgentOptions.kt b/src/main/kotlin/io/prometheus/agent/AgentOptions.kt
@@ -69,54 +69,55 @@ class AgentOptions(argv: Array<String>, exitOnMissingConfig: Boolean) :
 
   override fun assignConfigVals() {
 
-    configVals.agent.also { agent ->
-
-      if (proxyHostname.isEmpty()) {
-        val configHostname = agent.proxy.hostname
-        proxyHostname = PROXY_HOSTNAME.getEnv(if (":" in configHostname)
-                                                configHostname
-                                              else
-                                                "$configHostname:${agent.proxy.port}")
+    configVals.agent
+      .also { agentConfigVals ->
+
+        if (proxyHostname.isEmpty()) {
+          val configHostname = agentConfigVals.proxy.hostname
+          proxyHostname = PROXY_HOSTNAME.getEnv(if (":" in configHostname)
+                                                  configHostname
+                                                else
+                                                  "$configHostname:${agentConfigVals.proxy.port}")
+        }
+        logger.info { "proxyHostname: $proxyHostname" }
+
+
+        if (agentName.isEmpty())
+          agentName = AGENT_NAME.getEnv(agentConfigVals.name)
+        logger.info { "agentName: $agentName" }
+
+        if (!consolidated)
+          consolidated = CONSOLIDATED.getEnv(agentConfigVals.consolidated)
+        logger.info { "consolidated: $consolidated" }
+
+        if (scrapeTimeoutSecs == -1)
+          scrapeTimeoutSecs = SCRAPE_TIMEOUT_SECS.getEnv(agentConfigVals.scrapeTimeoutSecs)
+        logger.info { "scrapeTimeoutSecs: ${scrapeTimeoutSecs.seconds}" }
+
+        if (chunkContentSizeKbs == -1)
+          chunkContentSizeKbs = CHUNK_CONTENT_SIZE_KBS.getEnv(agentConfigVals.chunkContentSizeKbs)
+        // Multiply the value time KB
+        chunkContentSizeKbs *= 1024
+        logger.info { "chunkContentSizeKbs: $chunkContentSizeKbs" }
+
+        if (minGzipSizeBytes == -1)
+          minGzipSizeBytes = MIN_GZIP_SIZE_BYTES.getEnv(agentConfigVals.minGzipSizeBytes)
+        logger.info { "minGzipSizeBytes: $minGzipSizeBytes" }
+
+        if (overrideAuthority.isEmpty())
+          overrideAuthority = OVERRIDE_AUTHORITY.getEnv(agentConfigVals.tls.overrideAuthority)
+        logger.info { "overrideAuthority: $overrideAuthority" }
+
+        assignAdminEnabled(agentConfigVals.admin.enabled)
+        assignAdminPort(agentConfigVals.admin.port)
+        assignMetricsEnabled(agentConfigVals.metrics.enabled)
+        assignMetricsPort(agentConfigVals.metrics.port)
+        assignDebugEnabled(agentConfigVals.admin.debugEnabled)
+
+        assignCertChainFilePath(agentConfigVals.tls.certChainFilePath)
+        assignPrivateKeyFilePath(agentConfigVals.tls.privateKeyFilePath)
+        assignTrustCertCollectionFilePath(agentConfigVals.tls.trustCertCollectionFilePath)
       }
-      logger.info { "proxyHostname: $proxyHostname" }
-
-
-      if (agentName.isEmpty())
-        agentName = AGENT_NAME.getEnv(agent.name)
-      logger.info { "agentName: $agentName" }
-
-      if (!consolidated)
-        consolidated = CONSOLIDATED.getEnv(agent.consolidated)
-      logger.info { "consolidated: $consolidated" }
-
-      if (scrapeTimeoutSecs == -1)
-        scrapeTimeoutSecs = SCRAPE_TIMEOUT_SECS.getEnv(agent.scrapeTimeoutSecs)
-      logger.info { "scrapeTimeoutSecs: ${scrapeTimeoutSecs.seconds}" }
-
-      if (chunkContentSizeKbs == -1)
-        chunkContentSizeKbs = CHUNK_CONTENT_SIZE_KBS.getEnv(agent.chunkContentSizeKbs)
-      // Multiply the value time KB
-      chunkContentSizeKbs *= 1024
-      logger.info { "chunkContentSizeKbs: $chunkContentSizeKbs" }
-
-      if (minGzipSizeBytes == -1)
-        minGzipSizeBytes = MIN_GZIP_SIZE_BYTES.getEnv(agent.minGzipSizeBytes)
-      logger.info { "minGzipSizeBytes: $minGzipSizeBytes" }
-
-      if (overrideAuthority.isEmpty())
-        overrideAuthority = OVERRIDE_AUTHORITY.getEnv(agent.tls.overrideAuthority)
-      logger.info { "overrideAuthority: $overrideAuthority" }
-
-      assignAdminEnabled(agent.admin.enabled)
-      assignAdminPort(agent.admin.port)
-      assignMetricsEnabled(agent.metrics.enabled)
-      assignMetricsPort(agent.metrics.port)
-      assignDebugEnabled(agent.admin.debugEnabled)
-
-      assignCertChainFilePath(agent.tls.certChainFilePath)
-      assignPrivateKeyFilePath(agent.tls.privateKeyFilePath)
-      assignTrustCertCollectionFilePath(agent.tls.trustCertCollectionFilePath)
-    }
   }
 
   companion object : KLogging()

diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt b/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt
@@ -32,6 +32,7 @@ import io.ktor.response.*
 import io.ktor.routing.*
 import io.prometheus.Proxy
 import kotlinx.coroutines.async
+import kotlinx.coroutines.delay
 import mu.KLogging
 import org.slf4j.event.Level
 import kotlin.time.Duration
@@ -84,6 +85,10 @@ internal object ProxyHttpConfig : KLogging() {
     }
 
     routing {
+      get("/__test__") {
+        delay(30.seconds)
+        call.respondWith("Test value", Plain, OK)
+      }
       get("/*") {
         call.response.header(HttpHeaders.CacheControl, "must-revalidate,no-cache,no-store")
 

diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt b/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt
@@ -50,16 +50,17 @@ class ProxyOptions(argv: Array<String>) : BaseOptions(Proxy::class.java.simpleNa
       proxyAgentPort = AGENT_PORT.getEnv(configVals.proxy.agent.port)
     logger.info { "proxyAgentPort: $proxyAgentPort" }
 
-    configVals.proxy.also { proxy ->
-      assignAdminEnabled(proxy.admin.enabled)
-      assignAdminPort(proxy.admin.port)
-      assignMetricsEnabled(proxy.metrics.enabled)
-      assignMetricsPort(proxy.metrics.port)
-      assignDebugEnabled(proxy.admin.debugEnabled)
+    configVals.proxy
+      .also { proxyConfigVals ->
+        assignAdminEnabled(proxyConfigVals.admin.enabled)
+        assignAdminPort(proxyConfigVals.admin.port)
+        assignMetricsEnabled(proxyConfigVals.metrics.enabled)
+        assignMetricsPort(proxyConfigVals.metrics.port)
+        assignDebugEnabled(proxyConfigVals.admin.debugEnabled)
 
-      assignCertChainFilePath(proxy.tls.certChainFilePath)
-      assignPrivateKeyFilePath(proxy.tls.privateKeyFilePath)
-      assignTrustCertCollectionFilePath(proxy.tls.trustCertCollectionFilePath)
-    }
+        assignCertChainFilePath(proxyConfigVals.tls.certChainFilePath)
+        assignPrivateKeyFilePath(proxyConfigVals.tls.privateKeyFilePath)
+        assignTrustCertCollectionFilePath(proxyConfigVals.tls.trustCertCollectionFilePath)
+      }
   }
 }