From 218750f59bf4b28fee840104b9bd58cf801b2b17 Mon Sep 17 00:00:00 2001 From: Pritam Kadam Date: Thu, 10 Jun 2021 17:25:01 +0530 Subject: [PATCH] Retry count for validate links task #488 Selectively retry status codes 500, 502, 503, 504 and SocketTimeoutException up to a configured max retries when validating links. --- .../lightbend/paradox/ParadoxProcessor.scala | 35 +++++++++++++++---- docs/src/main/paradox/validation.md | 8 +++++ .../lightbend/paradox/sbt/ParadoxKeys.scala | 1 + .../lightbend/paradox/sbt/ParadoxPlugin.scala | 2 ++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/com/lightbend/paradox/ParadoxProcessor.scala b/core/src/main/scala/com/lightbend/paradox/ParadoxProcessor.scala index 00c82571..867bc339 100644 --- a/core/src/main/scala/com/lightbend/paradox/ParadoxProcessor.scala +++ b/core/src/main/scala/com/lightbend/paradox/ParadoxProcessor.scala @@ -19,13 +19,14 @@ package com.lightbend.paradox import com.lightbend.paradox.template.PageTemplate import com.lightbend.paradox.markdown._ import com.lightbend.paradox.tree.Tree.{ Forest, Location } + import java.io.{ File, FileOutputStream, OutputStreamWriter } import java.nio.charset.StandardCharsets - -import org.jsoup.Jsoup +import org.jsoup.{ Connection, Jsoup } import org.jsoup.nodes.Document import org.pegdown.ast._ +import java.net.SocketTimeoutException import scala.annotation.tailrec import scala.collection.JavaConverters._ import scala.util.control.NonFatal @@ -108,6 +109,7 @@ class ParadoxProcessor(reader: Reader = new Reader, writer: Writer = new Writer, groups: Map[String, Seq[String]], properties: Map[String, String], ignorePaths: List[Regex], + retryCount: Int, validateAbsolute: Boolean, logger: ParadoxLogger): Int = { @@ -151,7 +153,7 @@ class ParadoxProcessor(reader: Reader = new Reader, writer: Writer = new Writer, reportErrorOnSources(errorCollector, c.allSources)(s"Could not find path [${uri.getPath}] in site") } case absolute if validateAbsolute => - validateExternalLink(absolute, errorCollector, logger) + validateExternalLink(absolute, retryCount, errorCollector, logger) case _ => // Ignore } @@ -160,19 +162,20 @@ class ParadoxProcessor(reader: Reader = new Reader, writer: Writer = new Writer, errorCollector.errorCount } - private def validateExternalLink(capturedLink: CapturedLink, errorContext: ErrorContext, logger: ParadoxLogger) = { + private def validateExternalLink(capturedLink: CapturedLink, retryCount: Int, errorContext: ErrorContext, logger: ParadoxLogger) = { logger.info(s"Validating external link: ${capturedLink.link}") def reportError = reportErrorOnSources(errorContext, capturedLink.allSources)(_) val url = capturedLink.link.toString try { - val response = Jsoup.connect(url) + val request = Jsoup.connect(url) .userAgent("Paradox Link Validator ") .followRedirects(false) .ignoreHttpErrors(true) .ignoreContentType(true) - .execute() + + val response = Validator.validateWithRetries(request, retryCount) // jsoup doesn't offer any simple way to clean up, the only way to close is to get the body stream and close it, // but if you've already read the response body, that will throw an exception, and there's no way to check if @@ -493,3 +496,23 @@ class ParadoxProcessor(reader: Reader = new Reader, writer: Writer = new Writer, } } + +object Validator { + + //500 Internal Server Error + //502 Bad Gateway + //503 Service Unavailable + //504 Gateway Timeout + private val retryableStatusCodes = Set(500, 502, 503, 504) + + def validateWithRetries(request: Connection, retryCount: Int): Connection.Response = + try { + val res = request.execute() + if (retryCount == 0 || res.statusCode() == 200 || !retryableStatusCodes.contains(res.statusCode())) res + else validateWithRetries(request, retryCount - 1) + } + catch { + case e: SocketTimeoutException => if(retryCount == 0) throw e else validateWithRetries(request, retryCount -1) + } + +} \ No newline at end of file diff --git a/docs/src/main/paradox/validation.md b/docs/src/main/paradox/validation.md index e4efa7aa..bafd9b74 100644 --- a/docs/src/main/paradox/validation.md +++ b/docs/src/main/paradox/validation.md @@ -53,3 +53,11 @@ paradoxValidationIgnorePaths ++= Seq( "/docs/version/(?!latest).*" ) ``` + +## Retrying links check + +`paradoxValidateLinksRetryCount` setting allows retrying link check for the provided number of times in case linked server does not reply, or returns a possibly temporary failure response code (500, 502, 503 or 504). + +```scala +paradoxValidateLinksRetryCount := 3 // retries link check 3 times in case of non 200 response code +``` diff --git a/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxKeys.scala b/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxKeys.scala index 1696385c..b615d87a 100644 --- a/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxKeys.scala +++ b/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxKeys.scala @@ -52,6 +52,7 @@ trait ParadoxKeys { val paradoxBrowse = taskKey[Unit]("Open the docs in the default browser") val paradoxValidateInternalLinks = taskKey[Unit]("Validate internal, non ref paradox links.") val paradoxValidateLinks = taskKey[Unit]("Validate all non ref paradox links.") + val paradoxValidateLinksRetryCount = taskKey[Int]("Number of retries for validate links task.") val paradoxValidationIgnorePaths = settingKey[List[Regex]]("List of regular expressions to apply to paths to determine if they should be ignored.") val paradoxValidationSiteBasePath = settingKey[Option[String]]("The base path that the documentation is deployed to, allows validating links on the docs site that are outside of the documentation root tree") val paradoxSingle = taskKey[File]("Build the single page HTML Paradox site") diff --git a/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxPlugin.scala b/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxPlugin.scala index c894d652..bdb3b9b3 100644 --- a/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxPlugin.scala +++ b/plugin/src/main/scala/com/lightbend/paradox/sbt/ParadoxPlugin.scala @@ -65,6 +65,7 @@ object ParadoxPlugin extends AutoPlugin { paradoxLeadingBreadcrumbs := Nil, paradoxGroups := Map.empty, libraryDependencies ++= paradoxTheme.value.toSeq map (_ % ParadoxTheme), + paradoxValidateLinksRetryCount := 0, paradoxValidationIgnorePaths := List("http://localhost.*".r), paradoxValidationSiteBasePath := None ) @@ -378,6 +379,7 @@ object ParadoxPlugin extends AutoPlugin { paradoxGroups.value, paradoxProperties.value, paradoxValidationIgnorePaths.value, + paradoxValidateLinksRetryCount.value, validateAbsolute, new SbtParadoxLogger(strms.log) )