diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index c42b2d9..92e2682 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: ['8', '17'] + java: ['11', '17'] steps: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -34,7 +34,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: 8 + java-version: 11 - name: Check Binary Compatibility run: ./mill -i __.mimaReportBinaryIssues @@ -55,7 +55,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: 8 + java-version: 11 - name: Publish to Maven Central run: | if [[ $(git tag --points-at HEAD) != '' ]]; then diff --git a/readme.md b/readme.md index 9bf68e8..a5b26c7 100644 --- a/readme.md +++ b/readme.md @@ -660,11 +660,9 @@ know. As it turns out, Kenneth Reitz's Requests is [not a lot of code](https://github.com/requests/requests/tree/main/requests). Most of the heavy lifting is done in other libraries, and his library is a just -thin-shim that makes the API 10x better. It turns out on the JVM most of the -heavy lifting is also done for you, by `java.net.HttpUrlConnection` in the -simplest case, and other libraries like -[AsyncHttpClient](https://github.com/AsyncHttpClient/async-http-client) for more -advanced use cases. +thin-shim that makes the API 10x better. Similarly, it turns out on the JVM most of the +heavy lifting is also done for you. There have always been options, but +since JDK 11 a decent HTTP client is provided in the standard library. Given that's the case, how hard can it be to port over a dozen Python files to Scala? This library attempts to do that: class by class, method by method, diff --git a/requests/src/requests/Model.scala b/requests/src/requests/Model.scala index 4d7c679..6ba7135 100644 --- a/requests/src/requests/Model.scala +++ b/requests/src/requests/Model.scala @@ -64,8 +64,8 @@ case class Request(url: String, /** * Represents the different things you can upload in the body of a HTTP - * request. By default allows form-encoded key-value pairs, arrays of bytes, - * strings, files, and inputstreams. These types can be passed directly to + * request. By default, allows form-encoded key-value pairs, arrays of bytes, + * strings, files, and InputStreams. These types can be passed directly to * the `data` parameter of [[Requester.apply]] and will be wrapped automatically * by the implicit constructors. */ @@ -76,6 +76,7 @@ trait RequestBlob{ object RequestBlob{ object EmptyRequestBlob extends RequestBlob{ def write(out: java.io.OutputStream): Unit = () + override def headers = Seq("Content-Length" -> "0") } implicit class ByteSourceRequestBlob[T](x: T)(implicit f: T => geny.Writable) extends RequestBlob{ @@ -176,18 +177,21 @@ class ResponseBlob(val bytes: Array[Byte]){ /** - * Represents a HTTP response - * - * @param url the URL that the original request was made to - * @param statusCode the status code of the response - * @param statusMessage the status message of the response - * @param headers the raw headers the server sent back with the response - * @param data the response body; may contain HTML, JSON, or binary or textual data - * @param history the response of any redirects that were performed before - * arriving at the current response - */ + * Represents a HTTP response + * + * @param url the URL that the original request was made to + * @param statusCode the status code of the response + * @param statusMessage a string that describes the status code. + * This is not the reason phrase sent by the server, + * but a string describing [[statusCode]], as hardcoded in this library + * @param headers the raw headers the server sent back with the response + * @param data the response body; may contain HTML, JSON, or binary or textual data + * @param history the response of any redirects that were performed before + * arriving at the current response + */ case class Response(url: String, statusCode: Int, + @deprecated("Value is inferred from `statusCode`", "0.9.0") statusMessage: String, data: geny.Bytes, headers: Map[String, Seq[String]], @@ -222,11 +226,12 @@ case class Response(url: String, f(new java.io.ByteArrayInputStream(data.array)) } override def httpContentType: Option[String] = contentType - override def contentLength: Option[Long] = Some(data.array.size) + override def contentLength: Option[Long] = Some(data.array.length) } case class StreamHeaders(url: String, statusCode: Int, + @deprecated("Value is inferred from `statusCode`", "0.9.0") statusMessage: String, headers: Map[String, Seq[String]], history: Option[Response]){ diff --git a/requests/src/requests/Requester.scala b/requests/src/requests/Requester.scala index 974c7aa..7b4b158 100644 --- a/requests/src/requests/Requester.scala +++ b/requests/src/requests/Requester.scala @@ -1,11 +1,20 @@ package requests -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, OutputStream} -import java.net.{HttpCookie, HttpURLConnection, InetSocketAddress} +import java.io._ +import java.net.http._ +import java.net.{UnknownHostException => _, _} +import java.nio.ByteBuffer +import java.time.Duration +import java.util.concurrent.Flow +import java.util.function.Supplier import java.util.zip.{GZIPInputStream, InflaterInputStream} -import javax.net.ssl._ -import collection.JavaConverters._ + +import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.concurrent.{ExecutionException, Future} + +import javax.net.ssl.SSLContext + trait BaseSession{ def headers: Map[String, String] @@ -39,7 +48,6 @@ object BaseSession{ val defaultHeaders = Map( "User-Agent" -> "requests-scala", "Accept-Encoding" -> "gzip, deflate", - "Connection" -> "keep-alive", "Accept" -> "*/*" ) } @@ -64,7 +72,7 @@ case class Requester(verb: String, * * @param url The URL to which you want to make this HTTP request * @param auth HTTP authentication you want to use with this request; defaults to none - * @param params URL params to pass to this request, for GETs and DELETEs + * @param params URL params to pass to this request, for `GET`s and `DELETE`s * @param headers Custom headers to use, in addition to the defaults * @param data Body data to pass to this request, for POSTs and PUTs. Can be a * Map[String, String] of form data, bulk data as a String or Array[Byte], @@ -103,22 +111,40 @@ case class Requester(verb: String, val out = new ByteArrayOutputStream() var streamHeaders: StreamHeaders = null - val w = stream( - url, auth, params, data.headers, headers, data, readTimeout, - connectTimeout, proxy, cert, sslContext, cookies, cookieValues, maxRedirects, - verifySslCerts, autoDecompress, compress, keepAlive, check, chunkedUpload, - onHeadersReceived = sh => streamHeaders = sh - ) + val w = + stream( + url = url, + auth = auth, + params = params, + blobHeaders = data.headers, + headers = headers, + data = data, + readTimeout = readTimeout, + connectTimeout = connectTimeout, + proxy = proxy, + cert = cert, + sslContext = sslContext, + cookies = cookies, + cookieValues = cookieValues, + maxRedirects = maxRedirects, + verifySslCerts = verifySslCerts, + autoDecompress = autoDecompress, + compress = compress, + keepAlive = keepAlive, + check = check, + chunkedUpload = chunkedUpload, + onHeadersReceived = sh => streamHeaders = sh + ) w.writeBytesTo(out) Response( - streamHeaders.url, - streamHeaders.statusCode, - streamHeaders.statusMessage, - new geny.Bytes(out.toByteArray), - streamHeaders.headers, - streamHeaders.history + url = streamHeaders.url, + statusCode = streamHeaders.statusCode, + statusMessage = streamHeaders.statusMessage, + data = new geny.Bytes(out.toByteArray), + headers = streamHeaders.headers, + history = streamHeaders.history ) } @@ -172,203 +198,206 @@ case class Requester(verb: String, new java.net.URL(url + firstSep + encodedParams) } else url0 - var connection: HttpURLConnection = null + val httpClient: HttpClient = + HttpClient + .newBuilder() + .followRedirects(HttpClient.Redirect.NEVER) + .proxy(proxy match { + case null => ProxySelector.getDefault + case (ip, port) => ProxySelector.of(new InetSocketAddress(ip, port)) + }) + .sslContext( + if (cert != null) + Util.clientCertSSLContext(cert, verifySslCerts) + else if (sslContext != null) + sslContext + else if (!verifySslCerts) + Util.noVerifySSLContext + else + SSLContext.getDefault + ) + .connectTimeout(Duration.ofMillis(connectTimeout)) + .build() + + val sessionCookieValues = for { + c <- (sess.cookies ++ cookies).valuesIterator + if !c.hasExpired + if c.getDomain == null || c.getDomain == url1.getHost + if c.getPath == null || url1.getPath.startsWith(c.getPath) + } yield (c.getName, c.getValue) + + val allCookies = sessionCookieValues ++ cookieValues + + val (contentLengthHeader, otherBlobHeaders) = blobHeaders.partition(_._1.equalsIgnoreCase("Content-Length")) + + val allHeaders = + otherBlobHeaders ++ + sess.headers ++ + headers ++ + compress.headers ++ + auth.header.map("Authorization" -> _) ++ + (if (allCookies.isEmpty) None + else Some("Cookie" -> allCookies + .map { case (k, v) => s"""$k="$v"""" } + .mkString("; ") + )) + val allHeadersFlat = allHeaders.toList.flatMap { case (k, v) => Seq(k, v) } + + val requestBodyInputStream = new PipedInputStream() + val requestBodyOutputStream = new PipedOutputStream(requestBodyInputStream) + + val bodyPublisher: HttpRequest.BodyPublisher = + HttpRequest.BodyPublishers.ofInputStream(new Supplier[InputStream] { + override def get() = requestBodyInputStream + }) + + val requestBuilder = + HttpRequest.newBuilder() + .uri(url1.toURI) + .timeout(Duration.ofMillis(readTimeout)) + .headers(allHeadersFlat: _*) + .method(upperCaseVerb, + (contentLengthHeader.headOption.map(_._2), compress) match { + case (Some("0"), _) => HttpRequest.BodyPublishers.noBody() + case (Some(n), Compress.None) => HttpRequest.BodyPublishers.fromPublisher(bodyPublisher, n.toInt) + case _ => bodyPublisher + } + ) - try { + val fut = httpClient.sendAsync(requestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()) - val conn = - if (proxy == null) url1.openConnection - else { - val (ip, port) = proxy - val p = new java.net.Proxy( - java.net.Proxy.Type.HTTP, new InetSocketAddress(ip, port) - ) - url1.openConnection(p) - } + usingOutputStream(compress.wrap(requestBodyOutputStream)) { os => + data.write(os) + } - connection = conn match{ - case c: HttpsURLConnection => - if (cert != null) { - c.setSSLSocketFactory(Util.clientCertSocketFactory(cert, verifySslCerts)) - if (!verifySslCerts) c.setHostnameVerifier(new HostnameVerifier { def verify(h: String, s: SSLSession) = true }) - } else if (sslContext != null) { - c.setSSLSocketFactory(sslContext.getSocketFactory) - if (!verifySslCerts) c.setHostnameVerifier(new HostnameVerifier { def verify(h: String, s: SSLSession) = true }) - } else if (!verifySslCerts) { - c.setSSLSocketFactory(Util.noVerifySocketFactory) - c.setHostnameVerifier(new HostnameVerifier { def verify(h: String, s: SSLSession) = true }) + val response = + try + fut.get() + catch { + case e: ExecutionException => + throw e.getCause match { + case e: javax.net.ssl.SSLHandshakeException => new InvalidCertException(url, e) + case _: HttpConnectTimeoutException | _: HttpTimeoutException => + new TimeoutException(url, readTimeout, connectTimeout) + case e: java.net.UnknownHostException => + new UnknownHostException(url, e.getMessage) + case e: java.net.ConnectException => + new UnknownHostException(url, e.getMessage) + case e => + new RequestsException(e.getMessage, Some(e)) } - c - case c: HttpURLConnection => c } - connection.setInstanceFollowRedirects(false) - if (Requester.officialHttpMethods.contains(upperCaseVerb)) { - connection.setRequestMethod(upperCaseVerb) - } else { - // HttpURLConnection enforces a list of official http METHODs, but not everyone abides by the spec - // this hack allows us set an unofficial http method - connection match { - case cs: HttpsURLConnection => - cs.getClass.getDeclaredFields.find(_.getName == "delegate").foreach{ del => - del.setAccessible(true) - Requester.methodField.set(del.get(cs), upperCaseVerb) - } - case c => - Requester.methodField.set(c, upperCaseVerb) - } + val responseCode = response.statusCode() + val headerFields = + response.headers().map.asScala + .filter(_._1 != null) + .map { case (k, v) => (k.toLowerCase(), v.asScala.toList) }.toMap + + val deGzip = autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("gzip")) + val deDeflate = + autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("deflate")) + def persistCookies() = { + if (sess.persistCookies) { + headerFields + .get("set-cookie") + .iterator + .flatten + .flatMap(HttpCookie.parse(_).asScala) + .foreach(c => sess.cookies(c.getName) = c) } + } - for((k, v) <- blobHeaders) connection.setRequestProperty(k, v) - - for((k, v) <- sess.headers) connection.setRequestProperty(k, v) - - for((k, v) <- headers) connection.setRequestProperty(k, v) - - for((k, v) <- compress.headers) connection.setRequestProperty(k, v) - - connection.setReadTimeout(readTimeout) - auth.header.foreach(connection.setRequestProperty("Authorization", _)) - connection.setConnectTimeout(connectTimeout) - connection.setUseCaches(false) - connection.setDoOutput(true) - - val sessionCookieValues = for{ - c <- (sess.cookies ++ cookies).valuesIterator - if !c.hasExpired - if c.getDomain == null || c.getDomain == url1.getHost - if c.getPath == null || url1.getPath.startsWith(c.getPath) - } yield (c.getName, c.getValue) - - val allCookies = sessionCookieValues ++ cookieValues - if (allCookies.nonEmpty){ - connection.setRequestProperty( - "Cookie", - allCookies - .map{case (k, v) => s"""$k="$v""""} - .mkString("; ") - ) - } - - if (upperCaseVerb == "POST" || upperCaseVerb == "PUT" || upperCaseVerb == "PATCH" || upperCaseVerb == "DELETE") { - if (!chunkedUpload) { - val bytes = new ByteArrayOutputStream() - usingOutputStream(compress.wrap(bytes)) { os => data.write(os) } - val byteArray = bytes.toByteArray - connection.setFixedLengthStreamingMode(byteArray.length) - usingOutputStream(connection.getOutputStream) { os => os.write(byteArray) } + if (responseCode.toString.startsWith("3") && + responseCode.toString != "304" && + maxRedirects > 0) { + val out = new ByteArrayOutputStream() + Util.transferTo(response.body, out) + val bytes = out.toByteArray + + val current = Response( + url = url, + statusCode = responseCode, + statusMessage = StatusMessages.byStatusCode.getOrElse(responseCode, ""), + data = new geny.Bytes(bytes), + headers = headerFields, + history = redirectedFrom + ) + persistCookies() + val newUrl = current.headers("location").head + stream( + url = new URL(url1, newUrl).toString, + auth = auth, + params = params, + blobHeaders = blobHeaders, + headers = headers, + data = data, + readTimeout = readTimeout, + connectTimeout = connectTimeout, + proxy = proxy, + cert = cert, + sslContext = sslContext, + cookies = cookies, + cookieValues = cookieValues, + maxRedirects = maxRedirects - 1, + verifySslCerts = verifySslCerts, + autoDecompress = autoDecompress, + compress = compress, + keepAlive = keepAlive, + check = check, + chunkedUpload = chunkedUpload, + redirectedFrom = Some(current), + onHeadersReceived = onHeadersReceived + ).readBytesThrough(f) + } else { + persistCookies() + val streamHeaders = StreamHeaders( + url = url, + statusCode = responseCode, + statusMessage = StatusMessages.byStatusCode.getOrElse(responseCode, ""), + headers = headerFields, + history = redirectedFrom + ) + if (onHeadersReceived != null) onHeadersReceived(streamHeaders) + + val stream = response.body() + + def processWrappedStream[V](f: java.io.InputStream => V): V = { + // The HEAD method is identical to GET except that the server + // MUST NOT return a message-body in the response. + // https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 + if (upperCaseVerb == "HEAD") f(new ByteArrayInputStream(Array())) + else if (stream != null) { + try f( + if (deGzip) new GZIPInputStream(stream) + else if (deDeflate) new InflaterInputStream(stream) + else stream + ) finally if (!keepAlive) stream.close() } else { - connection.setChunkedStreamingMode(0) - usingOutputStream(compress.wrap(connection.getOutputStream)) { os => data.write(os) } - } - } - - val (responseCode, responseMsg, headerFields) = try {( - connection.getResponseCode, - connection.getResponseMessage, - connection.getHeaderFields.asScala - .filter(_._1 != null) - .map{case (k, v) => (k.toLowerCase(), v.asScala.toSeq)}.toMap - )} catch{ - case e: java.net.SocketTimeoutException => - throw new TimeoutException(url, readTimeout, connectTimeout) - case e: java.net.UnknownHostException => - throw new UnknownHostException(url, e.getMessage) - case e: javax.net.ssl.SSLHandshakeException => - throw new InvalidCertException(url, e) - } - - val deGzip = autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("gzip")) - val deDeflate = autoDecompress && headerFields.get("content-encoding").toSeq.flatten.exists(_.contains("deflate")) - def persistCookies() = { - if (sess.persistCookies) { - headerFields - .get("set-cookie") - .iterator - .flatten - .flatMap(HttpCookie.parse(_).asScala) - .foreach(c => sess.cookies(c.getName) = c) + f(new ByteArrayInputStream(Array())) } } - if (responseCode.toString.startsWith("3") && - responseCode.toString != "304" && - maxRedirects > 0){ - val out = new ByteArrayOutputStream() - Util.transferTo(connection.getInputStream, out) - val bytes = out.toByteArray - - val current = Response( - url, - responseCode, - responseMsg, - new geny.Bytes(bytes), - headerFields, - redirectedFrom - ) - persistCookies() - val newUrl = current.headers("location").head - stream( - new java.net.URL(url1, newUrl).toString, auth, params, blobHeaders, - headers, data, readTimeout, connectTimeout, proxy, cert, sslContext, cookies, - cookieValues, maxRedirects - 1, verifySslCerts, autoDecompress, - compress, keepAlive, check, chunkedUpload, Some(current), - onHeadersReceived - ).readBytesThrough(f) - }else{ - persistCookies() - val streamHeaders = StreamHeaders( - url, - responseCode, - responseMsg, - headerFields, - redirectedFrom - ) - if (onHeadersReceived != null) onHeadersReceived(streamHeaders) - - val stream = - if (connection.getResponseCode.toString.startsWith("2")) connection.getInputStream - else connection.getErrorStream - - def processWrappedStream[V](f: java.io.InputStream => V): V = { - // The HEAD method is identical to GET except that the server - // MUST NOT return a message-body in the response. - // https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html section 9.4 - if (upperCaseVerb == "HEAD") f(new ByteArrayInputStream(Array())) - else if (stream != null) { - try f( - if (deGzip) new GZIPInputStream(stream) - else if (deDeflate) new InflaterInputStream(stream) - else stream - ) finally if (!keepAlive) stream.close() - }else{ - f(new ByteArrayInputStream(Array())) - } - } - - if (streamHeaders.statusCode == 304 || streamHeaders.is2xx || !check) processWrappedStream(f) - else { - val errorOutput = new ByteArrayOutputStream() - processWrappedStream(geny.Internal.transfer(_, errorOutput)) - throw new RequestFailedException( - Response( - streamHeaders.url, - streamHeaders.statusCode, - streamHeaders.statusMessage, - new geny.Bytes(errorOutput.toByteArray), - streamHeaders.headers, - streamHeaders.history - ) + if (streamHeaders.statusCode == 304 || streamHeaders.is2xx || !check) processWrappedStream(f) + else { + val errorOutput = new ByteArrayOutputStream() + processWrappedStream(geny.Internal.transfer(_, errorOutput)) + throw new RequestFailedException( + Response( + url = streamHeaders.url, + statusCode = streamHeaders.statusCode, + statusMessage = streamHeaders.statusMessage, + data = new geny.Bytes(errorOutput.toByteArray), + headers = streamHeaders.headers, + history = streamHeaders.history ) - } + ) } - } finally if (!keepAlive && connection != null) { - connection.disconnect() } } } - - private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T): Unit = + + private def usingOutputStream[T](os: OutputStream)(fn: OutputStream => T): Unit = try fn(os) finally os.close() /** @@ -397,33 +426,34 @@ case class Requester(verb: String, ) /** - * Overload of [[Requester.stream]] that takes a [[Request]] object as configuration - */ + * Overload of [[Requester.stream]] that takes a [[Request]] object as configuration + */ def stream(r: Request, data: RequestBlob, chunkedUpload: Boolean, - onHeadersReceived: StreamHeaders => Unit): geny.Writable = stream( - r.url, - r.auth, - r.params, - Seq.empty[(String, String)], - r.headers, - data, - r.readTimeout, - r.connectTimeout, - r.proxy, - r.cert, - r.sslContext, - r.cookies, - r.cookieValues, - r.maxRedirects, - r.verifySslCerts, - r.autoDecompress, - r.compress, - r.keepAlive, - r.check, - chunkedUpload, - None, - onHeadersReceived - ) + onHeadersReceived: StreamHeaders => Unit): geny.Writable = + stream( + url = r.url, + auth = r.auth, + params = r.params, + blobHeaders = Seq.empty[(String, String)], + headers = r.headers, + data = data, + readTimeout = r.readTimeout, + connectTimeout = r.connectTimeout, + proxy = r.proxy, + cert = r.cert, + sslContext = r.sslContext, + cookies = r.cookies, + cookieValues = r.cookieValues, + maxRedirects = r.maxRedirects, + verifySslCerts = r.verifySslCerts, + autoDecompress = r.autoDecompress, + compress = r.compress, + keepAlive = r.keepAlive, + check = r.check, + chunkedUpload = chunkedUpload, + redirectedFrom = None, + onHeadersReceived = onHeadersReceived + ) } diff --git a/requests/src/requests/StatusMessages.scala b/requests/src/requests/StatusMessages.scala new file mode 100644 index 0000000..56930cb --- /dev/null +++ b/requests/src/requests/StatusMessages.scala @@ -0,0 +1,95 @@ +package requests + +object StatusMessages { + val byStatusCode: Map[Int, String] = Map( + 100 -> "Continue", + 101 -> "Switching Protocols", + 102 -> "Processing", + 103 -> "Early Hints", + 200 -> "OK", + 201 -> "Created", + 202 -> "Accepted", + 203 -> "Non-Authoritative Information", + 204 -> "No Content", + 205 -> "Reset Content", + 206 -> "Partial Content", + 207 -> "Multi-Status", + 208 -> "Already Reported", + 218 -> "This is fine", + 226 -> "IM Used", + 300 -> "Multiple Choices", + 301 -> "Moved Permanently", + 302 -> "Found", + 303 -> "See Other", + 304 -> "Not Modified", + 305 -> "Use Proxy", + 306 -> "(Unused)", + 307 -> "Temporary Redirect", + 308 -> "Permanent Redirect", + 400 -> "Bad Request", + 401 -> "Unauthorized", + 402 -> "Payment Required", + 403 -> "Forbidden", + 404 -> "Not Found", + 405 -> "Method Not Allowed", + 406 -> "Not Acceptable", + 407 -> "Proxy Authentication Required", + 408 -> "Request Timeout", + 409 -> "Conflict", + 410 -> "Gone", + 411 -> "Length Required", + 412 -> "Precondition Failed", + 413 -> "Payload Too Large", + 414 -> "URI Too Long", + 415 -> "Unsupported Media Type", + 416 -> "Range Not Satisfiable", + 417 -> "Expectation Failed", + 418 -> "I'm a teapot", + 419 -> "Page Expired", + 420 -> "Enhance Your Calm", + 421 -> "Misdirected Request", + 422 -> "Unprocessable Entity", + 423 -> "Locked", + 424 -> "Failed Dependency", + 425 -> "Too Early", + 426 -> "Upgrade Required", + 428 -> "Precondition Required", + 429 -> "Too Many Requests", + 430 -> "Request Header Fields Too Large", + 431 -> "Request Header Fields Too Large", + 440 -> "Login Time-out", + 444 -> "No Response", + 449 -> "Retry With", + 450 -> "Blocked by Windows Parental Controls", + 451 -> "Unavailable For Legal Reasons", + 460 -> "Client Closed Request", + 494 -> "Request Header Too Large", + 495 -> "SSL Certificate Error", + 496 -> "SSL Certificate Required", + 497 -> "HTTP Request Sent to HTTPS Port", + 499 -> "Client Closed Request", + 500 -> "Internal Server Error", + 501 -> "Not Implemented", + 502 -> "Bad Gateway", + 503 -> "Service Unavailable", + 504 -> "Gateway Timeout", + 505 -> "HTTP Version Not Supported", + 506 -> "Variant Also Negotiates", + 507 -> "Insufficient Storage", + 508 -> "Loop Detected", + 509 -> "Bandwidth Limit Exceeded", + 510 -> "Not Extended", + 511 -> "Network Authentication Required", + 520 -> "Web Server Returned an Unknown Error", + 521 -> "Web Server Is Down", + 522 -> "Connection Timed Out", + 523 -> "Origin Is Unreachable", + 524 -> "A Timeout Occurred", + 525 -> "SSL Handshake Failed", + 526 -> "Invalid SSL Certificate", + 527 -> "Railgun Error", + 530 -> "Site is Frozen", + 598 -> "Network Read Timeout Error", + 599 -> "Network Connect Timeout Error" + ) +} diff --git a/requests/src/requests/Util.scala b/requests/src/requests/Util.scala index c7bf3f5..3d6c425 100644 --- a/requests/src/requests/Util.scala +++ b/requests/src/requests/Util.scala @@ -26,16 +26,20 @@ object Util { .mkString("&") } - private[requests] val noVerifySocketFactory = { + private[requests] val noVerifySSLContext = { // Install the all-trusting trust manager val sc = SSLContext.getInstance("SSL") sc.init(null, trustAllCerts, new java.security.SecureRandom()) - sc.getSocketFactory + sc } - private[requests] def clientCertSocketFactory(cert: Cert, verifySslCerts: Boolean) = cert match { + @deprecated("No longer used", "0.9.0") + private[requests] val noVerifySocketFactory = + noVerifySSLContext.getSocketFactory + + private[requests] def clientCertSSLContext(cert: Cert, verifySslCerts: Boolean) = cert match { case Cert.P12(path, password) => val pass = password.map(_.toCharArray).getOrElse(Array.emptyCharArray) @@ -53,11 +57,15 @@ object Util { val trustManagers = if (verifySslCerts) null else trustAllCerts sc.init(keyManagers, trustManagers, new java.security.SecureRandom()) - sc.getSocketFactory + sc } + @deprecated("No longer used", "0.9.0") + private[requests] def clientCertSocketFactory(cert: Cert, verifySslCerts: Boolean) = + clientCertSSLContext(cert, verifySslCerts).getSocketFactory + private lazy val trustAllCerts = Array[TrustManager](new X509TrustManager() { - def getAcceptedIssuers() = new Array[X509Certificate](0) + def getAcceptedIssuers = new Array[X509Certificate](0) def checkClientTrusted(chain: Array[X509Certificate], authType: String) = {} diff --git a/requests/test/src/requests/RequestTests.scala b/requests/test/src/requests/RequestTests.scala index e0780dd..6d935ac 100644 --- a/requests/test/src/requests/RequestTests.scala +++ b/requests/test/src/requests/RequestTests.scala @@ -202,7 +202,7 @@ object RequestTests extends TestSuite{ compress = requests.Compress.Deflate, data = new RequestBlob.ByteSourceRequestBlob("Hear me moo") ) - assert(read(new String(res3.bytes))("data").toString == + assert(read(new String(res3.bytes))("data").toString == """"data:application/octet-stream;base64,eJzzSE0sUshNVcjNzwcAFokD3g=="""") } @@ -212,10 +212,9 @@ object RequestTests extends TestSuite{ val hs = read(res)("headers").obj assert(hs("User-Agent").str == "requests-scala") assert(hs("Accept-Encoding").str == "gzip, deflate") - assert(hs("Pragma").str == "no-cache") assert(hs("Accept").str == "*/*") test("hasNoCookie"){ - assert(hs.get("Cookie").isEmpty) + assert(!hs.contains("Cookie")) } } } @@ -278,7 +277,6 @@ object RequestTests extends TestSuite{ test("gzipError"){ val response = requests.head("https://api.github.com/users/lihaoyi") assert(response.statusCode == 200) - assert(response.statusMessage == "OK") assert(response.data.array.isEmpty) assert(response.headers.keySet.map(_.toLowerCase).contains("content-length")) assert(response.headers.keySet.map(_.toLowerCase).contains("content-type")) @@ -291,15 +289,17 @@ object RequestTests extends TestSuite{ test("compressionData") { import requests.Compress._ val str = "I am deflater mouse" - Seq(None, Gzip, Deflate).foreach(c => + Seq(None, Gzip, Deflate).foreach { c => ServerUtils.usingEchoServer { port => - assert(str == requests.post( - s"http://localhost:$port/echo", - compress = c, - data = new RequestBlob.ByteSourceRequestBlob(str) - ).data.toString) + val response = + requests.post( + s"http://localhost:$port/echo", + compress = c, + data = new RequestBlob.ByteSourceRequestBlob(str) + ) + assert(str == response.data.toString) } - ) + } } } } diff --git a/requests/test/src/requests/ServerUtils.scala b/requests/test/src/requests/ServerUtils.scala index d9ceebd..81b0ea0 100644 --- a/requests/test/src/requests/ServerUtils.scala +++ b/requests/test/src/requests/ServerUtils.scala @@ -25,7 +25,7 @@ object ServerUtils { def stop(): Unit = server.stop(0) - override def handle(t: HttpExchange): Unit = { + override def handle(t: HttpExchange): Unit = try { val h: java.util.List[String] = t.getRequestHeaders.get("Content-encoding") val c: Compress = @@ -36,6 +36,10 @@ object ServerUtils { val msg = new Plumper(c).decompress(t.getRequestBody) t.sendResponseHeaders(200, msg.length) t.getResponseBody.write(msg.getBytes()) + } catch { + case e: Exception => + e.printStackTrace() + t.sendResponseHeaders(500, -1) } }