diff --git a/build.sbt b/build.sbt index 04f79c00..19151847 100644 --- a/build.sbt +++ b/build.sbt @@ -67,6 +67,8 @@ lazy val commonSettings = Seq( compilerPlugin("org.typelevel" %% "kind-projector" % "0.11.3" cross CrossVersion.full), ).filterNot(_ => scalaVersion.value.startsWith("3.")), + resolvers += Resolver.sonatypeRepo("snapshots"), // until opentelemetry release 1.4.0 or 1.3.1 + // dottydoc really doesn't work at all right now Compile / doc / sources := { val old = (Compile / doc / sources).value @@ -100,8 +102,8 @@ lazy val natchez = project crossScalaVersions := Nil, publish / skip := true ) - .dependsOn(coreJS, coreJVM, jaeger, honeycomb, opencensus, opentracing, datadog, lightstep, lightstepGrpc, lightstepHttp, logJS, logJVM, mtlJS, mtlJVM, noop, mock, newrelic, logOdin, examples) - .aggregate(coreJS, coreJVM, jaeger, honeycomb, opencensus, opentracing, datadog, lightstep, lightstepGrpc, lightstepHttp, logJS, logJVM, mtlJS, mtlJVM, noop, mock, newrelic, logOdin, examples) + .dependsOn(coreJS, coreJVM, jaeger, honeycomb, opencensus, opentracing, datadog, lightstep, lightstepGrpc, lightstepHttp, logJS, logJVM, mtlJS, mtlJVM, noop, mock, newrelic, opentelemetry, logOdin, examples) + .aggregate(coreJS, coreJVM, jaeger, honeycomb, opencensus, opentracing, datadog, lightstep, lightstepGrpc, lightstepHttp, logJS, logJVM, mtlJS, mtlJVM, noop, mock, newrelic, opentelemetry, logOdin, examples) lazy val core = crossProject(JSPlatform, JVMPlatform) .in(file("modules/core")) @@ -278,6 +280,21 @@ lazy val newrelic = project ) ) +lazy val opentelemetry = project + .in(file("modules/opentelemetry")) + .dependsOn(coreJVM) + .enablePlugins(AutomateHeaderPlugin) + .settings(commonSettings) + .settings( + name := "opentelemetry", + description := "OpenTelemetry bindings for Natchez.", + libraryDependencies ++= Seq( + "org.scala-lang.modules" %% "scala-collection-compat" % collectionCompatVersion, + "io.opentelemetry" % "opentelemetry-api" % "1.4.0-SNAPSHOT", + "io.opentelemetry" % "opentelemetry-sdk" % "1.4.0-SNAPSHOT", + ) + ) + lazy val mtl = crossProject(JSPlatform, JVMPlatform) .in(file("modules/mtl")) .enablePlugins(AutomateHeaderPlugin) @@ -325,7 +342,7 @@ lazy val mock = project lazy val examples = project .in(file("modules/examples")) - .dependsOn(coreJVM, jaeger, honeycomb, lightstepHttp, datadog, logJVM, newrelic, logOdin) + .dependsOn(coreJVM, jaeger, honeycomb, lightstepHttp, datadog, logJVM, newrelic, logOdin, opentelemetry) .enablePlugins(AutomateHeaderPlugin) .settings(commonSettings) .settings( @@ -334,11 +351,13 @@ lazy val examples = project description := "Example programs for Natchez.", scalacOptions -= "-Xfatal-warnings", libraryDependencies ++= Seq( - "org.typelevel" %% "log4cats-slf4j" % "1.3.1", - "org.slf4j" % "slf4j-simple" % "1.7.30", - "eu.timepit" %% "refined" % "0.9.25", - "is.cir" %% "ciris" % "1.2.1" - ).filterNot(_ => scalaVersion.value.startsWith("3.")) + "org.typelevel" %% "log4cats-slf4j" % "1.3.1", + "org.slf4j" % "slf4j-simple" % "1.7.30", + "eu.timepit" %% "refined" % "0.9.25", + "is.cir" %% "ciris" % "1.2.1", + "io.opentelemetry" % "opentelemetry-exporter-otlp" % "1.4.0-SNAPSHOT", + "io.grpc" % "grpc-okhttp" % "1.38.0", // required for the OpenTelemetry exporter +).filterNot(_ => scalaVersion.value.startsWith("3.")) ) lazy val logOdin = project @@ -359,7 +378,7 @@ lazy val logOdin = project lazy val docs = project .in(file("modules/docs")) - .dependsOn(mtlJVM, honeycomb, jaeger, logJVM) + .dependsOn(mtlJVM, honeycomb, jaeger, logJVM, opentelemetry) .enablePlugins(AutomateHeaderPlugin) .enablePlugins(ParadoxPlugin) .enablePlugins(ParadoxSitePlugin) @@ -390,6 +409,7 @@ lazy val docs = project "org.http4s" %% "http4s-client" % "0.21.15", "org.typelevel" %% "log4cats-slf4j" % "1.3.1", "org.slf4j" % "slf4j-simple" % "1.7.30", + "io.opentelemetry" % "opentelemetry-exporter-otlp" % "1.4.0-SNAPSHOT", // for the opentelemetry example ) ) diff --git a/modules/docs/src/main/paradox/backends/index.md b/modules/docs/src/main/paradox/backends/index.md index 3a01e850..54664851 100644 --- a/modules/docs/src/main/paradox/backends/index.md +++ b/modules/docs/src/main/paradox/backends/index.md @@ -14,6 +14,7 @@ Natchez supports the following tracing back ends. If you're not sure which one y * [No-Op](noop.md) * [Odin](odin.md) * [OpenCensus](opencensus.md) +* [OpenTelemetry](opentelemetry.md) @@@ diff --git a/modules/docs/src/main/paradox/backends/opentelemetry.md b/modules/docs/src/main/paradox/backends/opentelemetry.md new file mode 100644 index 00000000..b56aaf6d --- /dev/null +++ b/modules/docs/src/main/paradox/backends/opentelemetry.md @@ -0,0 +1,89 @@ +# OpenTelemetry + +The `natchez-opentelemetry` module provides a backend that uses [OpenTelemetry](https://opentelemetry.io) to report spans. + +To use it, add the following dependency. + +@@dependency[sbt,Maven,Gradle] { +group="$org$" +artifact="natchez-opentelemetry-2.13" +version="$version$" +} + +Then add any exporter, for example: + +@@dependency[sbt,Maven,Gradle] { +group="io.opentelemetry" +artifact="opentelemetry-exporter-otlp" +version="1.4.0-SNAPSHOT" +} + +## Note on the OpenTelemetry version numbers + +Currently, this depends on `1.4.0-SNAPSHOT` because of a bug I discovered upstream while writing this, where the `shutdown()` calls could block indefinitely, this has been fixed in the snapshot build, and this note can be removed once there's a stable release with this fix. + +## Configuring an OpenTelemetry entrypoint + +There are two methods you'll need to construct an `OpenTelemetry` `EndPoint`. + +`OpenTelemetry.lift` is used to turn an `F[_]` that constructs a `SpanExporter`, `SpanProcessor` or `SdkTraceProvider` into a `Resource` that will shut it down cleanly. +This takes a `String` of what you've constructed, so we can give a nice error if it fails to shut down cleanly. + +The `OpenTelemetry.entryPoint` method takes a boolean called `globallyRegister` which tells it whether to register this `OpenTelemetrySdk` globally, this may be helpful if you have other java dependencies that use the global tracer, it defaults to false. +It also takes an `OpenTelemetrySdkBuilder => Resource[F, OpenTelemetrySdkBuilder]` so that you can configure the Sdk. + +Here's an example of configuring one with the `otlp` exporter with batch span processing: + +```scala mdoc:passthrough +import natchez.EntryPoint +import natchez.opentelemetry.OpenTelemetry +import cats.effect._ +import io.opentelemetry.api.common.Attributes +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes +import io.opentelemetry.sdk.resources.{Resource => OtelResource} +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator +import io.opentelemetry.context.propagation.ContextPropagators +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.BatchSpanProcessor + + def entryPoint[F[_]: Async]: Resource[F, EntryPoint[F]] = + for { + exporter <- OpenTelemetry.lift( + "OtlpGrpcSpanExporter", + Sync[F].delay { + OtlpGrpcSpanExporter.builder() + .setEndpoint("http://localhost:4317") + .build() + } + ) + processor <- OpenTelemetry.lift( + "BatchSpanProcessor", + Sync[F].delay { + BatchSpanProcessor.builder(exporter).build() + } + ) + tracer <- OpenTelemetry.lift( + "Tracer", + Sync[F].delay { + SdkTracerProvider.builder() + .setResource( + OtelResource.create( + Attributes.of(ResourceAttributes.SERVICE_NAME, "OpenTelemetryExample") + ) + ) + .addSpanProcessor(processor) + .build() + } + ) + ep <- OpenTelemetry.entryPoint(globallyRegister = true) { builder => + Resource.eval(Sync[F].delay { + builder + .setTracerProvider(tracer) + .setPropagators( + ContextPropagators.create(W3CTraceContextPropagator.getInstance()) + ) + } + )} + } yield ep +``` \ No newline at end of file diff --git a/modules/examples/src/main/scala-2/OpenTelemetryExample.scala b/modules/examples/src/main/scala-2/OpenTelemetryExample.scala new file mode 100644 index 00000000..59e987a8 --- /dev/null +++ b/modules/examples/src/main/scala-2/OpenTelemetryExample.scala @@ -0,0 +1,79 @@ +import cats.data.Kleisli +import cats.effect._ +import cats.implicits._ +import io.opentelemetry.api.common.Attributes +import io.opentelemetry.sdk.resources.{Resource => OtelResource} +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator +import io.opentelemetry.context.propagation.ContextPropagators +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.BatchSpanProcessor +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes +import natchez.{EntryPoint, Span, Trace} +import natchez.opentelemetry.OpenTelemetry + +import java.util.concurrent.TimeUnit +import scala.concurrent.duration.DurationInt + +// change this into an object if you'd like to run it +object OpenTelemetryExample extends IOApp { + def entryPoint[F[_]: Async]: Resource[F, EntryPoint[F]] = + for { + exporter <- OpenTelemetry.lift( + "OtlpGrpcSpanExporter", + Sync[F].delay { + OtlpGrpcSpanExporter.builder() + .setEndpoint("http://localhost:4317") + .build() + } + ) + processor <- OpenTelemetry.lift( + "BatchSpanProcessor", + Sync[F].delay { + BatchSpanProcessor.builder(exporter).build() + } + ) + tracer <- OpenTelemetry.lift( + "Tracer", + Sync[F].delay { + SdkTracerProvider.builder() + .setResource( + OtelResource.create( + Attributes.of(ResourceAttributes.SERVICE_NAME, "OpenTelemetryExample") + ) + ) + .addSpanProcessor(processor) + .build() + } + ) + ep <- OpenTelemetry.entryPoint(globallyRegister = true) { builder => + Resource.eval(Sync[F].delay { + builder + .setTracerProvider(tracer) + .setPropagators( + ContextPropagators.create(W3CTraceContextPropagator.getInstance()) + ) + } + )} + } yield ep + + override def run(args: List[String]): IO[ExitCode] = + entryPoint[IO].use { ep => + ep.root("root span").use { span => + span.put("service.name" -> "natchez opentelemetry example") *> + program[Kleisli[IO, Span[IO], *]].apply(span).as(ExitCode.Success) + } + } + + def program[F[_]: Sync: Trace: Timer]: F[Unit] = + Trace[F].traceId.flatTap(tid => Sync[F].delay { println(s"did some work with traceid of $tid") }) *> + Trace[F].span("outer span") { + Trace[F].put("foo" -> "bar") *> + (Trace[F].span("first thing") { + Timer[F].sleep(2.seconds) + }, + Trace[F].span("second thing") { + Timer[F].sleep(2.seconds) + }).tupled + }.void +} diff --git a/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetry.scala b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetry.scala new file mode 100644 index 00000000..22f56503 --- /dev/null +++ b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetry.scala @@ -0,0 +1,60 @@ +// Copyright (c) 2019-2020 by Rob Norris and Contributors +// This software is licensed under the MIT License (MIT). +// For more information see LICENSE or https://opensource.org/licenses/MIT + +package natchez +package opentelemetry + +import cats.effect._ +import cats.implicits._ +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.sdk.{OpenTelemetrySdk, OpenTelemetrySdkBuilder} + +import java.net.URI + +object OpenTelemetry { + private final val instrumentationName = "natchez.opentelemetry" + + // Helper methods to help you construct Otel resources that clean themselves up + // We need a name so the failure error can contain something useful + def lift[F[_]: Async, T: Shutdownable](name: String, create: F[T]): Resource[F, T] = + Resource.make(create) { t => + Sync[F].delay { Shutdownable[T].shutdown(t) } + .flatMap(Utils.asyncFromCompletableResultCode(s"$name cleanup", _)) + } + + def entryPoint[F[_] : Sync](uriPrefix: Option[URI] = None, globallyRegister: Boolean = false)( + configure: OpenTelemetrySdkBuilder => Resource[F, OpenTelemetrySdkBuilder] + ): Resource[F, EntryPoint[F]] = { + val register: OpenTelemetrySdkBuilder => Resource[F, (OpenTelemetrySdk, Tracer)] = { b => + Resource.make( + Sync[F].delay { + val sdk = if (globallyRegister) + b.buildAndRegisterGlobal() + else + b.build() + val tracer = sdk.getTracer(instrumentationName) + (sdk, tracer) + } + ) { case (_, _) => + Sync[F].delay { + if (globallyRegister) + GlobalOpenTelemetry.resetForTest() // this seems to be the only way to deregister it + } + } + } + Resource.eval(Sync[F].delay { OpenTelemetrySdk.builder() }) + .flatMap(configure) + .flatMap(register) + .map { case (sdk, tracer) => + OpenTelemetryEntryPoint(sdk, tracer, uriPrefix) + } + } + + def globalEntryPoint[F[_]: Sync](uriPrefix: Option[URI] = None): F[OpenTelemetryEntryPoint[F]] = + Sync[F].delay { + val ot = GlobalOpenTelemetry.get() + OpenTelemetryEntryPoint(ot, ot.getTracer(instrumentationName), uriPrefix) + } +} diff --git a/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetryEntryPoint.scala b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetryEntryPoint.scala new file mode 100644 index 00000000..411f63e7 --- /dev/null +++ b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetryEntryPoint.scala @@ -0,0 +1,50 @@ +// Copyright (c) 2019-2020 by Rob Norris and Contributors +// This software is licensed under the MIT License (MIT). +// For more information see LICENSE or https://opensource.org/licenses/MIT + +package natchez +package opentelemetry + +import cats.effect.{Resource, Sync} +import io.opentelemetry.api.{OpenTelemetry => OtelOpenTelemetry} +import io.opentelemetry.api.trace.{Span => OtelSpan, Tracer => OtelTracer} +import io.opentelemetry.context.Context +import io.opentelemetry.context.propagation.TextMapGetter + +import java.lang +import java.net.URI +import scala.jdk.CollectionConverters._ + +final case class OpenTelemetryEntryPoint[F[_]: Sync](sdk: OtelOpenTelemetry, tracer: OtelTracer, uriPrefix: Option[URI]) extends EntryPoint[F] { + override def root(name: String): Resource[F, Span[F]] = + Resource.make[F, OtelSpan] ( + Sync[F].delay { tracer.spanBuilder(name).startSpan() } + ) { s => + Sync[F].delay { s.end() } + }.map { span => + OpenTelemetrySpan(sdk, tracer, span, uriPrefix) + } + + object MapGetter extends TextMapGetter[Map[String, String]] { + override def keys(carrier: Map[String, String]): lang.Iterable[String] = carrier.keys.asJava + override def get(carrier: Map[String, String], key: String): String = carrier.get(key).orNull + } + + override def continue(name: String, kernel: Kernel): Resource[F, Span[F]] = + Resource.make( + Sync[F].delay { + val c = Context.root() + val p = sdk.getPropagators.getTextMapPropagator.extract( + c, + kernel.toHeaders, + MapGetter + ) + tracer.spanBuilder(name).setParent(p).startSpan() + } + ) { s => + Sync[F].delay { s.end() } + }.map(OpenTelemetrySpan(sdk, tracer, _, uriPrefix)) + + override def continueOrElseRoot(name: String, kernel: Kernel): Resource[F, Span[F]] = + continue(name, kernel) // this is already the behaviour of `continue` since it defaults to not changing the context +} diff --git a/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetrySpan.scala b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetrySpan.scala new file mode 100644 index 00000000..88772fae --- /dev/null +++ b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/OpenTelemetrySpan.scala @@ -0,0 +1,83 @@ +// Copyright (c) 2019-2020 by Rob Norris and Contributors +// This software is licensed under the MIT License (MIT). +// For more information see LICENSE or https://opensource.org/licenses/MIT + +package natchez +package opentelemetry + +import cats.data.Nested +import io.opentelemetry.api.trace.{Span => OtelSpan, Tracer => OtelTracer} +import cats.effect._ +import cats.implicits._ +import io.opentelemetry.api.{OpenTelemetry => OtelOpenTelemetry} +import io.opentelemetry.context.Context +import io.opentelemetry.context.propagation.TextMapSetter +import natchez.TraceValue._ + +import scala.collection.mutable +import java.net.URI + +final case class OpenTelemetrySpan[F[_] : Sync](sdk: OtelOpenTelemetry, tracer: OtelTracer, span: OtelSpan, prefix: Option[URI]) extends Span[F] { + override def put(fields: (String, TraceValue)*): F[Unit] = + fields.toList.traverse_ { + case (k, StringValue(v)) => Sync[F].delay(span.setAttribute(k, v)) + // all integer types are cast up to Long, since that's all OpenTelemetry lets us use + case (k, NumberValue(n: java.lang.Byte)) => Sync[F].delay(span.setAttribute(k, n.toLong)) + case (k, NumberValue(n: java.lang.Short)) => Sync[F].delay(span.setAttribute(k, n.toLong)) + case (k, NumberValue(n: java.lang.Integer)) => Sync[F].delay(span.setAttribute(k, n.toLong)) + case (k, NumberValue(n: java.lang.Long)) => Sync[F].delay(span.setAttribute(k, n)) + // and all float types are changed to Double + case (k, NumberValue(n: java.lang.Float)) => Sync[F].delay(span.setAttribute(k, n.toDouble)) + case (k, NumberValue(n: java.lang.Double)) => Sync[F].delay(span.setAttribute(k, n)) + // anything which could be too big to put in a Long is converted to a String + case (k, NumberValue(n: java.math.BigDecimal)) => Sync[F].delay(span.setAttribute(k, n.toString)) + case (k, NumberValue(n: java.math.BigInteger)) => Sync[F].delay(span.setAttribute(k, n.toString)) + case (k, NumberValue(n: BigDecimal)) => Sync[F].delay(span.setAttribute(k, n.toString)) + case (k, NumberValue(n: BigInt)) => Sync[F].delay(span.setAttribute(k, n.toString)) + // and any other Number can fall back to a Double + case (k, NumberValue(v)) => Sync[F].delay(span.setAttribute(k, v.doubleValue())) + case (k, BooleanValue(v)) => Sync[F].delay(span.setAttribute(k, v)) + } + + object MutableMapKeySetter extends TextMapSetter[mutable.HashMap[String, String]] { + override def set(carrier: mutable.HashMap[String, String], key: String, value: String): Unit = + carrier.put(key, value): Unit + } + + override def kernel: F[Kernel] = Sync[F].delay { + val m = new mutable.HashMap[String, String] + sdk.getPropagators.getTextMapPropagator.inject( + Context.current.`with`(span), + m, + MutableMapKeySetter + ) + Kernel(m.toMap) + } + + override def span(name: String): Resource[F, Span[F]] = + Span.putErrorFields( + Resource.make[F, OtelSpan]( + Sync[F].delay { + tracer.spanBuilder(name).setParent(Context.current().`with`(span)).startSpan() + }) { s => + Sync[F].delay { + s.end() + } + }.map(OpenTelemetrySpan(sdk, tracer, _, prefix)) + ) + + override def traceId: F[Option[String]] = Sync[F].delay { + val rawId = span.getSpanContext.getTraceId + if (rawId.nonEmpty) rawId.some else none + } + + override def spanId: F[Option[String]] = Sync[F].delay { + val rawId = span.getSpanContext.getSpanId + if (rawId.nonEmpty) rawId.some else none + } + + override def traceUri: F[Option[URI]] = + (Nested(prefix.pure[F]), Nested(traceId)).mapN { (uri, id) => + uri.resolve(s"/trace/$id") + }.value +} diff --git a/modules/opentelemetry/src/main/scala/natchez/opentelemetry/Shutdownable.scala b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/Shutdownable.scala new file mode 100644 index 00000000..cd57bb90 --- /dev/null +++ b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/Shutdownable.scala @@ -0,0 +1,20 @@ +// Copyright (c) 2019-2020 by Rob Norris and Contributors +// This software is licensed under the MIT License (MIT). +// For more information see LICENSE or https://opensource.org/licenses/MIT + +package natchez.opentelemetry + +import io.opentelemetry.sdk.common.CompletableResultCode +import io.opentelemetry.sdk.trace.{SdkTracerProvider, SpanProcessor} +import io.opentelemetry.sdk.trace.`export`.SpanExporter + +// abstracts over all the ways Otel classes can be shut down, they don't have a common interface so let's make one +trait Shutdownable[-T] { + def shutdown(t: T): CompletableResultCode +} +object Shutdownable { + def apply[T: Shutdownable]: Shutdownable[T] = implicitly + implicit val spanExporter: Shutdownable[SpanExporter] = (t: SpanExporter) => t.shutdown() + implicit val spanProcessor: Shutdownable[SpanProcessor] = (t: SpanProcessor) => t.shutdown() + implicit val tracer: Shutdownable[SdkTracerProvider] = (t: SdkTracerProvider) => t.shutdown() +} \ No newline at end of file diff --git a/modules/opentelemetry/src/main/scala/natchez/opentelemetry/Utils.scala b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/Utils.scala new file mode 100644 index 00000000..05fbc60b --- /dev/null +++ b/modules/opentelemetry/src/main/scala/natchez/opentelemetry/Utils.scala @@ -0,0 +1,29 @@ +// Copyright (c) 2019-2020 by Rob Norris and Contributors +// This software is licensed under the MIT License (MIT). +// For more information see LICENSE or https://opensource.org/licenses/MIT + +package natchez.opentelemetry + +import cats.effect.{Async, Sync} +import cats.implicits._ +import io.opentelemetry.sdk.common.CompletableResultCode + +object Utils { + case class CompletableResultCodeFailure(s: String) extends RuntimeException(s) + + // Converts an OpenTelemetry CompletableResultCode into an Async action of return type unit + // This will raise if the action failed, but we don't get any information on the failure so we throw an empty exception + def asyncFromCompletableResultCode[F[_]: Async](name: String, crc: CompletableResultCode): F[Unit] = { + Async[F].asyncF[Unit] { cb => + Sync[F].delay { + crc.whenComplete( + () => if (crc.isSuccess) { + cb(().asRight) + } else { + cb(CompletableResultCodeFailure(s"The OpenTelemetry action '$name' failed").asLeft) + } + ): Unit + } + } + } +}