Merge branch 'master' into move_adaptive_sampling_over

CHANGELOG.md

@@ -1,13 +1,28 @@
 Changes by Version
 ==================
 
-1.8.0 (unreleased)
+1.8.0 (2018-11-12)
 ------------------
 
 #### Backend Changes
 
 ##### Breaking Changes
 
+- Refactor agent configuration ([#1092](https://github.com/jaegertracing/jaeger/pull/1092), [@pavolloffay](https://github.com/pavolloffay))
+
+The following agent flags has has been deprecated in order to support multiple reporters:
+```bash
+--collector.host-port
+--discovery.conn-check-timeout
+--discovery.min-peers
+```
+New flags:
+```bash
+--reporter.tchannel.host-port
+--reporter.tchannel.discovery.conn-check-timeout
+--reporter.tchannel.discovery.min-peers
+```
+
 - Various changes around metrics produced by jaeger-query: Names scoped to the query component, generated for all span readers (not just ES), consolidate query metrics and include result tag ([#1074](https://github.com/jaegertracing/jaeger/pull/1074), [#1075](https://github.com/jaegertracing/jaeger/pull/1075) and [#1096](https://github.com/jaegertracing/jaeger/pull/1096), [@objectiser](https://github.com/objectiser))
 
 For example, sample of metrics produced for `find_traces` operation before:
@@ -31,13 +46,32 @@ jaeger_query_requests{operation="find_traces",result="ok"} 2
 jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 ```
 
+##### New Features
+
+- Configurable deadlock detector interval for ingester ([#1134](https://github.com/jaegertracing/jaeger/pull/1134), [@marqc](https://github.com/marqc))
+- Emit spans for elastic storage backend ([#1128](https://github.com/jaegertracing/jaeger/pull/1128), [@annanay25](https://github.com/annanay25))
+- Allow to use TLS certificates for Elasticsearch authentication ([#1139](https://github.com/jaegertracing/jaeger/pull/1139), [@clyang82](https://github.com/clyang82))
+- Add ingester metrics, healthcheck and rename Kafka cli flags ([#1094](https://github.com/jaegertracing/jaeger/pull/1094), [@ledor473](https://github.com/ledor473))
+- Add a metric for number of partitions held ([#1154](https://github.com/jaegertracing/jaeger/pull/1154), [@vprithvi](https://github.com/vprithvi))
+- Log jaeger-collector tchannel port ([#1136](https://github.com/jaegertracing/jaeger/pull/1136), [@mindaugasrukas](https://github.com/mindaugasrukas))
+- Support tracer env based initialization in hotrod ([#1115](https://github.com/jaegertracing/jaeger/pull/1115), [@eundoosong](https://github.com/eundoosong))
+- Publish ingester as binaries and docker image ([#1086](https://github.com/jaegertracing/jaeger/pull/1086), [@ledor473](https://github.com/ledor473))
+- Use Go 1.11 ([#1104](https://github.com/jaegertracing/jaeger/pull/1104), [@isaachier](https://github.com/isaachier))
+- Tag images with commit SHA and publish to `-snapshot` repository ([#1082](https://github.com/jaegertracing/jaeger/pull/1082), [@pavolloffay](https://github.com/pavolloffay))
+
+##### Bug fixes, Minor Improvements
+
+- Fix child span context while tracing cassandra queries ([#1131](https://github.com/jaegertracing/jaeger/pull/1131), [@annanay25](https://github.com/annanay25))
+- Deadlock detector hack for Kafka driver instability ([#1087](https://github.com/jaegertracing/jaeger/pull/1087), [@vprithvi](https://github.com/vprithvi))
+- Fix processor overriding data in a buffer ([#1099](https://github.com/jaegertracing/jaeger/pull/1099), [@pavolloffay](https://github.com/pavolloffay))
+
 #### UI Changes
 
 ##### New Features
 
-- Span Search - Highlight search results ([#238](https://github.com/jaegertracing/jaeger-ui/pull/238)),  thanks to David Yeghshatyan [@davit-y](https://github.com/davit-y)
-- Span Search - Improve search logic ([#237](https://github.com/jaegertracing/jaeger-ui/pull/237)),  thanks to David Yeghshatyan [@davit-y](https://github.com/davit-y)
-- Span Search - Add result count, navigation and clear buttons ([#234](https://github.com/jaegertracing/jaeger-ui/pull/234)), thanks to David Yeghshatyan [@davit-y](https://github.com/davit-y)
+- Span Search - Highlight search results ([#238](https://github.com/jaegertracing/jaeger-ui/pull/238)), [@davit-y](https://github.com/davit-y)
+- Span Search - Improve search logic ([#237](https://github.com/jaegertracing/jaeger-ui/pull/237)),  [@davit-y](https://github.com/davit-y)
+- Span Search - Add result count, navigation and clear buttons ([#234](https://github.com/jaegertracing/jaeger-ui/pull/234)), [@davit-y](https://github.com/davit-y)
 
 ##### Bug Fixes, Minor Improvements
 
@@ -59,7 +93,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 
 #### Backend Changes
 
-##### Breaking changes
+##### Breaking Changes
 
 - `jaeger-standalone` binary has been renamed to `jaeger-all-in-one`. This change also includes package rename from `standalone` to `all-in-one` ([#1062](https://github.com/jaegertracing/jaeger/pull/1062), [@pavolloffay](https://github.com/pavolloffay))
 
@@ -78,7 +112,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 
 #### Backend Changes
 
-##### Breaking Changes!!!
+##### Breaking Changes
 
 - The storage implementations no longer write the parentSpanID field to storage (#856).
   If you are upgrading to this version, **you must upgrade query service first**!
@@ -178,7 +212,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 
 - Support traces with >10k spans in Elasticsearch (#668) - thanks to @sramakr
 
-##### Fixes
+##### Bug Fixes, Minor Improvements
 
 - Allow slash '/' in service names (#586)
 - Log errors from HotROD services (#769)
@@ -200,7 +234,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 - Collect metrics from ES bulk service (#688) <Pavol Loffay>
 - Allow zero replicas for Elasticsearch (#754) <bharat-p>
 
-##### Fixes
+##### Bug Fixes, Minor Improvements
 
 - Apply namespace when creating Prometheus metrics factory (fix for #732) (#733) <Yuri Shkuro>
 - Disable double compression on Prom Handler - fixes #697 (#735) <Juraci Paixão Kröhling>
@@ -220,7 +254,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 - Track Javascript errors via Google Analytics (fixes [#39](https://github.com/jaegertracing/jaeger-ui/issues/39))
 - Add Google Analytics event tracking for actions in trace view ([#191](https://github.com/jaegertracing/jaeger-ui/issues/191))
 
-##### Fixes
+##### Bug Fixes, Minor Improvements
 
 - Clearly identify traces without a root span (fixes [#190](https://github.com/jaegertracing/jaeger-ui/issues/190))
 - Fix [#166](https://github.com/jaegertracing/jaeger-ui/issues/166) JS error on search page after viewing 404 trace
@@ -241,7 +275,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 - Make agent host port configurable in hotrod (#663) <Pavol Loffay>
 - Add signal handling to standalone (#657) <Pavol Loffay>
 
-##### Fixes
+##### Bug Fixes, Minor Improvements
 
 - Remove the override of GOMAXPROCS (#679) <Cruth kvinc>
 - Use UTC timezone for ES indices (#646) <Pavol Loffay>
@@ -256,7 +290,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
   - Fix [#165](https://github.com/jaegertracing/jaeger-ui/issues/165) - Search results are shown without a date
   - Fix [#69](https://github.com/jaegertracing/jaeger-ui/issues/69) - Missing endpoints in jaeger ui dropdown
 
-##### Fixes
+##### Bug Fixes, Minor Improvements
 
 - Fix 2 digit lookback (12h, 24h) parsing (https://github.com/jaegertracing/jaeger-ui/issues/167)
 
@@ -272,7 +306,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 - Add CA certificates to collector/query images (#485)
 - Parse zipkin v2 high trace id (#596)
 
-##### Fixes
+##### Bug Fixes, Minor Improvements
 
 - Skip nil and zero length hits in ElasticSearch storage (#601)
 - Make Cassandra service_name_index inserts idempotent (#587)
@@ -290,7 +324,7 @@ jaeger_query_responses_bucket{operation="find_traces",le="0.005"} 2
 - Add View Option for raw/unadjusted trace (https://github.com/jaegertracing/jaeger-ui/issues/153)
 - Add timezone tooltip to custom lookback form-field (https://github.com/jaegertracing/jaeger-ui/pull/161)
 
-##### Fixes
+##### Bug Fixes, Minor Improvements
 
 - Use consistent icons for logs expanded/collapsed (https://github.com/jaegertracing/jaeger-ui/issues/86)
 - Encode service name in API calls to allow '/' (https://github.com/jaegertracing/jaeger-ui/issues/138)

Makefile

@@ -6,6 +6,7 @@ TOP_PKGS := $(shell glide novendor | \
 	grep -v \
 		-e ./thrift-gen/... \
 		-e ./swagger-gen/... \
+		-e ./proto-gen/... \
 		-e ./examples/... \
 		-e ./scripts/...\
 	)
@@ -17,6 +18,7 @@ ALL_SRC := $(shell find . -name "*.go" | \
 		-e vendor \
 		-e /thrift-gen/ \
 		-e /swagger-gen/ \
+		-e /proto-gen/ \
 		-e /examples/ \
 		-e doc.go \
 		-e model.pb.go \
@@ -335,6 +337,20 @@ generate-mocks: install-mockery
 echo-version:
 	@echo $(GIT_CLOSEST_TAG)
 
+PROTO_INCLUDES := \
+	-I model/proto \
+	-I vendor/github.com/grpc-ecosystem/grpc-gateway \
+	-I vendor/github.com/gogo/googleapis \
+	-I vendor/github.com/gogo/protobuf
+# Remapping of std types to gogo types (must not contain spaces)
+PROTO_GOGO_MAPPINGS := $(shell echo \
+		Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types, \
+		Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types, \
+		Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types, \
+		Mgoogle/api/annotations.proto=github.com/gogo/googleapis/google/api, \
+		Mmodel.proto=github.com/jaegertracing/jaeger/model \
+	| sed 's/ //g')
+
 .PHONY: proto
 proto:
 	# Generate gogo, gRPC-Gateway, swagger, go-validators output.
@@ -362,31 +378,25 @@ proto:
 	# (https://medium.com/@linchenon/generate-grpc-and-protobuf-libraries-with-containers-c15ba4e4f3ad)
 	#
 	protoc \
-		-I model/proto \
-		-I vendor/github.com/grpc-ecosystem/grpc-gateway/ \
-		-I vendor/github.com/gogo/googleapis/ \
-		-I vendor/ \
-		--gogo_out=plugins=grpc,\
-Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,\
-Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types,\
-Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types,\
-Mgoogle/api/annotations.proto=github.com/gogo/googleapis/google/api:\
-$$GOPATH/src/github.com/jaegertracing/jaeger/model/ \
-		--grpc-gateway_out=\
-Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,\
-Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types,\
-Mgoogle/api/annotations.proto=github.com/gogo/googleapis/google/api:\
-$$GOPATH/src/github.com/jaegertracing/jaeger/model \
-		--swagger_out=model/proto/openapi/ \
+		$(PROTO_INCLUDES) \
+		--gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/model/ \
 		model/proto/model.proto
 
+	protoc \
+		$(PROTO_INCLUDES) \
+		--gogo_out=plugins=grpc,$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2/ \
+		--grpc-gateway_out=$(PROTO_GOGO_MAPPINGS):$(PWD)/proto-gen/api_v2/ \
+		--swagger_out=$(PWD)/proto-gen/openapi/ \
+		model/proto/api_v2.proto
+
 	protoc \
 		-I model/proto \
-		--go_out=$$GOPATH/src/github.com/jaegertracing/jaeger/model/prototest/ \
+		--go_out=$(PWD)/model/prototest/ \
 		model/proto/model_test.proto
 
 .PHONY: proto-install
 proto-install:
+	go get -u github.com/golang/glog
 	go install \
 		./vendor/github.com/golang/protobuf/protoc-gen-go \
 		./vendor/github.com/gogo/protobuf/protoc-gen-gogo \

README.md

@@ -33,7 +33,7 @@ For example, any given Jaeger installation at Uber is typically processing sever
 
 ### Native support for OpenTracing
 
-Jaeger backend, Web UI, and instrumentation libraries have been designed from ground up to support the OpenTracing standard.
+Jaeger backend, Web UI, and instrumentation libraries have been designed from ground up to support the [OpenTracing standard](https://opentracing.io/specification/).
   * Represent traces as directed acyclic graphs (not just trees) via [span references](https://github.com/opentracing/specification/blob/master/specification.md#references-between-spans)
   * Support strongly typed span _tags_ and _structured logs_
   * Support general distributed context propagation mechanism via _baggage_

cmd/agent/app/agent.go

@@ -49,6 +49,11 @@ func NewAgent(
 	return a
 }
 
+// GetServer returns HTTP server used by the agent.
+func (a *Agent) GetServer() *http.Server {
+	return a.httpServer
+}
+
 // Run runs all of agent UDP and HTTP servers in separate go-routines.
 // It returns an error when it's immediately apparent on startup, but
 // any errors happening after starting the servers are only logged.
@@ -60,6 +65,7 @@ func (a *Agent) Run() error {
 	a.httpAddr.Store(listener.Addr().String())
 	a.closer = listener
 	go func() {
+		a.logger.Info("Starting jaeger-agent HTTP server", zap.Int("http-port", listener.Addr().(*net.TCPAddr).Port))
 		if err := a.httpServer.Serve(listener); err != nil {
 			a.logger.Error("http server failure", zap.Error(err))
 		}

cmd/agent/app/agent_test.go

@@ -65,7 +65,7 @@ func TestAgentSamplingEndpoint(t *testing.T) {
 		require.NoError(t, err)
 		body, err := ioutil.ReadAll(resp.Body)
 		assert.NoError(t, err)
-		assert.Equal(t, "tcollector error: no peers available\n", string(body))
+		assert.Equal(t, "collector error: no peers available\n", string(body))
 	})
 }
 
@@ -95,21 +95,24 @@ func withRunningAgent(t *testing.T, testcase func(string, chan error)) {
 		HTTPServer: HTTPServerConfiguration{
 			HostPort: ":0",
 		},
-		Metrics: jmetrics.Builder{
-			Backend:   "prometheus",
-			HTTPRoute: "/metrics",
-		},
 	}
 	logger, logBuf := testutils.NewLogger()
-	f, _ := cfg.Metrics.CreateMetricsFactory("jaeger")
-	agent, err := cfg.CreateAgent(fakeCollectorProxy{}, logger, f)
+	//f, _ := cfg.Metrics.CreateMetricsFactory("jaeger")
+	mBldr := &jmetrics.Builder{HTTPRoute: "/metrics", Backend: "prometheus"}
+	mFactory, err := mBldr.CreateMetricsFactory("jaeger")
+	require.NoError(t, err)
+	agent, err := cfg.CreateAgent(fakeCollectorProxy{}, logger, mFactory)
 	require.NoError(t, err)
 	ch := make(chan error, 2)
 	go func() {
 		if err := agent.Run(); err != nil {
 			t.Errorf("error from agent.Run(): %s", err)
 			ch <- err
 		}
+		if h := mBldr.Handler(); mFactory != nil && h != nil {
+			logger.Info("Registering metrics handler with HTTP server", zap.String("route", mBldr.HTTPRoute))
+			agent.GetServer().Handler.(*http.ServeMux).Handle(mBldr.HTTPRoute, h)
+		}
 		close(ch)
 	}()
 

cmd/agent/app/builder.go

@@ -28,7 +28,6 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/agent/app/reporter"
 	"github.com/jaegertracing/jaeger/cmd/agent/app/servers"
 	"github.com/jaegertracing/jaeger/cmd/agent/app/servers/thriftudp"
-	jmetrics "github.com/jaegertracing/jaeger/pkg/metrics"
 	zipkinThrift "github.com/jaegertracing/jaeger/thrift-gen/agent"
 	jaegerThrift "github.com/jaegertracing/jaeger/thrift-gen/jaeger"
 )
@@ -72,7 +71,6 @@ type CollectorProxy interface {
 type Builder struct {
 	Processors []ProcessorConfiguration `yaml:"processors"`
 	HTTPServer HTTPServerConfiguration  `yaml:"httpServer"`
-	Metrics    jmetrics.Builder         `yaml:"metrics"`
 
 	reporters []reporter.Reporter
 }
@@ -110,7 +108,7 @@ func (b *Builder) CreateAgent(primaryProxy CollectorProxy, logger *zap.Logger, m
 	if err != nil {
 		return nil, err
 	}
-	server := b.HTTPServer.getHTTPServer(primaryProxy.GetManager(), mFactory, &b.Metrics)
+	server := b.HTTPServer.getHTTPServer(primaryProxy.GetManager(), mFactory)
 	return NewAgent(processors, server, logger), nil
 }
 
@@ -156,15 +154,11 @@ func (b *Builder) getProcessors(rep reporter.Reporter, mFactory metrics.Factory,
 }
 
 // GetHTTPServer creates an HTTP server that provides sampling strategies and baggage restrictions to client libraries.
-func (c HTTPServerConfiguration) getHTTPServer(manager httpserver.ClientConfigManager, mFactory metrics.Factory, mBuilder *jmetrics.Builder) *http.Server {
+func (c HTTPServerConfiguration) getHTTPServer(manager httpserver.ClientConfigManager, mFactory metrics.Factory) *http.Server {
 	if c.HostPort == "" {
 		c.HostPort = defaultHTTPServerHostPort
 	}
-	server := httpserver.NewHTTPServer(c.HostPort, manager, mFactory)
-	if h := mBuilder.Handler(); mFactory != nil && h != nil {
-		server.Handler.(*http.ServeMux).Handle(mBuilder.HTTPRoute, h)
-	}
-	return server
+	return httpserver.NewHTTPServer(c.HostPort, manager, mFactory)
 }
 
 // GetThriftProcessor gets a TBufferedServer backed Processor using the collector configuration

cmd/agent/app/builder_test.go

@@ -107,15 +107,6 @@ func TestBuilderWithExtraReporter(t *testing.T) {
 	assert.NotNil(t, agent)
 }
 
-func TestBuilderMetricsHandler(t *testing.T) {
-	b := &Builder{}
-	b.Metrics.Backend = "expvar"
-	b.Metrics.HTTPRoute = "/expvar"
-	agent, err := b.CreateAgent(fakeCollectorProxy{}, zap.NewNop(), metrics.NullFactory)
-	assert.NoError(t, err)
-	assert.NotNil(t, agent)
-}
-
 func TestBuilderWithProcessorErrors(t *testing.T) {
 	testCases := []struct {
 		model       Model

cmd/agent/app/flags.go

@@ -51,13 +51,11 @@ func AddFlags(flags *flag.FlagSet) {
 	flags.String(
 		httpServerHostPort,
 		defaultHTTPServerHostPort,
-		"host:port of the http server (e.g. for /sampling point and /baggage endpoint)")
+		"host:port of the http server (e.g. for /sampling point and /baggageRestrictions endpoint)")
 }
 
 // InitFromViper initializes Builder with properties retrieved from Viper.
 func (b *Builder) InitFromViper(v *viper.Viper) *Builder {
-	b.Metrics.InitFromViper(v)
-
 	for _, processor := range defaultProcessors {
 		prefix := fmt.Sprintf("processor.%s-%s.", processor.model, processor.protocol)
 		p := &ProcessorConfiguration{Model: processor.model, Protocol: processor.protocol}

cmd/agent/app/httpserver/server.go

@@ -108,7 +108,7 @@ func (h *httpHandler) serveSamplingHTTP(w http.ResponseWriter, r *http.Request,
 	resp, err := h.manager.GetSamplingStrategy(service)
 	if err != nil {
 		h.metrics.TCollectorProxyFailures.Inc(1)
-		http.Error(w, fmt.Sprintf("tcollector error: %+v", err), http.StatusInternalServerError)
+		http.Error(w, fmt.Sprintf("collector error: %+v", err), http.StatusInternalServerError)
 		return
 	}
 	jsonBytes, err := json.Marshal(resp)
@@ -138,7 +138,7 @@ func (h *httpHandler) serveBaggageHTTP(w http.ResponseWriter, r *http.Request) {
 	resp, err := h.manager.GetBaggageRestrictions(service)
 	if err != nil {
 		h.metrics.TCollectorProxyFailures.Inc(1)
-		http.Error(w, fmt.Sprintf("tcollector error: %+v", err), http.StatusInternalServerError)
+		http.Error(w, fmt.Sprintf("collector error: %+v", err), http.StatusInternalServerError)
 		return
 	}
 	// NB. it's literally impossible for this Marshal to fail