Merge branch 'master' into jaeger-1359

CHANGELOG.md

@@ -1,7 +1,21 @@
 Changes by Version
 ==================
 
-1.14.0 (unreleased)
+1.15.0 (unreleased)
+------------------
+
+#### Backend Changes
+
+##### Breaking Changes
+
+##### New Features
+
+##### Bug fixes, Minor Improvements
+
+#### UI Changes
+
+
+1.14.0 (2019-09-02)
 ------------------
 
 #### Backend Changes
@@ -15,10 +29,51 @@ Changes by Version
 
 ##### New Features
 
+* Add Elasticsearch version configuration to rollover script ([#1769](https://github.com/jaegertracing/jaeger/pull/1769), [@pavolloffay](https://github.com/pavolloffay))
+* Add Elasticsearch version flag ([#1753](https://github.com/jaegertracing/jaeger/pull/1753), [@pavolloffay](https://github.com/pavolloffay))
+* Add Elasticsearch 7 support ([#1690](https://github.com/jaegertracing/jaeger/pull/1690), [@gregoryfranklin](https://github.com/gregoryfranklin))
+
+  The index mappings in Elasticsearch 7 are not backwards compatible with the older versions. 
+  Therefore using Elasticsearch 7 with data created with older version would not work.
+  Elasticsearch 6.8 supports 7.x, 6.x, 5.x compatible mappings. The upgrade has to be done
+  first to ES 6.8, then apply data migration or wait until old daily indices are removed (this requires
+  to start Jaeger with `--es.version=7` to force using ES 7.x mappings for newly created indices).
+
+  Jaeger by default uses Elasticsearch ping endpoint (`/`) to derive the version which is used
+  for index mappings selection. The version can be overridden by flag `--es.version`.
+
+* Support for Zipkin Protobuf spans over HTTP ([#1695](https://github.com/jaegertracing/jaeger/pull/1695), [@jan25](https://github.com/jan25))
+* Added support for hot reload of UI config ([#1688](https://github.com/jaegertracing/jaeger/pull/1688), [@jpkrohling](https://github.com/jpkrohling))
+* Added base Grafana dashboard and Alert rules ([#1745](https://github.com/jaegertracing/jaeger/pull/1745), [@jpkrohling](https://github.com/jpkrohling))
+* Add the jaeger-mixin for monitoring ([#1668](https://github.com/jaegertracing/jaeger/pull/1668), [@gouthamve](https://github.com/gouthamve))
+* Added flags for driving cassandra connection compression through config ([#1675](https://github.com/jaegertracing/jaeger/pull/1675), [@sagaranand015](https://github.com/sagaranand015))
+* Support index cleaner for rollover indices and add integration tests ([#1689](https://github.com/jaegertracing/jaeger/pull/1689), [@pavolloffay](https://github.com/pavolloffay))
+* Add client TLS auth to gRPC reporter ([#1591](https://github.com/jaegertracing/jaeger/pull/1591), [@tcolgate](https://github.com/tcolgate))
+* Collector kafka producer protocol version config ([#1658](https://github.com/jaegertracing/jaeger/pull/1658), [@marqc](https://github.com/marqc))
+* Configurable kafka protocol version for msg consuming by jaeger ingester ([#1640](https://github.com/jaegertracing/jaeger/pull/1640), [@marqc](https://github.com/marqc))
+* Use credentials when describing keyspaces in cassandra schema builder ([#1655](https://github.com/jaegertracing/jaeger/pull/1655), [@MiLk](https://github.com/MiLk))
+* Add connect-timeout for Cassandra ([#1647](https://github.com/jaegertracing/jaeger/pull/1647), [@sagaranand015](https://github.com/sagaranand015))
+
 ##### Bug fixes, Minor Improvements
 
+* Fix gRPC over cmux and add unit tests ([#1758](https://github.com/jaegertracing/jaeger/pull/1758), [@yurishkuro](https://github.com/yurishkuro))
+* Add CA certificates to agent image ([#1764](https://github.com/jaegertracing/jaeger/pull/1764), [@yurishkuro](https://github.com/yurishkuro))
+* Fix badger merge-join algorithm to correctly filter indexes ([#1721](https://github.com/jaegertracing/jaeger/pull/1721), [@burmanm](https://github.com/burmanm))
+* Change Zipkin CORS origins and headers to comma separated list ([#1556](https://github.com/jaegertracing/jaeger/pull/1556), [@JonasVerhofste](https://github.com/JonasVerhofste))
+* Added null guards to 'Process' when processing an incoming span ([#1723](https://github.com/jaegertracing/jaeger/pull/1723), [@jpkrohling](https://github.com/jpkrohling))
+* Export expvar metrics of badger to the metricsFactory ([#1704](https://github.com/jaegertracing/jaeger/pull/1704), [@burmanm](https://github.com/burmanm))
+* Pass TTL as int, not as float64 ([#1710](https://github.com/jaegertracing/jaeger/pull/1710), [@yurishkuro](https://github.com/yurishkuro))
+* Use find by regex for archive index in index cleaner ([#1693](https://github.com/jaegertracing/jaeger/pull/1693), [@pavolloffay](https://github.com/pavolloffay))
+* Allow token propagation if token type is not specified ([#1685](https://github.com/jaegertracing/jaeger/pull/1685), [@rubenvp8510](https://github.com/rubenvp8510))
+* Fix duplicated spans when querying Elasticsearch ([#1677](https://github.com/jaegertracing/jaeger/pull/1677), [@pavolloffay](https://github.com/pavolloffay))
+* Fix the threshold precision issue ([#1665](https://github.com/jaegertracing/jaeger/pull/1665), [@guanw](https://github.com/guanw))
+* Badger filter duplicate results from a single indexSeek ([#1649](https://github.com/jaegertracing/jaeger/pull/1649), [@burmanm](https://github.com/burmanm))
+* Badger make default dirs work in Windows ([#1653](https://github.com/jaegertracing/jaeger/pull/1653), [@burmanm](https://github.com/burmanm))
+
 #### UI Changes
 
+* UI pinned to version 1.4.0. The changelog is available here [v1.4.0](https://github.com/jaegertracing/jaeger-ui/blob/master/CHANGELOG.md#v130-june-21-2019)
+
 1.13.1 (2019-06-28)
 ------------------
 

Gopkg.toml

@@ -116,8 +116,8 @@ required = [
   version = "=1.20.1"
 
 [[constraint]]
-  name = "gopkg.in/olivere/elastic.v5"
-  version = "^5.0.53"
+  name = "github.com/olivere/elastic"
+  version = "^6.2.22"
 
 [[constraint]]
   name = "github.com/gocql/gocql"

cmd/agent/Dockerfile

@@ -1,5 +1,10 @@
+FROM alpine:latest as certs
+RUN apk add --update --no-cache ca-certificates
+
 FROM scratch
 
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
 EXPOSE 5775/udp 6831/udp 6832/udp 5778
 COPY agent-linux /go/bin/
 ENTRYPOINT ["/go/bin/agent-linux"]

cmd/collector/app/metrics.go

@@ -126,7 +126,7 @@ type SpanCountsByFormat map[SpanFormat]SpanCountsByTransport
 // SpanCountsByTransport groups metrics by inbound transport (e.g http, grpc, tchannel)
 type SpanCountsByTransport map[InboundTransport]SpanCounts
 
-// SpanCounts contains countrs for received and rejected spans.
+// SpanCounts contains counts for received and rejected spans.
 type SpanCounts struct {
 	// ReceivedBySvc maintain by-service metrics.
 	ReceivedBySvc metricsBySvc

cmd/collector/app/service_name_normalizer.go

@@ -42,7 +42,7 @@ func newServiceNameReplacer() *strings.Replacer {
 		mapping[i] = byte(i)
 
 	}
-	// upper case latters are safe, but convert them to lower case
+	// upper case letters are safe, but convert them to lower case
 	for i := 'A'; i <= 'Z'; i++ {
 		mapping[i] = byte(i - 'A' + 'a')
 	}

cmd/collector/app/zipkin/http_handler_test.go

@@ -295,9 +295,9 @@ func TestSaveProtoSpansV2(t *testing.T) {
 		resBody    string
 	}{
 		{Span: zipkinProto.Span{Id: validID, TraceId: validTraceID, LocalEndpoint: &zipkinProto.Endpoint{Ipv4: randBytesOfLen(4)}, Kind: zipkinProto.Span_CLIENT}, StatusCode: http.StatusAccepted},
-		{Span: zipkinProto.Span{Id: randBytesOfLen(4)}, StatusCode: http.StatusBadRequest, resBody: "Unable to process request body: invalid length for Span ID\n"},
-		{Span: zipkinProto.Span{Id: validID, TraceId: randBytesOfLen(32)}, StatusCode: http.StatusBadRequest, resBody: "Unable to process request body: invalid length for traceId\n"},
-		{Span: zipkinProto.Span{Id: validID, TraceId: validTraceID, ParentId: randBytesOfLen(16)}, StatusCode: http.StatusBadRequest, resBody: "Unable to process request body: invalid length for parentId\n"},
+		{Span: zipkinProto.Span{Id: randBytesOfLen(4)}, StatusCode: http.StatusBadRequest, resBody: "Unable to process request body: invalid length for SpanID\n"},
+		{Span: zipkinProto.Span{Id: validID, TraceId: randBytesOfLen(32)}, StatusCode: http.StatusBadRequest, resBody: "Unable to process request body: invalid length for TraceID\n"},
+		{Span: zipkinProto.Span{Id: validID, TraceId: validTraceID, ParentId: randBytesOfLen(16)}, StatusCode: http.StatusBadRequest, resBody: "Unable to process request body: invalid length for SpanID\n"},
 		{Span: zipkinProto.Span{Id: validID, TraceId: validTraceID, LocalEndpoint: &zipkinProto.Endpoint{Ipv4: randBytesOfLen(2)}}, StatusCode: http.StatusBadRequest, resBody: "Unable to process request body: wrong Ipv4\n"},
 	}
 	for _, test := range tests {

cmd/collector/app/zipkin/protov2.go

@@ -38,14 +38,17 @@ func protoSpansV2ToThrift(listOfSpans *zipkinProto.ListOfSpans) ([]*zipkincore.S
 }
 
 func protoSpanV2ToThrift(s *zipkinProto.Span) (*zipkincore.Span, error) {
-	if len(s.Id) != model.TraceIDShortBytesLen {
-		return nil, fmt.Errorf("invalid length for Span ID")
+	var id model.SpanID
+	var err error
+	if id, err = model.SpanIDFromBytes(s.Id); err != nil {
+		return nil, err
 	}
-	id := binary.BigEndian.Uint64(s.Id)
-	traceID, err := traceIDFromBytes(s.TraceId)
-	if err != nil {
+
+	var traceID model.TraceID
+	if traceID, err = model.TraceIDFromBytes(s.TraceId); err != nil {
 		return nil, err
 	}
+
 	ts, d := int64(s.Timestamp), int64(s.Duration)
 	tSpan := &zipkincore.Span{
 		ID:        int64(id),
@@ -61,18 +64,17 @@ func protoSpanV2ToThrift(s *zipkinProto.Span) (*zipkincore.Span, error) {
 	}
 
 	if len(s.ParentId) > 0 {
-		if len(s.ParentId) != model.TraceIDShortBytesLen {
-			return nil, fmt.Errorf("invalid length for parentId")
+		var parentID model.SpanID
+		if parentID, err = model.SpanIDFromBytes(s.ParentId); err != nil {
+			return nil, err
 		}
-		parentID := binary.BigEndian.Uint64(s.ParentId)
 		signed := int64(parentID)
 		tSpan.ParentID = &signed
 	}
 
 	var localE *zipkincore.Endpoint
 	if s.LocalEndpoint != nil {
-		localE, err = protoEndpointV2ToThrift(s.LocalEndpoint)
-		if err != nil {
+		if localE, err = protoEndpointV2ToThrift(s.LocalEndpoint); err != nil {
 			return nil, err
 		}
 	}
@@ -105,21 +107,6 @@ func protoSpanV2ToThrift(s *zipkinProto.Span) (*zipkincore.Span, error) {
 	return tSpan, nil
 }
 
-func traceIDFromBytes(tid []byte) (model.TraceID, error) {
-	var hi, lo uint64
-	switch {
-	case len(tid) > model.TraceIDLongBytesLen:
-		return model.TraceID{}, fmt.Errorf("invalid length for traceId")
-	case len(tid) > model.TraceIDShortBytesLen:
-		hiLen := len(tid) - model.TraceIDShortBytesLen
-		hi = binary.BigEndian.Uint64(tid[:hiLen])
-		lo = binary.BigEndian.Uint64(tid[hiLen:])
-	default:
-		lo = binary.BigEndian.Uint64(tid)
-	}
-	return model.TraceID{High: hi, Low: lo}, nil
-}
-
 func protoRemoteEndpToAddrAnno(e *zipkinProto.Endpoint, kind zipkinProto.Span_Kind) (*zipkincore.BinaryAnnotation, error) {
 	rEndp, err := protoEndpointV2ToThrift(e)
 	if err != nil {

cmd/collector/app/zipkin/protov2_test.go

@@ -82,9 +82,9 @@ func TestIdErrs(t *testing.T) {
 		span   zipkinProto.Span
 		errMsg string
 	}{
-		{span: zipkinProto.Span{Id: randBytesOfLen(16)}, errMsg: "invalid length for Span ID"},
-		{span: zipkinProto.Span{Id: validID, TraceId: invalidTraceID}, errMsg: "invalid length for traceId"},
-		{span: zipkinProto.Span{Id: validID, TraceId: validTraceID, ParentId: invalidParentID}, errMsg: "invalid length for parentId"},
+		{span: zipkinProto.Span{Id: randBytesOfLen(16)}, errMsg: "invalid length for SpanID"},
+		{span: zipkinProto.Span{Id: validID, TraceId: invalidTraceID}, errMsg: "invalid length for TraceID"},
+		{span: zipkinProto.Span{Id: validID, TraceId: validTraceID, ParentId: invalidParentID}, errMsg: "invalid length for SpanID"},
 	}
 	for _, test := range tests {
 		_, err := protoSpanV2ToThrift(&test.span)

cmd/query/app/grpc_handler_test.go

@@ -40,7 +40,6 @@ import (
 )
 
 var (
-	grpcServerPort       = ":0"
 	errStorageMsgGRPC    = "Storage error"
 	errStorageGRPC       = errors.New(errStorageMsgGRPC)
 	errStatusStorageGRPC = status.Error(2, errStorageMsgGRPC)
@@ -138,7 +137,7 @@ type grpcClient struct {
 }
 
 func newGRPCServer(t *testing.T, q *querysvc.QueryService, logger *zap.Logger, tracer opentracing.Tracer) (*grpc.Server, net.Addr) {
-	lis, _ := net.Listen("tcp", grpcServerPort)
+	lis, _ := net.Listen("tcp", ":0")
 	grpcServer := grpc.NewServer()
 	grpcHandler := NewGRPCHandler(q, logger, tracer)
 	api_v2.RegisterQueryServiceServer(grpcServer, grpcHandler)
@@ -151,8 +150,10 @@ func newGRPCServer(t *testing.T, q *querysvc.QueryService, logger *zap.Logger, t
 	return grpcServer, lis.Addr()
 }
 
-func newGRPCClient(t *testing.T, addr net.Addr) *grpcClient {
-	conn, err := grpc.Dial(addr.String(), grpc.WithInsecure())
+func newGRPCClient(t *testing.T, addr string) *grpcClient {
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure())
 	require.NoError(t, err)
 
 	return &grpcClient{
@@ -192,7 +193,7 @@ func initializeTestServerGRPCWithOptions(t *testing.T) *grpcServer {
 
 func withServerAndClient(t *testing.T, actualTest func(server *grpcServer, client *grpcClient)) {
 	server := initializeTestServerGRPCWithOptions(t)
-	client := newGRPCClient(t, server.lisAddr)
+	client := newGRPCClient(t, server.lisAddr.String())
 	defer server.server.Stop()
 	defer client.conn.Close()
 

cmd/query/app/server.go

@@ -102,9 +102,10 @@ func (s *Server) Start() error {
 	// cmux server acts as a reverse-proxy between HTTP and GRPC backends.
 	cmuxServer := cmux.New(s.conn)
 
-	grpcListener := cmuxServer.Match(
-		cmux.HTTP2HeaderField("content-type", "application/grpc"),
-		cmux.HTTP2HeaderField("content-type", "application/grpc+proto"))
+	grpcListener := cmuxServer.MatchWithWriters(
+		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"),
+		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc+proto"),
+	)
 	httpListener := cmuxServer.Match(cmux.Any())
 
 	go func() {