diff --git a/atlas-druid/src/main/scala/com/netflix/atlas/druid/DruidClient.scala b/atlas-druid/src/main/scala/com/netflix/atlas/druid/DruidClient.scala index 391c50b0..eb18d463 100644 --- a/atlas-druid/src/main/scala/com/netflix/atlas/druid/DruidClient.scala +++ b/atlas-druid/src/main/scala/com/netflix/atlas/druid/DruidClient.scala @@ -249,14 +249,14 @@ object DruidClient { case class Datasource(dimensions: List[String], metrics: List[Metric]) - case class Metric(name: String, dataType: String = "DOUBLE") { + case class Metric(name: String, dataType: String = "doubleSum") { def isSketch: Boolean = { - dataType == "HLLSketch" + dataType == "HLLSketchMerge" } def isCounter: Boolean = - dataType == "DOUBLE" || dataType == "FLOAT" || dataType == "LONG" || isSketch + dataType == "doubleSum" || dataType == "floatSum" || dataType == "longSum" || isSketch def isTimer: Boolean = { dataType == "spectatorHistogramTimer" @@ -278,7 +278,7 @@ object DruidClient { toInclude: Option[ToInclude] = None, merge: Boolean = true, analysisTypes: List[String] = List("aggregators"), - lenientAggregatorMerge: Boolean = false + lenientAggregatorMerge: Boolean = true ) { val queryType: String = "segmentMetadata" } @@ -307,9 +307,7 @@ object DruidClient { def toDatasource: Datasource = { val dimensions = columns.filter(_._2.isDimension).keys.toList.sorted - val metrics = columns - .filter(_._1 != "__time") - .filter(_._2.isMetric) + val metrics = aggregators .map { case (name, column) => Metric(name, column.`type`) } @@ -323,11 +321,13 @@ object DruidClient { `type`: String, hasMultipleValues: Boolean, size: Long, - cardinality: Long + cardinality: Long, + errorMessage: String ) { - def isDimension: Boolean = `type` == "STRING" - def isMetric: Boolean = !isDimension + def isDimension: Boolean = `type` == "STRING" && !isError + def isMetric: Boolean = !isDimension && !isError + def isError: Boolean = errorMessage != null } case class Aggregator( diff --git a/atlas-druid/src/test/resources/segmentMetadataResponse.json b/atlas-druid/src/test/resources/segmentMetadataResponse.json index a8bf6d62..21bbaba8 100644 --- a/atlas-druid/src/test/resources/segmentMetadataResponse.json +++ b/atlas-druid/src/test/resources/segmentMetadataResponse.json @@ -76,6 +76,17 @@ "minValue": null, "maxValue": null, "errorMessage": null + }, + "changed.metric": { + "typeSignature": "STRING", + "type": "STRING", + "hasMultipleValues": false, + "hasNulls": false, + "size": -1, + "cardinality": null, + "minValue": null, + "maxValue": null, + "errorMessage": "error:cannot_merge_diff_types: [LONG] and [spectatorHistogramTimer]" } }, "size": 158732338, @@ -98,6 +109,33 @@ "compression": 50, "histogram": true, "staticBufferMax": 5000 + }, + "test.metric.hllsketch": { + "type": "HLLSketchMerge", + "name": "test.metric.hllsketch", + "fieldName": "test.metric.hllsketch", + "lgK": 12, + "tgtHllType": "HLL_4" + }, + "test.metric.histogram.dist.1": { + "type": "spectatorHistogram", + "name": "test.metric.histogram.dist.1", + "fieldName": "test.metric.histogram.dist.1" + }, + "test.metric.histogram.dist.2": { + "type": "spectatorHistogramDistribution", + "name": "test.metric.histogram.dist.2", + "fieldName": "test.metric.histogram.dist.2" + }, + "test.metric.histogram.timer": { + "type": "spectatorHistogramTimer", + "name": "test.metric.histogram.timer", + "fieldName": "test.metric.histogram.timer" + }, + "changed.metric": { + "type": "spectatorHistogramTimer", + "name": "changed.metric", + "fieldName": "changed.metric" } }, "timestampSpec": null, diff --git a/atlas-druid/src/test/scala/com/netflix/atlas/druid/DruidClientSuite.scala b/atlas-druid/src/test/scala/com/netflix/atlas/druid/DruidClientSuite.scala index c7f9fb7a..b312d8b6 100644 --- a/atlas-druid/src/test/scala/com/netflix/atlas/druid/DruidClientSuite.scala +++ b/atlas-druid/src/test/scala/com/netflix/atlas/druid/DruidClientSuite.scala @@ -144,7 +144,7 @@ class DruidClientSuite extends FunSuite { val result = executeSegmentMetadataRequest assertEquals(result.size, 1) - val columns = result.head.columns + val columns = result.head.columns.filter(!_._2.isError) val expected = Set( "__time", @@ -173,6 +173,17 @@ class DruidClientSuite extends FunSuite { test("segmentMetadata metrics") { val ds = executeSegmentMetadataRequest.head.toDatasource + + val expected = Set( + "test.metric.counter", + "test.metric.histogram.dist.1", + "test.metric.histogram.dist.2", + "test.metric.histogram.timer", + "test.metric.hllsketch", + "changed.metric" + ) + assertEquals(ds.metrics.map(m => m.name).toSet, expected) + ds.metrics.foreach { m => m.name match { case "test.metric.counter" => assert(m.isCounter) @@ -180,20 +191,57 @@ class DruidClientSuite extends FunSuite { case "test.metric.histogram.dist.2" => assert(m.isDistSummary) case "test.metric.histogram.timer" => assert(m.isTimer) case "test.metric.hllsketch" => assert(m.isSketch) + case "changed.metric" => assert(m.isTimer) case name => throw new MatchError(name) } } } + test("segmentMetadata dimensions") { + val ds = executeSegmentMetadataRequest.head.toDatasource + + val expected = Set( + "test.dim.1", + "test.dim.2" + ) + assertEquals(ds.dimensions.toSet, expected) + + ds.dimensions.foreach { d => + d match { + case "test.dim.1" => assert(true) + case "test.dim.2" => assert(true) + case name => throw new MatchError(name) + } + } + } + test("segmentMetadata aggregators") { val aggregators = executeSegmentMetadataRequest.head.aggregators - assertEquals(aggregators.size, 2) val expected = Set( "test.metric.counter", - "test.metric.histogram" + "test.metric.histogram", + "test.metric.hllsketch", + "test.metric.histogram.dist.1", + "test.metric.histogram.dist.2", + "test.metric.histogram.timer", + "changed.metric" ) assertEquals(aggregators.keySet, expected) + + aggregators.values.foreach { a => + a.name match { + case "test.metric.counter" => assertEquals(a.`type`, "longSum") + case "test.metric.histogram" => assertEquals(a.`type`, "netflixHistogram") + case "test.metric.hllsketch" => assertEquals(a.`type`, "HLLSketchMerge") + case "test.metric.histogram.dist.1" => assertEquals(a.`type`, "spectatorHistogram") + case "test.metric.histogram.dist.2" => + assertEquals(a.`type`, "spectatorHistogramDistribution") + case "test.metric.histogram.timer" => assertEquals(a.`type`, "spectatorHistogramTimer") + case "changed.metric" => assertEquals(a.`type`, "spectatorHistogramTimer") + case name => throw new MatchError(name) + } + } } private def executeGroupByRequest: List[GroupByDatapoint] = {