Support regex tags search for Elasticseach backend (#2049)

* Add regex tag filter for ES backend Signed-off-by: Annanay <[email protected]> * [ES] Add support to check for presence of a tag Signed-off-by: Annanay <[email protected]> * Revert "[ES] Add support to check for presence of a tag" This reverts commit f94c220. Signed-off-by: Annanay <[email protected]> * Add query builder and integration tests Signed-off-by: Annanay <[email protected]> * Fix tests Signed-off-by: Annanay <[email protected]> * Add ES specific integration tests Signed-off-by: Annanay <[email protected]> * Improve tag search tests Signed-off-by: Annanay <[email protected]> * Nit, clean function params Signed-off-by: Annanay <[email protected]>
jaegertracing · Mar 4, 2020 · e4a6ee6 · e4a6ee6
1 parent bf48066
commit e4a6ee6
Show file tree

Hide file tree

Showing 12 changed files with 407 additions and 18 deletions.
diff --git a/plugin/storage/es/spanstore/fixtures/query_01.json b/plugin/storage/es/spanstore/fixtures/query_01.json
@@ -4,9 +4,9 @@
       {
         "bool":{
           "must":{
-            "match":{
+            "regexp":{
               "tag.bat@foo":{
-                "query":"spook"
+                "value":"spook"
               }
             }
           }
@@ -15,9 +15,9 @@
       {
         "bool":{
           "must":{
-            "match":{
+            "regexp":{
               "process.tag.bat@foo":{
-                "query":"spook"
+                "value":"spook"
               }
             }
           }
@@ -37,9 +37,9 @@
                   }
                 },
                 {
-                  "match":{
+                  "regexp":{
                     "tags.value":{
-                      "query":"spook"
+                      "value":"spook"
                     }
                   }
                 }
@@ -62,9 +62,9 @@
                   }
                 },
                 {
-                  "match":{
+                  "regexp":{
                     "process.tags.value":{
-                      "query":"spook"
+                      "value":"spook"
                     }
                   }
                 }
@@ -87,9 +87,9 @@
                   }
                 },
                 {
-                  "match":{
+                  "regexp":{
                     "logs.fields.value":{
-                      "query":"spook"
+                      "value":"spook"
                     }
                   }
                 }

diff --git a/plugin/storage/es/spanstore/fixtures/query_02.json b/plugin/storage/es/spanstore/fixtures/query_02.json
@@ -0,0 +1,103 @@
+{
+  "bool":{
+    "should":[
+      {
+        "bool":{
+          "must":{
+            "regexp":{
+              "tag.bat@foo":{
+                "value":"spo.*"
+              }
+            }
+          }
+        }
+      },
+      {
+        "bool":{
+          "must":{
+            "regexp":{
+              "process.tag.bat@foo":{
+                "value":"spo.*"
+              }
+            }
+          }
+        }
+      },
+      {
+        "nested":{
+          "path":"tags",
+          "query":{
+            "bool":{
+              "must":[
+                {
+                  "match":{
+                    "tags.key":{
+                      "query":"bat.foo"
+                    }
+                  }
+                },
+                {
+                  "regexp":{
+                    "tags.value":{
+                      "value":"spo.*"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+      },
+      {
+        "nested":{
+          "path":"process.tags",
+          "query":{
+            "bool":{
+              "must":[
+                {
+                  "match":{
+                    "process.tags.key":{
+                      "query":"bat.foo"
+                    }
+                  }
+                },
+                {
+                  "regexp":{
+                    "process.tags.value":{
+                      "value":"spo.*"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+      },
+      {
+        "nested":{
+          "path":"logs.fields",
+          "query":{
+            "bool":{
+              "must":[
+                {
+                  "match":{
+                    "logs.fields.key":{
+                      "query":"bat.foo"
+                    }
+                  }
+                },
+                {
+                  "regexp":{
+                    "logs.fields.value":{
+                      "value":"spo.*"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }
+    ]
+  }
+}
diff --git a/plugin/storage/es/spanstore/fixtures/query_03.json b/plugin/storage/es/spanstore/fixtures/query_03.json
@@ -0,0 +1,103 @@
+{
+  "bool":{
+    "should":[
+      {
+        "bool":{
+          "must":{
+            "regexp":{
+              "tag.bat@foo":{
+                "value":"spo\\*"
+              }
+            }
+          }
+        }
+      },
+      {
+        "bool":{
+          "must":{
+            "regexp":{
+              "process.tag.bat@foo":{
+                "value":"spo\\*"
+              }
+            }
+          }
+        }
+      },
+      {
+        "nested":{
+          "path":"tags",
+          "query":{
+            "bool":{
+              "must":[
+                {
+                  "match":{
+                    "tags.key":{
+                      "query":"bat.foo"
+                    }
+                  }
+                },
+                {
+                  "regexp":{
+                    "tags.value":{
+                      "value":"spo\\*"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+      },
+      {
+        "nested":{
+          "path":"process.tags",
+          "query":{
+            "bool":{
+              "must":[
+                {
+                  "match":{
+                    "process.tags.key":{
+                      "query":"bat.foo"
+                    }
+                  }
+                },
+                {
+                  "regexp":{
+                    "process.tags.value":{
+                      "value":"spo\\*"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+      },
+      {
+        "nested":{
+          "path":"logs.fields",
+          "query":{
+            "bool":{
+              "must":[
+                {
+                  "match":{
+                    "logs.fields.key":{
+                      "query":"bat.foo"
+                    }
+                  }
+                },
+                {
+                  "regexp":{
+                    "logs.fields.value":{
+                      "value":"spo\\*"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }
+    ]
+  }
+}
diff --git a/plugin/storage/es/spanstore/reader.go b/plugin/storage/es/spanstore/reader.go
@@ -622,14 +622,14 @@ func (s *SpanReader) buildNestedQuery(field string, k string, v string) elastic.
 	keyField := fmt.Sprintf("%s.%s", field, tagKeyField)
 	valueField := fmt.Sprintf("%s.%s", field, tagValueField)
 	keyQuery := elastic.NewMatchQuery(keyField, k)
-	valueQuery := elastic.NewMatchQuery(valueField, v)
+	valueQuery := elastic.NewRegexpQuery(valueField, v)
 	tagBoolQuery := elastic.NewBoolQuery().Must(keyQuery, valueQuery)
 	return elastic.NewNestedQuery(field, tagBoolQuery)
 }
 
 func (s *SpanReader) buildObjectQuery(field string, k string, v string) elastic.Query {
 	keyField := fmt.Sprintf("%s.%s", field, k)
-	keyQuery := elastic.NewMatchQuery(keyField, v)
+	keyQuery := elastic.NewRegexpQuery(keyField, v)
 	return elastic.NewBoolQuery().Must(keyQuery)
 }
 

diff --git a/plugin/storage/es/spanstore/reader_test.go b/plugin/storage/es/spanstore/reader_test.go
@@ -1039,6 +1039,36 @@ func TestSpanReader_buildTagQuery(t *testing.T) {
 	})
 }
 
+func TestSpanReader_buildTagRegexQuery(t *testing.T) {
+	inStr, err := ioutil.ReadFile("fixtures/query_02.json")
+	require.NoError(t, err)
+	withSpanReader(func(r *spanReaderTest) {
+		tagQuery := r.reader.buildTagQuery("bat.foo", "spo.*")
+		actual, err := tagQuery.Source()
+		require.NoError(t, err)
+
+		expected := make(map[string]interface{})
+		json.Unmarshal(inStr, &expected)
+
+		assert.EqualValues(t, expected, actual)
+	})
+}
+
+func TestSpanReader_buildTagRegexEscapedQuery(t *testing.T) {
+	inStr, err := ioutil.ReadFile("fixtures/query_03.json")
+	require.NoError(t, err)
+	withSpanReader(func(r *spanReaderTest) {
+		tagQuery := r.reader.buildTagQuery("bat.foo", "spo\\*")
+		actual, err := tagQuery.Source()
+		require.NoError(t, err)
+
+		expected := make(map[string]interface{})
+		json.Unmarshal(inStr, &expected)
+
+		assert.EqualValues(t, expected, actual)
+	})
+}
+
 func TestSpanReader_GetEmptyIndex(t *testing.T) {
 	withSpanReader(func(r *spanReaderTest) {
 		mockSearchService(r).

diff --git a/plugin/storage/integration/elasticsearch_test.go b/plugin/storage/integration/elasticsearch_test.go
@@ -171,6 +171,8 @@ func testElasticsearchStorage(t *testing.T, allTagsAsFields, archive bool) {
 	s := &ESStorageIntegration{}
 	require.NoError(t, s.initializeES(allTagsAsFields, archive))
 
+	s.Fixtures = loadAndParseQueryTestCases(t, "fixtures/queries_es.json")
+
 	if archive {
 		t.Run("ArchiveTrace", s.testArchiveTrace)
 	} else {

diff --git a/plugin/storage/integration/fixtures/queries_es.json b/plugin/storage/integration/fixtures/queries_es.json
@@ -0,0 +1,34 @@
+[
+    {
+        "Caption": "Tag escaped operator + Operation name + max Duration",
+        "Query": {
+        "ServiceName": "query23-service",
+        "OperationName": "query23-operation",
+        "Tags": {
+            "sameplacetag1":"same\\*"
+        },
+        "StartTimeMin": "2017-01-26T15:46:31.639875Z",
+        "StartTimeMax": "2017-01-26T17:46:31.639875Z",
+        "DurationMin": 0,
+        "DurationMax": 1000,
+        "NumTraces": 1000
+        },
+        "ExpectedFixtures": ["tags_escaped_operator_trace_1"]
+    },
+    {
+        "Caption": "Tag wildcard regex",
+        "Query": {
+        "ServiceName": "query24-service",
+        "OperationName": "",
+        "Tags": {
+            "sameplacetag1":"same.*"
+        },
+        "StartTimeMin": "2017-01-26T15:46:31.639875Z",
+        "StartTimeMax": "2017-01-26T17:46:31.639875Z",
+        "DurationMin": 0,
+        "DurationMax": 0,
+        "NumTraces": 1000
+        },
+        "ExpectedFixtures": ["tags_wildcard_regex_1", "tags_wildcard_regex_2"]
+    }
+]