Skip to content

Commit

Permalink
Revert "refactor custom string query for pinot_query_validator (#6230)"
Browse files Browse the repository at this point in the history
This reverts commit 67fcf12.
  • Loading branch information
bowenxia committed Aug 21, 2024
1 parent 64301b7 commit 88763bd
Show file tree
Hide file tree
Showing 6 changed files with 19 additions and 19 deletions.
8 changes: 4 additions & 4 deletions common/persistence/pinot/pinot_visibility_store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1210,7 +1210,7 @@ LIMIT 0, 10
expectedOutput: fmt.Sprintf(`SELECT *
FROM %s
WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
AND (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*String and or order by.*'')')
AND (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'String and or order by*'))
Order BY StartTime DESC
LIMIT 0, 10
`, testTableName),
Expand All @@ -1228,7 +1228,7 @@ LIMIT 0, 10
expectedOutput: fmt.Sprintf(`SELECT *
FROM %s
WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
AND (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*Or.*'')') or JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*and.*'')'))
AND ((JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'Or*')) or (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'and*')))
Order by StartTime DESC
LIMIT 0, 10
`, testTableName),
Expand All @@ -1246,7 +1246,7 @@ LIMIT 0, 10
expectedOutput: fmt.Sprintf(`SELECT *
FROM %s
WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
AND WorkflowID = 'wid' and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*custom and custom2 or custom3 order by.*'')') or (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))
AND WorkflowID = 'wid' and ((JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))
Order BY StartTime DESC
LIMIT 0, 10
`, testTableName),
Expand Down Expand Up @@ -1300,7 +1300,7 @@ LIMIT 0, 10
expectedOutput: fmt.Sprintf(`SELECT *
FROM %s
WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
AND CloseStatus < 0 and (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10) and JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*String field is for text.*'')')
AND CloseStatus < 0 and (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10) and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'String field is for text*'))
Order by DomainID Desc
LIMIT 11, 10
`, testTableName),
Expand Down
4 changes: 2 additions & 2 deletions common/pinot/pinotQueryValidator.go
Original file line number Diff line number Diff line change
Expand Up @@ -466,8 +466,8 @@ func processCustomString(comparisonExpr *sqlparser.ComparisonExpr, colNameStr st
"AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.%s', 'string'), '^$'))", colNameStr, colNameStr)
}

return fmt.Sprintf("JSON_MATCH(Attr, '\"$.%s\" is not null') "+
"AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.%s\", ''.*%s.*'')')", colNameStr, colNameStr, colValStr)
return fmt.Sprintf("(JSON_MATCH(Attr, '\"$.%s\" is not null') "+
"AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.%s', 'string'), '%s*'))", colNameStr, colNameStr, colValStr)
}

func trimTimeFieldValueFromNanoToMilliSeconds(original *sqlparser.SQLVal) (*sqlparser.SQLVal, error) {
Expand Down
8 changes: 4 additions & 4 deletions common/pinot/pinotQueryValidator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,23 +55,23 @@ func TestValidateQuery(t *testing.T) {
err: "right comparison is invalid: &{<nil> wid { }}"},
"Case3-1: query with custom field": {
query: "CustomStringField = 'custom'",
validated: "JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*custom.*'')')",
validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom*'))",
},
"Case3-2: query with custom field value is empty": {
query: "CustomStringField = ''",
validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), '^$'))",
},
"Case4: custom field query with or in string": {
query: "CustomStringField='Or'",
validated: "JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*Or.*'')')",
validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'Or*'))",
},
"Case5: custom keyword field query": {
query: "CustomKeywordField = 'custom'",
validated: "(JSON_MATCH(Attr, '\"$.CustomKeywordField\"=''custom''') or JSON_MATCH(Attr, '\"$.CustomKeywordField[*]\"=''custom'''))",
},
"Case6-1: complex query I: with parenthesis": {
query: "(CustomStringField = 'custom and custom2 or custom3 order by') or CustomIntField between 1 and 10",
validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*custom and custom2 or custom3 order by.*'')') or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
validated: "((JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
},
"Case6-2: complex query II: with only system keys": {
query: "DomainID = 'd-id' and (RunID = 'run-id' or WorkflowID = 'wid')",
Expand All @@ -83,7 +83,7 @@ func TestValidateQuery(t *testing.T) {
},
"Case6-4: complex query IV": {
query: "WorkflowID = 'wid' and (CustomStringField = 'custom and custom2 or custom3 order by' or CustomIntField between 1 and 10)",
validated: "WorkflowID = 'wid' and (JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*custom and custom2 or custom3 order by.*'')') or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
validated: "WorkflowID = 'wid' and ((JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
},
"Case6-5: complex query with partial match": {
query: "RunID like '123' or WorkflowID like '123'",
Expand Down
6 changes: 3 additions & 3 deletions docker/buildkite/docker-compose-local-pinot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ services:
- zookeeper

pinot-controller:
image: apachepinot/pinot:1.3.0
image: apachepinot/pinot:0.12.1
command: "StartController -zkAddress zookeeper:2181 -controllerPort 9001"
container_name: pinot-controller
restart: unless-stopped
Expand All @@ -97,7 +97,7 @@ services:
aliases:
- pinot-controller
pinot-broker:
image: apachepinot/pinot:1.3.0
image: apachepinot/pinot:0.12.1
command: "StartBroker -zkAddress zookeeper:2181"
restart: unless-stopped
container_name: "pinot-broker"
Expand All @@ -112,7 +112,7 @@ services:
aliases:
- pinot-broker
pinot-server:
image: apachepinot/pinot:1.3.0
image: apachepinot/pinot:0.12.1
command: "StartServer -zkAddress zookeeper:2181"
restart: unless-stopped
container_name: "pinot-server"
Expand Down
6 changes: 3 additions & 3 deletions docker/buildkite/docker-compose-pinot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ services:
- zookeeper

pinot-controller:
image: apachepinot/pinot:latest
image: apachepinot/pinot:1.1.0
command: "StartController -zkAddress zookeeper:2181 -controllerPort 9001"
container_name: pinot-controller
restart: unless-stopped
Expand All @@ -97,7 +97,7 @@ services:
aliases:
- pinot-controller
pinot-broker:
image: apachepinot/pinot:latest
image: apachepinot/pinot:1.1.0
command: "StartBroker -zkAddress zookeeper:2181"
restart: unless-stopped
container_name: "pinot-broker"
Expand All @@ -112,7 +112,7 @@ services:
aliases:
- pinot-broker
pinot-server:
image: apachepinot/pinot:latest
image: apachepinot/pinot:1.1.0
command: "StartServer -zkAddress zookeeper:2181"
restart: unless-stopped
container_name: "pinot-server"
Expand Down
6 changes: 3 additions & 3 deletions docker/docker-compose-pinot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ services:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
pinot-controller:
image: apachepinot/pinot:1.3.0
image: apachepinot/pinot:1.1.0
command: "StartController -zkAddress zookeeper:2181"
container_name: pinot-controller
restart: unless-stopped
Expand All @@ -51,7 +51,7 @@ services:
depends_on:
- zookeeper
pinot-broker:
image: apachepinot/pinot:1.3.0
image: apachepinot/pinot:1.1.0
command: "StartBroker -zkAddress zookeeper:2181"
restart: unless-stopped
container_name: "pinot-broker"
Expand All @@ -62,7 +62,7 @@ services:
depends_on:
- pinot-controller
pinot-server:
image: apachepinot/pinot:1.3.0
image: apachepinot/pinot:1.1.0
command: "StartServer -zkAddress zookeeper:2181"
restart: unless-stopped
container_name: "pinot-server"
Expand Down

0 comments on commit 88763bd

Please sign in to comment.