Merge branch 'master' of github.com:uber/cadence

.buildkite/pipeline-master.yml

@@ -34,6 +34,18 @@ agents:
 
 steps:
 
+  - label: ":on: IDL submodule points to master"
+    plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                  - |-
+                    make .idl-status
+
   - label: ":golang: unit test"
     artifact_paths:
       - ".build/coverage/*.out"

.buildkite/pipeline-pull-request.yml

@@ -34,6 +34,18 @@ agents:
 
 steps:
 
+  - label: ":on: IDL submodule points to master"
+    plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                  - |-
+                    make .idl-status
+
   - label: ":golang: unit test"
     artifact_paths:
       - ".build/coverage/*.out"

.github/workflows/docker_publish.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
       - name: Push to Docker Hub (server build)
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v2
         with:
           username: ${{ secrets.CADENCE_SERVER_DOCKERHUB_USERNAME }}
           password: ${{ secrets.CADENCE_SERVER_DOCKERHUB_TOKEN }}
@@ -31,7 +31,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
       - name: Push to Docker Hub (auto-setup build)
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v2
         with:
           username: ${{ secrets.CADENCE_SERVER_DOCKERHUB_USERNAME }}
           password: ${{ secrets.CADENCE_SERVER_DOCKERHUB_TOKEN }}

Makefile

@@ -252,6 +252,30 @@ $(STABLE_BIN)/$(PROTOC_VERSION_BIN): | $(STABLE_BIN)
 	$Q unzip -q $(STABLE_BIN)/protoc.zip -d $(PROTOC_UNZIP_DIR)
 	$Q cp $(PROTOC_UNZIP_DIR)/bin/protoc $@
 
+# checks that the idl submodule points to a commit on master, and that it matches the go module (which must be a pseudo version).
+# this is only used in an explicit CI step, because it's expected to fail when developing.
+#
+# `git ls-tree HEAD idls` is selected because this only cares about the committed/checked-out target,
+# not whatever the current status is, because only the committed value will exist for others.
+#
+# and last but not least: this avoids using `go` to make this check take only a couple seconds in CI,
+# so the whole docker container doesn't have to be prepared.
+.idl-status:
+	branches="$$(git submodule foreach git branch master --contains HEAD)"; \
+	if ! (echo "$$branches" | grep -q master); then \
+	  >&2 echo "IDL submodule points to a commit ($$(git submodule foreach git rev-parse HEAD | tail -n 1)) that is not on master."; \
+	  >&2 echo "Make sure the IDL PR has been merged, and this PR is updated, before merging here."; \
+	  exit 1; \
+	fi
+	idlsha="$$(git ls-tree HEAD idls | awk '{print substr($$3,0,12)}')"; \
+	gosha="$$(grep github.com/uber/cadence-idl go.mod | tr '-' '\n' | tail -n1)"; \
+	if [[ "$$idlsha" != "$$gosha" ]]; then \
+	  >&2 echo "IDL submodule sha ($$idlsha) does not match go module sha ($$gosha)."; \
+	  >&2 echo "Make sure the IDL PR has been merged, and this PR is updated, before merging here."; \
+	  exit 1; \
+	fi
+
+
 # ====================================
 # Codegen targets
 # ====================================

common/metrics/scope.go

@@ -120,7 +120,7 @@ func (m *metricsScope) RecordHistogramValue(id int, value float64) {
 }
 
 func (m *metricsScope) Tagged(tags ...Tag) Scope {
-	domainTagged := false
+	domainTagged := m.isDomainTagged
 	tagMap := make(map[string]string, len(tags))
 	for _, tag := range tags {
 		if isDomainTagged(tag) {

common/persistence/pinot/pinot_visibility_store_test.go

@@ -1210,7 +1210,7 @@ LIMIT 0, 10
 			expectedOutput: fmt.Sprintf(`SELECT *
 FROM %s
 WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
-AND (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'String and or order by*'))
+AND (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*String and or order by.*'')')
 Order BY StartTime DESC
 LIMIT 0, 10
 `, testTableName),
@@ -1228,7 +1228,7 @@ LIMIT 0, 10
 			expectedOutput: fmt.Sprintf(`SELECT *
 FROM %s
 WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
-AND ((JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'Or*')) or (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'and*')))
+AND (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*Or.*'')') or JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*and.*'')'))
 Order by StartTime DESC
 LIMIT 0, 10
 `, testTableName),
@@ -1246,7 +1246,7 @@ LIMIT 0, 10
 			expectedOutput: fmt.Sprintf(`SELECT *
 FROM %s
 WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
-AND WorkflowID = 'wid' and ((JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))
+AND WorkflowID = 'wid' and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*custom and custom2 or custom3 order by.*'')') or (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))
 Order BY StartTime DESC
 LIMIT 0, 10
 `, testTableName),
@@ -1300,7 +1300,7 @@ LIMIT 0, 10
 			expectedOutput: fmt.Sprintf(`SELECT *
 FROM %s
 WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd'
-AND CloseStatus < 0 and (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10) and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'String field is for text*'))
+AND CloseStatus < 0 and (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and (JSON_MATCH(Attr, '"$.CustomIntField" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10) and JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE("$.CustomStringField", ''.*String field is for text.*'')')
 Order by DomainID Desc
 LIMIT 11, 10
 `, testTableName),

common/pinot/pinotQueryValidator.go

@@ -466,8 +466,8 @@ func processCustomString(comparisonExpr *sqlparser.ComparisonExpr, colNameStr st
 			"AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.%s', 'string'), '^$'))", colNameStr, colNameStr)
 	}
 
-	return fmt.Sprintf("(JSON_MATCH(Attr, '\"$.%s\" is not null') "+
-		"AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.%s', 'string'), '%s*'))", colNameStr, colNameStr, colValStr)
+	return fmt.Sprintf("JSON_MATCH(Attr, '\"$.%s\" is not null') "+
+		"AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.%s\", ''.*%s.*'')')", colNameStr, colNameStr, colValStr)
 }
 
 func trimTimeFieldValueFromNanoToMilliSeconds(original *sqlparser.SQLVal) (*sqlparser.SQLVal, error) {

common/pinot/pinotQueryValidator_test.go

@@ -55,23 +55,23 @@ func TestValidateQuery(t *testing.T) {
 			err:   "right comparison is invalid: &{<nil> wid { }}"},
 		"Case3-1: query with custom field": {
 			query:     "CustomStringField = 'custom'",
-			validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom*'))",
+			validated: "JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*custom.*'')')",
 		},
 		"Case3-2: query with custom field value is empty": {
 			query:     "CustomStringField = ''",
 			validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), '^$'))",
 		},
 		"Case4: custom field query with or in string": {
 			query:     "CustomStringField='Or'",
-			validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'Or*'))",
+			validated: "JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*Or.*'')')",
 		},
 		"Case5: custom keyword field query": {
 			query:     "CustomKeywordField = 'custom'",
 			validated: "(JSON_MATCH(Attr, '\"$.CustomKeywordField\"=''custom''') or JSON_MATCH(Attr, '\"$.CustomKeywordField[*]\"=''custom'''))",
 		},
 		"Case6-1: complex query I: with parenthesis": {
 			query:     "(CustomStringField = 'custom and custom2 or custom3 order by') or CustomIntField between 1 and 10",
-			validated: "((JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
+			validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*custom and custom2 or custom3 order by.*'')') or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
 		},
 		"Case6-2: complex query II: with only system keys": {
 			query:     "DomainID = 'd-id' and (RunID = 'run-id' or WorkflowID = 'wid')",
@@ -83,7 +83,7 @@ func TestValidateQuery(t *testing.T) {
 		},
 		"Case6-4: complex query IV": {
 			query:     "WorkflowID = 'wid' and (CustomStringField = 'custom and custom2 or custom3 order by' or CustomIntField between 1 and 10)",
-			validated: "WorkflowID = 'wid' and ((JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
+			validated: "WorkflowID = 'wid' and (JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND JSON_MATCH(Attr, 'REGEXP_LIKE(\"$.CustomStringField\", ''.*custom and custom2 or custom3 order by.*'')') or (JSON_MATCH(Attr, '\"$.CustomIntField\" is not null') AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) >= 1 AND CAST(JSON_EXTRACT_SCALAR(Attr, '$.CustomIntField') AS INT) <= 10))",
 		},
 		"Case6-5: complex query with partial match": {
 			query:     "RunID like '123' or WorkflowID like '123'",

docker/buildkite/docker-compose-local-pinot.yml

@@ -82,7 +82,7 @@ services:
           - zookeeper
 
   pinot-controller:
-    image: apachepinot/pinot:0.12.1
+    image: apachepinot/pinot:1.3.0
     command: "StartController -zkAddress zookeeper:2181 -controllerPort 9001"
     container_name: pinot-controller
     restart: unless-stopped
@@ -97,7 +97,7 @@ services:
         aliases:
           - pinot-controller
   pinot-broker:
-    image: apachepinot/pinot:0.12.1
+    image: apachepinot/pinot:1.3.0
     command: "StartBroker -zkAddress zookeeper:2181"
     restart: unless-stopped
     container_name: "pinot-broker"
@@ -112,7 +112,7 @@ services:
         aliases:
           - pinot-broker
   pinot-server:
-    image: apachepinot/pinot:0.12.1
+    image: apachepinot/pinot:1.3.0
     command: "StartServer -zkAddress zookeeper:2181"
     restart: unless-stopped
     container_name: "pinot-server"

docker/buildkite/docker-compose-pinot.yml

@@ -82,7 +82,7 @@ services:
           - zookeeper
 
   pinot-controller:
-    image: apachepinot/pinot:1.1.0
+    image: apachepinot/pinot:latest
     command: "StartController -zkAddress zookeeper:2181 -controllerPort 9001"
     container_name: pinot-controller
     restart: unless-stopped
@@ -97,7 +97,7 @@ services:
         aliases:
           - pinot-controller
   pinot-broker:
-    image: apachepinot/pinot:1.1.0
+    image: apachepinot/pinot:latest
     command: "StartBroker -zkAddress zookeeper:2181"
     restart: unless-stopped
     container_name: "pinot-broker"
@@ -112,7 +112,7 @@ services:
         aliases:
           - pinot-broker
   pinot-server:
-    image: apachepinot/pinot:1.1.0
+    image: apachepinot/pinot:latest
     command: "StartServer -zkAddress zookeeper:2181"
     restart: unless-stopped
     container_name: "pinot-server"

docker/docker-compose-pinot.yml

@@ -40,7 +40,7 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
   pinot-controller:
-    image: apachepinot/pinot:1.1.0
+    image: apachepinot/pinot:1.3.0
     command: "StartController -zkAddress zookeeper:2181"
     container_name: pinot-controller
     restart: unless-stopped
@@ -51,7 +51,7 @@ services:
     depends_on:
       - zookeeper
   pinot-broker:
-    image: apachepinot/pinot:1.1.0
+    image: apachepinot/pinot:1.3.0
     command: "StartBroker -zkAddress zookeeper:2181"
     restart: unless-stopped
     container_name: "pinot-broker"
@@ -62,7 +62,7 @@ services:
     depends_on:
       - pinot-controller
   pinot-server:
-    image: apachepinot/pinot:1.1.0
+    image: apachepinot/pinot:1.3.0
     command: "StartServer -zkAddress zookeeper:2181"
     restart: unless-stopped
     container_name: "pinot-server"

service/frontend/templates/metered.tmpl

@@ -12,6 +12,7 @@ import (
 {{- $nonDomainSpecificAPIs := list "Health" "DeprecateDomain" "DescribeDomain" "ListDomains" "RegisterDomain" "UpdateDomain" "GetSearchAttributes" "GetClusterInfo"}}
 {{- $domainIDAPIs := list "RecordActivityTaskHeartbeat" "RespondActivityTaskCanceled" "RespondActivityTaskCompleted" "RespondActivityTaskFailed" "RespondDecisionTaskCompleted" "RespondDecisionTaskFailed" "RespondQueryTaskCompleted"}}
 {{- $queryTaskTokenAPIs := list "RespondQueryTaskCompleted"}}
+{{- $pollerAPIs := list "PollForActivityTask" "PollForDecisionTask"}}
 
 {{- $interfaceName := .Interface.Name}}
 {{- $interfaceType := .Interface.Type}}
@@ -81,10 +82,19 @@ func (h *{{$decorator}}) {{$method.Declaration}} {
 	{{- end}}
 	{{- $domainMetricTag = printf "metrics.DomainTag(%s)" $domain}}
 	{{- end}}
-	scope := h.metricsClient.Scope({{$scope}}).Tagged({{$domainMetricTag}}).Tagged(metrics.GetContextTags(ctx)...)
+	{{- if has $method.Name $pollerAPIs}}
+	scope := common.NewPerTaskListScope({{(index $method.Params 1).Name}}.Domain, {{(index $method.Params 1).Name}}.TaskList.GetName(), {{(index $method.Params 1).Name}}.TaskList.GetKind(), h.metricsClient, {{$scope}}).Tagged(metrics.GetContextTags(ctx)...)
+	scope.IncCounter(metrics.CadenceRequestsPerTaskList)
+	sw := scope.StartTimer(metrics.CadenceLatencyPerTaskList)
+	defer sw.Stop()
+	swPerDomain := h.metricsClient.Scope({{$scope}}).Tagged(append(metrics.GetContextTags(ctx), {{$domainMetricTag}})...).StartTimer(metrics.CadenceLatency)
+	defer swPerDomain.Stop()
+	{{- else}}
+	scope := h.metricsClient.Scope({{$scope}}).Tagged(append(metrics.GetContextTags(ctx), {{$domainMetricTag}})...)
 	scope.IncCounter(metrics.CadenceRequests)
 	sw := scope.StartTimer(metrics.CadenceLatency)
 	defer sw.Stop()
+	{{- end}}
 	logger := h.logger.WithTags(tags...)
 
 	{{$method.ResultsNames}} = h.handler.{{$method.Call}}