System Field Discovery via Operations Manager

chart/compass/charts/director/templates/deployment.yaml

@@ -217,8 +217,6 @@ spec:
               value: {{ .Values.global.director.selfRegister.saasAppNameLabelKey }}
             - name: APP_SELF_REGISTER_SAAS_APP_NAME_PATH
               value: {{ .Values.global.director.selfRegister.saasAppNamePath }}
-            - name: APP_SELF_REGISTER_SAAS_REGISTRY_URL_PATH
-              value: {{ .Values.global.director.selfRegister.saasRegistryURLPath }}
             - name: APP_SELF_REGISTER_PATH
               value: {{ .Values.global.director.selfRegister.path }}
             - name: APP_SELF_REGISTER_NAME_QUERY_PARAM
@@ -325,6 +323,12 @@ spec:
               value: {{ .Values.global.systemFetcher.client.timeout }}
             - name: APP_SYSTEM_FETCHER_SKIP_SSL_VALIDATION
               value: {{ $.Values.global.http.client.skipSSLValidation | quote }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_ENGINE_SAAS_REGISTRY_API
+              value: "https://{{ .Values.global.gateway.tls.secure.internal.host }}.{{ .Values.global.ingress.domainName }}{{ .Values.global.tenantFetcher.prefix }}{{ .Values.global.systemFieldDiscoveryEngine.discoveryEndpoint }}"
+            - name: APP_SYSTEM_FIELD_DISCOVERY_ENGINE_CLIENT_TIMEOUT
+              value: {{ .Values.global.systemFieldDiscoveryEngine.client.timeout }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_ENGINE_SKIP_SSL_VALIDATION
+              value: {{ $.Values.global.http.client.skipSSLValidation | quote }}
             - name: APP_SUBJECT_CONSUMER_MAPPING_CONFIG
               value: {{ .Values.global.hydrator.subjectConsumerMappingConfig | quote }}
           livenessProbe:

chart/compass/charts/external-services-mock/templates/saas-app-names-secret.yaml

@@ -7,7 +7,7 @@ metadata:
 type: Opaque
 data:
   {{- $appNamePath := .Values.global.director.selfRegister.saasAppNamePath }}
-  {{- $saasRegURLPath := .Values.global.director.selfRegister.saasRegistryURLPath }}
+  {{- $saasRegURLPath := .Values.global.systemFieldDiscoveryEngine.saasRegistryURLPath }}
   {{- $clientIDPath := .Values.global.director.selfRegister.clientIdPath }}
   {{- $clientSecretPath := .Values.global.director.selfRegister.clientSecretPath }}
   {{- $urlPath := .Values.global.director.selfRegister.urlPath }}

chart/compass/charts/tenant-fetcher/templates/deployment.yaml

@@ -44,6 +44,9 @@ spec:
       - name: {{ .Values.global.portieris.imagePullSecretName }}
     {{ end }}
       volumes:
+      - name: self-reg-app-names-volume
+        secret:
+          secretName: {{ .Values.global.systemFieldDiscoveryEngine.secrets.saasAppNameCfg.name }}
       - name: credentials-secret
         secret:
           secretName: {{ .Values.global.tenantFetcher.k8sSecret.name }}
@@ -92,6 +95,18 @@ spec:
               value: {{.Values.global.tenantFetcher.tenantProvider.consumerTenantIdProperty }}
             - name: APP_TENANT_PROVIDER_SUBSCRIPTION_PROVIDER_APP_NAME_PROPERTY
               value: {{.Values.global.tenantFetcher.tenantProvider.subscriptionProviderAppNameProperty }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_SAAS_APP_SECRET_PATH
+              value: "{{ .Values.global.systemFieldDiscoveryEngine.secrets.saasAppNameCfg.path }}/{{ .Values.global.systemFieldDiscoveryEngine.secrets.saasAppNameCfg.key }}"
+            - name: APP_SYSTEM_FIELD_DISCOVERY_OAUTH_TOKEN_PATH
+              value: {{ .Values.global.systemFieldDiscoveryEngine.oauthTokenPath }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_INSTANCE_CLIENT_ID_PATH
+              value: {{ .Values.global.systemFieldDiscoveryEngine.clientIdPath }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_CLIENT_SECRET_PATH
+              value: {{ .Values.global.systemFieldDiscoveryEngine.clientSecretPath }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_URL_PATH
+              value: {{ .Values.global.systemFieldDiscoveryEngine.urlPath }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_SAAS_REGISTRY_URL_PATH
+              value: {{ .Values.global.systemFieldDiscoveryEngine.saasRegistryURLPath }}
             - name: APP_LOG_FORMAT
               value: {{.Values.global.log.format | quote }}
             - name: APP_ADDRESS
@@ -118,6 +133,8 @@ spec:
               value: {{ .Values.global.tenantFetcher.requiredAuthScope | quote }}
             - name: APP_FETCH_TENANT_ON_DEMAND_SCOPE
               value: {{ .Values.global.tenantFetcher.fetchTenantAuthScope | quote }}
+            - name: APP_SYSTEM_FIELD_DISCOVERY_SCOPE
+              value: {{ .Values.global.tenantFetcher.systemFieldDiscoveryAuthScope | quote }}
             - name: APP_DIRECTOR_GRAPHQL_ENDPOINT
               value: "https://{{ $.Values.global.gateway.tls.secure.internal.host }}.{{ $.Values.global.ingress.domainName }}{{ $.Values.global.director.prefix }}/graphql"
             - name: APP_SELF_REGISTER_DISTINGUISH_LABEL_KEY
@@ -401,6 +418,9 @@ spec:
             timeoutSeconds: {{ .Values.global.readinessProbe.timeoutSeconds }}
             periodSeconds: {{.Values.global.readinessProbe.periodSeconds }}
           volumeMounts:
+            - name: self-reg-app-names-volume
+              mountPath: {{ .Values.global.systemFieldDiscoveryEngine.secrets.saasAppNameCfg.path }}
+              readOnly: true
             - name: dependencies-config
               mountPath: "{{ .Values.global.tenantFetcher.dependenciesConfig.path }}"
               readOnly: true

chart/compass/values.yaml

@@ -175,7 +175,7 @@ global:
       name: compass-pairing-adapter
     director:
       dir: dev/incubator/
-      version: "PR-3852"
+      version: "PR-3858"
       name: compass-director
     hydrator:
       dir: dev/incubator/
@@ -211,7 +211,7 @@ global:
       name: compass-ord-service
     schema_migrator:
       dir: dev/incubator/
-      version: "PR-3857"
+      version: "PR-3858"
       name: compass-schema-migrator
     system_broker:
       dir: prod/incubator/
@@ -352,7 +352,6 @@ global:
       requestBodyPattern: '{"key": "%s"}'
       saasAppNameLabelKey: "CMPSaaSAppName"
       saasAppNamePath: "localSaaSAppNamePath"
-      saasRegistryURLPath: "localSaaSRegistryPath"
     clientIDHeaderKey: client_user
     suggestTokenHeaderKey: suggest_token
     runtimeTypeLabelKey: "runtimeType"
@@ -757,7 +756,7 @@ global:
       path: "/oauth2/certs"
     idTokenConfig:
       claims: '{"scopes": "{{ print .Extra.scope }}","tenant": "{{ .Extra.tenant }}", "consumerID": "{{ print .Extra.consumerID}}", "consumerType": "{{ print .Extra.consumerType }}", "flow": "{{ print .Extra.flow }}", "onBehalfOf": "{{ print .Extra.onBehalfOf }}", "region": "{{ print .Extra.region }}", "tokenClientID": "{{ print .Extra.tokenClientID }}", "subject": "{{ print .Extra.subject }}"}'
-      internalClaims: '{"scopes": "application:read application:write application.webhooks:read application.application_template:read application_template.webhooks:read webhooks.auth:read runtime:write runtime:read tenant:read tenant:write tenant_subscription:write ory_internal fetch_tenant application_template:read destinations_sensitive_data:read destinations:sync ord_aggregator:sync system_fetcher:sync certificate_subject_mapping:read certificate_subject_mapping:write bundle_instance_auth:write bundle.instance_auths:read","tenant":"{ {{ if .Header.Tenant }} \"consumerTenant\":\"{{ print (index .Header.Tenant 0) }}\", {{ end }} \"externalTenant\":\"\"}", "consumerType": "Internal Component", "flow": "Internal"}'
+      internalClaims: '{"scopes": "application:read application:write application.webhooks:read application.application_template:read application_template.webhooks:read webhooks.auth:read runtime:write runtime:read tenant:read tenant:write tenant_subscription:write ory_internal fetch_tenant system_field_discovery application_template:read destinations_sensitive_data:read destinations:sync ord_aggregator:sync system_fetcher:sync certificate_subject_mapping:read certificate_subject_mapping:write bundle_instance_auth:write bundle.instance_auths:read","tenant":"{ {{ if .Header.Tenant }} \"consumerTenant\":\"{{ print (index .Header.Tenant 0) }}\", {{ end }} \"externalTenant\":\"\"}", "consumerType": "Internal Component", "flow": "Internal"}'
     mutators:
       authenticationMappingServices:
         nsadapter:
@@ -920,6 +919,7 @@ global:
     omitDependenciesParamValue: ""
     requiredAuthScope: Callback
     fetchTenantAuthScope: fetch_tenant
+    systemFieldDiscoveryAuthScope: system_field_discovery
     authentication:
       jwksEndpoint: "http://ory-stack-oathkeeper-api.ory.svc.cluster.local:4456/.well-known/jwks.json"
     tenantProvider:
@@ -1146,6 +1146,20 @@ global:
     maintainOperationsJobInterval: 60m
     operationProcessorsQuietPeriod: 5s
     asyncRequestProcessors: 100
+  systemFieldDiscoveryEngine:
+    discoveryEndpoint: /system-field-discovery
+    oauthTokenPath: "/cert/token"
+    clientIdPath: "clientId"
+    clientSecretPath: "clientSecret"
+    urlPath: "url"
+    saasRegistryURLPath: "localSaaSRegistryPath"
+    secrets:
+      saasAppNameCfg:
+        name: "saas-app-names"
+        key: "appNameConfig"
+        path: "/tmp/appNameConfig"
+    client:
+      timeout: "30s"
   tenantFetchers:
     job1:
       enabled: false

components/director/cmd/director/main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/kyma-incubator/compass/components/hydrator/pkg/certsubjmapping"
 
 	ordapiclient "github.com/kyma-incubator/compass/components/director/internal/open_resource_discovery/apiclient"
+	systemfielddiscoveryapiclient "github.com/kyma-incubator/compass/components/director/internal/system-field-discovery-engine/apiclient"
 	sfapiclient "github.com/kyma-incubator/compass/components/director/internal/systemfetcher/apiclient"
 
 	"github.com/kyma-incubator/compass/components/director/internal/domain/certsubjectmapping"
@@ -165,8 +166,6 @@ type config struct {
 
 	SelfRegConfig configprovider.SelfRegConfig
 
-	SystemFieldDiscoveryEngineConfig configprovider.SystemFieldDiscoveryEngineConfig
-
 	OperationsNamespace string `envconfig:"default=compass-system"`
 
 	DisableAsyncMode bool `envconfig:"default=false"`
@@ -194,8 +193,9 @@ type config struct {
 
 	SkipSSLValidation bool `envconfig:"default=false,APP_HTTP_CLIENT_SKIP_SSL_VALIDATION"`
 
-	OrdAggregatorClientConfig     ordapiclient.OrdAggregatorClientConfig
-	SystemFetcherSyncClientConfig sfapiclient.SystemFetcherSyncClientConfig
+	OrdAggregatorClientConfig        ordapiclient.OrdAggregatorClientConfig
+	SystemFetcherSyncClientConfig    sfapiclient.SystemFetcherSyncClientConfig
+	SystemFieldDiscoveryClientConfig systemfielddiscoveryapiclient.SystemFieldDiscoveryEngineClientConfig
 
 	ORDWebhookMappings       string `envconfig:"APP_ORD_WEBHOOK_MAPPINGS"`
 	TenantMappingConfigPath  string `envconfig:"APP_TENANT_MAPPING_CONFIG_PATH"`
@@ -323,7 +323,6 @@ func main() {
 		securedHTTPClient,
 		mtlsHTTPClient,
 		cfg.SelfRegConfig,
-		cfg.SystemFieldDiscoveryEngineConfig,
 		cfg.OneTimeToken.Length,
 		adminURL,
 		accessStrategyExecutorProvider,
@@ -336,6 +335,7 @@ func main() {
 		cfg.DestinationCreatorConfig,
 		cfg.OrdAggregatorClientConfig,
 		cfg.SystemFetcherSyncClientConfig,
+		cfg.SystemFieldDiscoveryClientConfig,
 		certSubjects,
 	)
 	exitOnError(err, "Failed to initialize root resolver")

components/director/cmd/ordaggregator/main.go

@@ -422,7 +422,7 @@ func main() {
 	}()
 
 	go func() {
-		if err := operationsManager.StartRescheduleOperationsJob(ctx); err != nil {
+		if err := operationsManager.StartRescheduleOperationsJob(ctx, []string{"COMPLETED", "FAILED"}); err != nil {
 			log.C(ctx).WithError(err).Error("Failed to run  RescheduleOperationsJob. Stopping app...")
 			cancel()
 		}

components/director/cmd/systemfetcher/main.go

@@ -262,7 +262,7 @@ func main() {
 		}()
 
 		go func() {
-			if err := operationsManager.StartRescheduleOperationsJob(ctx); err != nil {
+			if err := operationsManager.StartRescheduleOperationsJob(ctx, []string{"COMPLETED", "FAILED"}); err != nil {
 				log.C(ctx).WithError(err).Error("Failed to run  RescheduleOperationsJob. Stopping app...")
 				cancel()
 			}