[orion-scheduler] Don't use fixed scheduler id for created tasks.

services/orion-decision/src/orion_decision/__init__.py

@@ -20,7 +20,6 @@
 MAX_RUN_TIME = timedelta(hours=1)
 OWNER_EMAIL = "[email protected]"
 PROVISIONER_ID = "proj-fuzzing"
-SCHEDULER_ID = "taskcluster-github"
 SOURCE_URL = "https://github.com/MozillaSecurity/orion"
 WORKER_TYPE = "ci"
 WORKER_TYPE_MSYS = "ci-windows"

services/orion-decision/src/orion_decision/ci_scheduler.py

@@ -8,6 +8,7 @@
 from itertools import chain
 from json import dumps as json_dump
 from logging import getLogger
+from os import getenv
 from pathlib import Path
 from string import Template
 from typing import Any, Dict, List
@@ -20,7 +21,6 @@
     DEADLINE,
     MAX_RUN_TIME,
     PROVISIONER_ID,
-    SCHEDULER_ID,
     TASKCLUSTER_ROOT_URL,
     WORKER_TYPE,
     WORKER_TYPE_BREW,
@@ -50,6 +50,7 @@ class CIScheduler:
         github_event: Github event that triggered this run.
         now: Taskcluster time when decision was triggered.
         task_group: Task group to create tasks in.
+        scheduler_id: TC scheduler ID to create tasks in.
         dry_run: Calculate what should be created, but don't actually
                  create tasks in Taskcluster.
         matrix: CI job matrix
@@ -61,6 +62,7 @@ def __init__(
         github_event: GithubEvent,
         now: datetime,
         task_group: str,
+        scheduler_id: str,
         matrix: Dict[str, Any],
         dry_run: bool = False,
     ) -> None:
@@ -71,6 +73,7 @@ def __init__(
             github_event: Github event that triggered this run.
             now: Taskcluster time when decision was triggered.
             task_group: Task group to create tasks in.
+            scheduler_id: TC scheduler ID to create tasks in.
             matrix: CI job matrix
             dry_run: Calculate what should be created, but don't actually
                      create tasks in Taskcluster.
@@ -79,6 +82,7 @@ def __init__(
         self.github_event = github_event
         self.now = now
         self.task_group = task_group
+        self.scheduler_id = scheduler_id
         self.dry_run = dry_run
         assert github_event.event_type is not None
         self.matrix = CIMatrix(
@@ -142,7 +146,7 @@ def create_tasks(self) -> None:
                     "now": stringDate(self.now),
                     "project": self.project_name,
                     "provisioner": PROVISIONER_ID,
-                    "scheduler": SCHEDULER_ID,
+                    "scheduler": self.scheduler_id,
                     "task_group": self.task_group,
                     "user": self.github_event.user,
                     "worker": WORKER_TYPES[job.platform],
@@ -193,6 +197,13 @@ def main(cls, args: argparse.Namespace) -> int:
         Returns:
             Shell return code.
         """
+        # get schedulerId from TC queue
+        if args.scheduler is None:
+            task_obj = Taskcluster.get_service("queue").task(getenv("TASK_ID"))
+            scheduler_id = task_obj["schedulerId"]
+        else:
+            scheduler_id = args.scheduler
+
         # get the github event & repo
         evt = GithubEvent.from_taskcluster(args.github_action, args.github_event)
         assert evt.commit_message is not None
@@ -210,6 +221,7 @@ def main(cls, args: argparse.Namespace) -> int:
                 evt,
                 args.now,
                 args.task_group,
+                scheduler_id,
                 args.matrix,
                 args.dry_run,
             )

services/orion-decision/src/orion_decision/cli.py

@@ -99,6 +99,11 @@ def _define_decision_args(parser: argparse.ArgumentParser) -> None:
         default=getenv("TASK_ID"),
         help="Create tasks in this task group (default: TASK_ID).",
     )
+    parser.add_argument(
+        "--scheduler",
+        default=None,
+        help="Create tasks with this scheduler ID (default: fetch from TaskCluster).",
+    )
     parser.add_argument(
         "--now",
         default=getenv("TASKCLUSTER_NOW", datetime.utcnow().isoformat()),

services/orion-decision/src/orion_decision/cron.py

@@ -6,6 +6,7 @@
 from argparse import Namespace
 from datetime import datetime
 from logging import getLogger
+from os import getenv
 from typing import Optional
 
 from dateutil.parser import isoparse
@@ -26,6 +27,7 @@ class CronScheduler(Scheduler):
         repo: The Orion repo containing services.
         now: The time to calculate task times from.
         task_group: The taskGroupID to add created tasks to.
+        scheduler_id: TC scheduler ID to create tasks in.
         docker_secret: The Taskcluster secret name holding Docker Hub credentials.
         services (Services): The services
         clone_url: Git repo url
@@ -38,6 +40,7 @@ def __init__(
         repo: GitRepo,
         now: Optional[datetime],
         task_group: str,
+        scheduler_id: str,
         docker_secret: str,
         clone_url: str,
         branch: str,
@@ -49,13 +52,15 @@ def __init__(
             repo: The Orion repo containing services.
             now: The time to calculate task times from.
             task_group: The taskGroupID to add created tasks to.
+            scheduler_id: TC scheduler ID to create tasks in.
             docker_secret: The Taskcluster secret name holding Docker Hub creds.
             branch: Git main branch
             dry_run: Don't actually queue tasks in Taskcluster.
         """
         self.repo = repo
         self.now = now
         self.task_group = task_group
+        self.scheduler_id = scheduler_id
         self.docker_secret = docker_secret
         self.dry_run = dry_run
         self.clone_url = clone_url
@@ -140,6 +145,13 @@ def main(cls, args: Namespace) -> int:
         Returns:
             Shell return code.
         """
+        # get schedulerId from TC queue
+        if args.scheduler is None:
+            task_obj = Taskcluster.get_service("queue").task(getenv("TASK_ID"))
+            scheduler_id = task_obj["schedulerId"]
+        else:
+            scheduler_id = args.scheduler
+
         # clone the git repo
         repo = GitRepo(args.clone_repo, args.push_branch, args.push_branch)
         try:
@@ -148,6 +160,7 @@ def main(cls, args: Namespace) -> int:
                 repo,
                 args.now,
                 args.task_group,
+                scheduler_id,
                 args.docker_hub_secret,
                 args.clone_repo,
                 args.push_branch,

services/orion-decision/src/orion_decision/scheduler.py

@@ -7,6 +7,7 @@
 import re
 from datetime import datetime
 from logging import getLogger
+from os import getenv
 from pathlib import Path
 from string import Template
 from typing import Dict, List, Optional, Set, Union
@@ -21,7 +22,6 @@
     MAX_RUN_TIME,
     OWNER_EMAIL,
     PROVISIONER_ID,
-    SCHEDULER_ID,
     SOURCE_URL,
     WORKER_TYPE,
     WORKER_TYPE_BREW,
@@ -56,6 +56,7 @@ class Scheduler:
         github_event: The event that triggered this decision.
         now: The time to calculate task times from.
         task_group: The taskGroupID to add created tasks to.
+        scheduler_id: TC scheduler ID to create tasks in.
         docker_secret: The Taskcluster secret name holding Docker Hub credentials.
         push_branch: The branch name that should trigger a push to Docker Hub.
         services (Services): The services
@@ -67,6 +68,7 @@ def __init__(
         github_event: GithubEvent,
         now: Optional[datetime],
         task_group: str,
+        scheduler_id: str,
         docker_secret: str,
         push_branch: str,
         dry_run: bool = False,
@@ -77,13 +79,15 @@ def __init__(
             github_event: The event that triggered this decision.
             now: The time to calculate task times from.
             task_group: The taskGroupID to add created tasks to.
+            scheduler_id: TC scheduler ID to create tasks in.
             docker_secret: The Taskcluster secret name holding Docker Hub creds.
             push_branch: The branch name that should trigger a push to Docker Hub.
             dry_run: Don't actually queue tasks in Taskcluster.
         """
         self.github_event = github_event
         self.now = now
         self.task_group = task_group
+        self.scheduler_id = scheduler_id
         self.docker_secret = docker_secret
         self.push_branch = push_branch
         self.dry_run = dry_run
@@ -178,7 +182,7 @@ def _create_build_task(
                     now=stringDate(self.now),
                     owner_email=OWNER_EMAIL,
                     provisioner=PROVISIONER_ID,
-                    scheduler=SCHEDULER_ID,
+                    scheduler=self.scheduler_id,
                     service_name=service.name,
                     setup_sh_path=str(
                         (service.root / "setup.sh").relative_to(service.context)
@@ -200,7 +204,7 @@ def _create_build_task(
                     now=stringDate(self.now),
                     owner_email=OWNER_EMAIL,
                     provisioner=PROVISIONER_ID,
-                    scheduler=SCHEDULER_ID,
+                    scheduler=self.scheduler_id,
                     service_name=service.name,
                     setup_sh_path=str(
                         (service.root / "setup.sh").relative_to(service.context)
@@ -223,7 +227,7 @@ def _create_build_task(
                     now=stringDate(self.now),
                     owner_email=OWNER_EMAIL,
                     provisioner=PROVISIONER_ID,
-                    scheduler=SCHEDULER_ID,
+                    scheduler=self.scheduler_id,
                     service_name=service.name,
                     source_url=SOURCE_URL,
                     task_group=self.task_group,
@@ -255,7 +259,7 @@ def _create_push_task(self, service, service_build_tasks):
                 now=stringDate(self.now),
                 owner_email=OWNER_EMAIL,
                 provisioner=PROVISIONER_ID,
-                scheduler=SCHEDULER_ID,
+                scheduler=self.scheduler_id,
                 service_name=service.name,
                 skip_docker=f"{isinstance(service, (ServiceHomebrew, ServiceMsys)):d}",
                 source_url=SOURCE_URL,
@@ -307,7 +311,7 @@ def _create_svc_test_task(
                 now=stringDate(self.now),
                 owner_email=OWNER_EMAIL,
                 provisioner=PROVISIONER_ID,
-                scheduler=SCHEDULER_ID,
+                scheduler=self.scheduler_id,
                 service_name=service.name,
                 source_url=SOURCE_URL,
                 task_group=self.task_group,
@@ -358,7 +362,7 @@ def _create_recipe_test_task(
                 owner_email=OWNER_EMAIL,
                 provisioner=PROVISIONER_ID,
                 recipe_name=recipe.name,
-                scheduler=SCHEDULER_ID,
+                scheduler=self.scheduler_id,
                 source_url=SOURCE_URL,
                 task_group=self.task_group,
                 worker=WORKER_TYPE,
@@ -502,6 +506,13 @@ def main(cls, args: argparse.Namespace) -> int:
         Returns:
             Shell return code.
         """
+        # get schedulerId from TC queue
+        if args.scheduler is None:
+            task_obj = Taskcluster.get_service("queue").task(getenv("TASK_ID"))
+            scheduler_id = task_obj["schedulerId"]
+        else:
+            scheduler_id = args.scheduler
+
         # get the github event & repo
         evt = GithubEvent.from_taskcluster(args.github_action, args.github_event)
         try:
@@ -510,6 +521,7 @@ def main(cls, args: argparse.Namespace) -> int:
                 evt,
                 args.now,
                 args.task_group,
+                scheduler_id,
                 args.docker_hub_secret,
                 args.push_branch,
                 args.dry_run,

services/orion-decision/tests/test_cischeduler.py

@@ -13,7 +13,7 @@
 from taskcluster.utils import stringDate
 from yaml import safe_load as yaml_load
 
-from orion_decision import DEADLINE, MAX_RUN_TIME, PROVISIONER_ID, SCHEDULER_ID
+from orion_decision import DEADLINE, MAX_RUN_TIME, PROVISIONER_ID
 from orion_decision.ci_matrix import (
     CISecret,
     CISecretEnv,
@@ -59,7 +59,7 @@ def test_ci_create_01(mocker: MockerFixture) -> None:
         spec=GithubEvent(),
     )
     mocker.patch("orion_decision.ci_scheduler.CIMatrix", autospec=True)
-    sched = CIScheduler("test", evt, now, "group", {})
+    sched = CIScheduler("test", evt, now, "group", "scheduler", {})
     sched.create_tasks()
     assert queue.createTask.call_count == 0
 
@@ -139,7 +139,7 @@ def _create_secret(kind: str) -> CISecret:
         sec = _create_secret(matrix_secret)
         secrets.append(sec)
     mtx.return_value.secrets = secrets
-    sched = CIScheduler("test", evt, now, "group", {})
+    sched = CIScheduler("test", evt, now, "group", "scheduler", {})
     sched.create_tasks()
     assert queue.createTask.call_count == 1
     _, task = queue.createTask.call_args[0]
@@ -158,7 +158,7 @@ def _create_secret(kind: str) -> CISecret:
         "now": stringDate(now),
         "project": "test",
         "provisioner": PROVISIONER_ID,
-        "scheduler": SCHEDULER_ID,
+        "scheduler": "scheduler",
         "task_group": "group",
         "user": evt.user,
         "worker": WORKER_TYPES[platform],
@@ -220,7 +220,7 @@ def test_ci_create_03(mocker: MockerFixture, previous_pass: bool) -> None:
     )
     mtx.return_value.jobs = [job1, job2]
     mtx.return_value.secrets = []
-    sched = CIScheduler("test", evt, now, "group", {})
+    sched = CIScheduler("test", evt, now, "group", "scheduler", {})
     sched.create_tasks()
     assert queue.createTask.call_count == 2
     task1_id, task1 = queue.createTask.call_args_list[0][0]
@@ -236,7 +236,7 @@ def test_ci_create_03(mocker: MockerFixture, previous_pass: bool) -> None:
         "now": stringDate(now),
         "project": "test",
         "provisioner": PROVISIONER_ID,
-        "scheduler": SCHEDULER_ID,
+        "scheduler": "scheduler",
         "task_group": "group",
         "user": evt.user,
         "worker": WORKER_TYPES[job1.platform],