1# yamllint disable rule:line-length
2# This file is rendered via JSON-e by
3# - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py
4#   {
5#     tasks_for: 'hg-push',
6#     push: {owner, comment, pushlog_id, pushdate},
7#     repository: {url, project, level},
8#     now,
9#     as_slugid: // function
10#     ownTaskId: // taskId of the task that will be created
11#   }
12#
13# - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py
14#   {
15#     tasks_for: 'cron',
16#     push: {revision, pushlog_id, pushdate, owner}
17#     repository: {url, project, level},
18#     cron: {task_id, job_name, job_symbol, quoted_args},
19#     now,
20#     ownTaskId: // taskId of the task that will be created
21#   }
22#
23# - action tasks - See:
24#   * taskcluster/gecko_taskgraph/actions/registry.py,
25#   * https://docs.taskcluster.net/docs/manual/using/actions/spec
26#   * ci-admin:ciadmin/generate/in_tree_actions.py
27#
28#   The registry generates the hookPayload that appears in actions.json, and
29#   contains data from the decision task as well as JSON-e code to combine that
30#   with data supplied as part of the action spec.  When the hook is fired, the
31#   hookPayload is rendered with JSON-e to produce a payload for the hook task
32#   template.
33#
34#   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
35#   JSON-e $let statement that produces the context described below, and
36#   installs that as the hook task template.
37#
38#   {
39#     tasks_for: 'action',
40#     push: {owner, pushlog_id, revision},
41#     repository: {url, project, level},
42#     input,
43#     taskId,      // targetted taskId
44#     taskGroupId, // targetted taskGroupId
45#     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
46#     ownTaskId:   // taskId of the task that will be created
47#     clientId:    // clientId that triggered this hook
48#   }
49---
50version: 1
51tasks:
52    # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
53    # takes place.
54    - $if: 'tasks_for in ["hg-push", "action", "cron"]'
55      then:
56          $let:
57              # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
58              ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
59              # ensure there's no trailing `/` on the repo URL
60              repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
61              # expire try earlier than other branches
62              expires:
63                  $if: 'repository.project == "try"'
64                  then: {$fromNow: '28 days'}
65                  else: {$fromNow: '1 year'}
66              trustDomain: gecko
67              treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
68          in:
69              taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
70              taskGroupId:
71                  $if: 'tasks_for == "action"'
72                  then:
73                      '${action.taskGroupId}'
74                  else:
75                      '${ownTaskId}'  # same as taskId; this is how automation identifies a decision task
76              schedulerId: '${trustDomain}-level-${repository.level}'
77
78              created: {$fromNow: ''}
79              deadline: {$fromNow: '1 day'}
80              expires: {$eval: 'expires'}
81              metadata:
82                  $merge:
83                      - owner: "${ownerEmail}"
84                        source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
85                      - $if: 'tasks_for == "hg-push"'
86                        then:
87                            name: "Gecko Decision Task"
88                            description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
89                        else:
90                            $if: 'tasks_for == "action"'
91                            then:
92                                name: "Action: ${action.title}"
93                                description: |
94                                    ${action.description}
95
96                                    ${treeherder_link}
97
98                                    Action triggered by clientID `${clientId}`
99                            else:
100                                name: "Decision Task for cron job ${cron.job_name}"
101                                description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'
102
103              provisionerId: "${trustDomain}-${repository.level}"
104              workerType: "decision"
105
106              tags:
107                  $if: 'tasks_for == "hg-push"'
108                  then:
109                      createdForUser: "${ownerEmail}"
110                      kind: decision-task
111                  else:
112                      $if: 'tasks_for == "action"'
113                      then:
114                          createdForUser: '${ownerEmail}'
115                          kind: 'action-callback'
116                      else:
117                          $if: 'tasks_for == "cron"'
118                          then:
119                              kind: cron-task
120
121              routes:
122                  $flattenDeep:
123                      - "tc-treeherder.v2.${repository.project}.${push.revision}"
124                      - $if: 'tasks_for == "hg-push"'
125                        then:
126                            - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
127                            - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
128                            - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
129                            - "notify.email.${ownerEmail}.on-failed"
130                            - "notify.email.${ownerEmail}.on-exception"
131                            # Send a notification email if the push comes from try
132                            - $if: 'repository.project == "try"'
133                              then:
134                                  "notify.email.${ownerEmail}.on-completed"
135                        else:
136                            $if: 'tasks_for == "action"'
137                            then:
138                                - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
139                                - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
140                            else:  # cron
141                                - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
142                                - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
143                                - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
144                                # list each cron task on this revision, so actions can find them
145                                - 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
146                                # BUG 1500166 Notify ciduty by email if a nightly hook fails
147                                - $if: 'repository.project != "try"'
148                                  then:
149                                      - "notify.email.ciduty+failedcron@mozilla.com.on-failed"
150                                      - "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
151                                      - "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
152                                      - "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
153
154              scopes:
155                  $if: 'tasks_for == "hg-push"'
156                  then:
157                      - 'assume:repo:${repoUrl[8:]}:branch:default'
158                      - 'queue:route:notify.email.${ownerEmail}.*'
159                      - 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
160                      - 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
161                  else:
162                      $if: 'tasks_for == "action"'
163                      then:
164                          # when all actions are hooks, we can calculate this directly rather than using a variable
165                          - '${action.repo_scope}'
166                      else:
167                          - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
168
169              dependencies: []
170              requires: all-completed
171
172              priority:
173                  # Most times, there is plenty of worker capacity so everything runs
174                  # quickly, but sometimes a storm of action tasks lands.  Then we
175                  # want, from highest to lowest:
176                  # - cron tasks (time-sensitive) (low)
177                  # - action tasks (avoid interfering with the other two) (very-low)
178                  # - decision tasks (minimize user-visible delay) (lowest)
179                  # SCM levels all use different workerTypes, so there is no need for priority
180                  # between levels; "low" is the highest priority available at all levels, and
181                  # nothing runs at any higher priority on these workerTypes.
182                  $if: "tasks_for == 'cron'"
183                  then: low
184                  else:
185                      $if: "tasks_for == 'action'"
186                      then: very-low
187                      else: lowest  # tasks_for == 'hg-push'
188              retries:
189                  $if: "tasks_for == 'hg-push'"
190                  then: 0
191                  else: 5
192
193              payload:
194                  env:
195                      # run-task uses these to check out the source; the inputs
196                      # to `mach taskgraph decision` are all on the command line.
197                      $merge:
198                          - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
199                            GECKO_HEAD_REPOSITORY: '${repoUrl}'
200                            GECKO_HEAD_REF: '${push.revision}'
201                            GECKO_HEAD_REV: '${push.revision}'
202                            HG_STORE_PATH: /builds/worker/checkouts/hg-store
203                            TASKCLUSTER_CACHES: /builds/worker/checkouts
204                            MOZ_AUTOMATION: '1'
205                            # mach generates pyc files when reading `mach_commands.py`
206                            # This causes cached_task digest generation to be random for
207                            # some tasks. Disable bytecode generation to work around that.
208                            PYTHONDONTWRITEBYTECODE: '1'
209                          - $if: 'tasks_for == "action"'
210                            then:
211                                ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task
212                                ACTION_TASK_ID: {$json: {$eval: 'taskId'}}  # taskId of the target task (JSON-encoded)
213                                ACTION_INPUT: {$json: {$eval: 'input'}}
214                                ACTION_CALLBACK: '${action.cb_name}'
215
216                  cache:
217                      "${trustDomain}-level-${repository.level}-checkouts-sparse-v2": /builds/worker/checkouts
218
219                  features:
220                      taskclusterProxy: true
221                      chainOfTrust: true
222
223                  # Note: This task is built server side without the context or tooling that
224                  # exist in tree so we must hard code the hash
225                  image: 'mozillareleases/gecko_decision:3.0.1@sha256:9964730ed951584e65032bd1d6a2b7ede6d880a8b7aefb72d0d7f82346d834e3'
226
227                  maxRunTime: 3600
228
229                  command:
230                      - /builds/worker/bin/run-task
231                      - '--gecko-checkout=/builds/worker/checkouts/gecko'
232                      - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
233                      - '--'
234                      - bash
235                      - -cx
236                      - $let:
237                            extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
238                        in:
239                            $if: 'tasks_for == "action"'
240                            then: >
241                                cd /builds/worker/checkouts/gecko &&
242                                ln -s /builds/worker/artifacts artifacts &&
243                                ./mach --log-no-times taskgraph action-callback
244                            else: >
245                                cd /builds/worker/checkouts/gecko &&
246                                ln -s /builds/worker/artifacts artifacts &&
247                                ./mach --log-no-times taskgraph decision
248                                --pushlog-id='${push.pushlog_id}'
249                                --pushdate='${push.pushdate}'
250                                --project='${repository.project}'
251                                --owner='${ownerEmail}'
252                                --level='${repository.level}'
253                                --tasks-for='${tasks_for}'
254                                --repository-type=hg
255                                --base-repository="$GECKO_BASE_REPOSITORY"
256                                --head-repository="$GECKO_HEAD_REPOSITORY"
257                                --head-ref="$GECKO_HEAD_REF"
258                                --head-rev="$GECKO_HEAD_REV"
259                                ${extraArgs}
260
261                  artifacts:
262                      'public':
263                          type: 'directory'
264                          path: '/builds/worker/artifacts'
265                          expires: {$eval: expires}
266                      'public/docker-contexts':
267                          type: 'directory'
268                          path: '/builds/worker/checkouts/gecko/docker-contexts'
269                          # This needs to be at least the deadline of the
270                          # decision task + the docker-image task deadlines.
271                          # It is set to a week to allow for some time for
272                          # debugging, but they are not useful long-term.
273                          expires: {$fromNow: '7 day'}
274
275              extra:
276                  $merge:
277                      - treeherder:
278                            $merge:
279                                - machine:
280                                      platform: gecko-decision
281                                - $if: 'tasks_for == "hg-push"'
282                                  then:
283                                      symbol: D
284                                  else:
285                                      $if: 'tasks_for == "action"'
286                                      then:
287                                          groupName: 'action-callback'
288                                          groupSymbol: AC
289                                          symbol: "${action.symbol}"
290                                      else:
291                                          groupSymbol: cron
292                                          symbol: "${cron.job_symbol}"
293                      - $if: 'tasks_for == "action"'
294                        then:
295                            parent: '${action.taskGroupId}'
296                            action:
297                                name: '${action.name}'
298                                context:
299                                    taskGroupId: '${action.taskGroupId}'
300                                    taskId: {$eval: 'taskId'}
301                                    input: {$eval: 'input'}
302                                    clientId: {$eval: 'clientId'}
303                      - $if: 'tasks_for == "cron"'
304                        then:
305                            cron: {$json: {$eval: 'cron'}}
306                      - tasks_for: '${tasks_for}'
307                      # Email for all pushes should link to treeherder
308                      - $if: 'tasks_for == "hg-push"'
309                        then:
310                            notify:
311                                email:
312                                    $merge:
313                                        - link:
314                                              text: "Treeherder Jobs"
315                                              href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"
316                                        # Email for try pushes should thank you for your revision
317                                        - $if: 'repository.project == "try"'
318                                          then:
319                                              subject: "Thank you for your try submission of ${push.revision}. It's the best!"
320                                              content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
321