Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed to the Apache Software Foundation (ASF) under one
2# or more contributor license agreements. See the NOTICE file
3# distributed with this work for additional information
4# regarding copyright ownership. The ASF licenses this file
5# to you under the Apache License, Version 2.0 (the
6# "License"); you may not use this file except in compliance
7# with the License. You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing,
12# software distributed under the License is distributed on an
13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14# KIND, either express or implied. See the License for the
15# specific language governing permissions and limitations
16# under the License.
17from __future__ import annotations
19from airflow.ti_deps.dependencies_states import (
20 BACKFILL_QUEUEABLE_STATES,
21 QUEUEABLE_STATES,
22 RUNNABLE_STATES,
23)
24from airflow.ti_deps.deps.dag_ti_slots_available_dep import DagTISlotsAvailableDep
25from airflow.ti_deps.deps.dag_unpaused_dep import DagUnpausedDep
26from airflow.ti_deps.deps.dagrun_backfill_dep import DagRunNotBackfillDep
27from airflow.ti_deps.deps.dagrun_exists_dep import DagrunRunningDep
28from airflow.ti_deps.deps.exec_date_after_start_date_dep import ExecDateAfterStartDateDep
29from airflow.ti_deps.deps.pool_slots_available_dep import PoolSlotsAvailableDep
30from airflow.ti_deps.deps.runnable_exec_date_dep import RunnableExecDateDep
31from airflow.ti_deps.deps.task_concurrency_dep import TaskConcurrencyDep
32from airflow.ti_deps.deps.task_not_running_dep import TaskNotRunningDep
33from airflow.ti_deps.deps.valid_state_dep import ValidStateDep
35# Dependencies that if met, task instance should be re-queued.
36REQUEUEABLE_DEPS = {
37 DagTISlotsAvailableDep(),
38 TaskConcurrencyDep(),
39 PoolSlotsAvailableDep(),
40}
42# Dependencies that need to be met for a given task instance to be set to 'RUNNING' state.
43RUNNING_DEPS = {
44 RunnableExecDateDep(),
45 ValidStateDep(RUNNABLE_STATES),
46 DagTISlotsAvailableDep(),
47 TaskConcurrencyDep(),
48 PoolSlotsAvailableDep(),
49 TaskNotRunningDep(),
50}
52BACKFILL_QUEUED_DEPS = {
53 RunnableExecDateDep(),
54 ValidStateDep(BACKFILL_QUEUEABLE_STATES),
55 DagrunRunningDep(),
56 TaskNotRunningDep(),
57}
59# TODO(aoen): SCHEDULER_QUEUED_DEPS is not coupled to actual scheduling/execution
60# in any way and could easily be modified or removed from the scheduler causing
61# this dependency to become outdated and incorrect. This coupling should be created
62# (e.g. via a dag_deps analog of ti_deps that will be used in the scheduler code,
63# or allow batch deps checks) to ensure that the logic here is equivalent to the logic
64# in the scheduler.
65# Right now there's one discrepancy between this context and how scheduler schedule tasks:
66# Scheduler will check if the executor has the task instance--it is not possible
67# to check the executor outside scheduler main process.
69# Dependencies that need to be met for a given task instance to be set to 'queued' state
70# by the scheduler.
71# This context has more DEPs than RUNNING_DEPS, as we can have task triggered by
72# components other than scheduler, e.g. webserver.
73SCHEDULER_QUEUED_DEPS = {
74 RunnableExecDateDep(),
75 ValidStateDep(QUEUEABLE_STATES),
76 DagTISlotsAvailableDep(),
77 TaskConcurrencyDep(),
78 PoolSlotsAvailableDep(),
79 DagrunRunningDep(),
80 DagRunNotBackfillDep(),
81 DagUnpausedDep(),
82 ExecDateAfterStartDateDep(),
83 TaskNotRunningDep(),
84}