Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# Licensed to the Apache Software Foundation (ASF) under one
3# or more contributor license agreements. See the NOTICE file
4# distributed with this work for additional information
5# regarding copyright ownership. The ASF licenses this file
6# to you under the Apache License, Version 2.0 (the
7# "License"); you may not use this file except in compliance
8# with the License. You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing,
13# software distributed under the License is distributed on an
14# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15# KIND, either express or implied. See the License for the
16# specific language governing permissions and limitations
17# under the License.
19from __future__ import annotations
21import inspect
22import logging
23from collections.abc import Callable
24from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast
26from typing_extensions import ParamSpec
28if TYPE_CHECKING:
29 from structlog.typing import FilteringBoundLogger as Logger
31 from airflow.sdk.types import OutletEventAccessorsProtocol
33P = ParamSpec("P")
34R = TypeVar("R")
37class _ExecutionCallableRunner(Generic[P, R]):
38 @staticmethod
39 def run(*args: P.args, **kwargs: P.kwargs) -> R: ... # type: ignore[empty-body]
42class ExecutionCallableRunner(Protocol):
43 def __call__(
44 self,
45 func: Callable[P, R],
46 outlet_events: OutletEventAccessorsProtocol,
47 *,
48 logger: logging.Logger | Logger,
49 ) -> _ExecutionCallableRunner[P, R]: ...
52def create_executable_runner(
53 func: Callable[P, R],
54 outlet_events: OutletEventAccessorsProtocol,
55 *,
56 logger: logging.Logger | Logger,
57) -> _ExecutionCallableRunner[P, R]:
58 """
59 Run an execution callable against a task context and given arguments.
61 If the callable is a simple function, this simply calls it with the supplied
62 arguments (including the context). If the callable is a generator function,
63 the generator is exhausted here, with the yielded values getting fed back
64 into the task context automatically for execution.
66 This convoluted implementation of inner class with closure is so *all*
67 arguments passed to ``run()`` can be forwarded to the wrapped function. This
68 is particularly important for the argument "self", which some use cases
69 need to receive. This is not possible if this is implemented as a normal
70 class, where "self" needs to point to the runner object, not the object
71 bounded to the inner callable.
73 :meta private:
74 """
76 class _ExecutionCallableRunnerImpl(_ExecutionCallableRunner):
77 @staticmethod
78 def run(*args: P.args, **kwargs: P.kwargs) -> R:
79 from airflow.sdk.definitions.asset.metadata import Metadata
81 if not inspect.isgeneratorfunction(func):
82 return func(*args, **kwargs)
84 result: R
86 if isinstance(logger, logging.Logger):
88 def _warn_unknown(metadata):
89 logger.warning("Ignoring unknown data of %r received from task", type(metadata))
90 logger.debug("Full yielded value: %r", metadata)
91 else:
93 def _warn_unknown(metadata):
94 logger.warning("Ignoring unknown type received from task", type=type(metadata))
95 logger.debug("Full yielded value", metadata=metadata)
97 def _run():
98 nonlocal result
99 result = yield from func(*args, **kwargs)
101 for metadata in _run():
102 if isinstance(metadata, Metadata):
103 outlet_events[metadata.asset].extra.update(metadata.extra)
104 if metadata.alias:
105 outlet_events[metadata.alias].add(metadata.asset, extra=metadata.extra)
106 else:
107 _warn_unknown(metadata)
109 return result # noqa: F821 # Ruff is not smart enough to know this is always set in _run().
111 return cast("_ExecutionCallableRunner[P, R]", _ExecutionCallableRunnerImpl)