Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/airflow/sdk/execution_time/callback_runner.py: 33%

1# 

2# Licensed to the Apache Software Foundation (ASF) under one 

3# or more contributor license agreements. See the NOTICE file 

4# distributed with this work for additional information 

5# regarding copyright ownership. The ASF licenses this file 

6# to you under the Apache License, Version 2.0 (the 

7# "License"); you may not use this file except in compliance 

8# with the License. You may obtain a copy of the License at 

9# 

10# http://www.apache.org/licenses/LICENSE-2.0 

11# 

12# Unless required by applicable law or agreed to in writing, 

13# software distributed under the License is distributed on an 

14# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 

15# KIND, either express or implied. See the License for the 

16# specific language governing permissions and limitations 

17# under the License. 

18 

19from __future__ import annotations 

20 

21import inspect 

22import logging 

23from collections.abc import AsyncIterator, Awaitable, Callable 

24from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar, cast 

25 

26from typing_extensions import ParamSpec 

27 

28if TYPE_CHECKING: 

29 from structlog.typing import FilteringBoundLogger as Logger 

30 

31 from airflow.sdk.types import OutletEventAccessorsProtocol 

32 

33P = ParamSpec("P") 

34R = TypeVar("R") 

35 

36 

37class _ExecutionCallableRunner(Generic[P, R]): 

38 @staticmethod 

39 def run(*args: P.args, **kwargs: P.kwargs) -> R: ... # type: ignore[empty-body] 

40 

41 

42class _AsyncExecutionCallableRunner(Generic[P, R]): 

43 @staticmethod 

44 async def run(*args: P.args, **kwargs: P.kwargs) -> R: ... # type: ignore[empty-body] 

45 

46 

47class ExecutionCallableRunner(Protocol): 

48 def __call__( 

49 self, 

50 func: Callable[P, R], 

51 outlet_events: OutletEventAccessorsProtocol, 

52 *, 

53 logger: logging.Logger | Logger, 

54 ) -> _ExecutionCallableRunner[P, R]: ... 

55 

56 

57class AsyncExecutionCallableRunner(Protocol): 

58 def __call__( 

59 self, 

60 func: Callable[P, R], 

61 outlet_events: OutletEventAccessorsProtocol, 

62 *, 

63 logger: logging.Logger | Logger, 

64 ) -> _AsyncExecutionCallableRunner[P, R]: ... 

65 

66 

67def create_executable_runner( 

68 func: Callable[P, R], 

69 outlet_events: OutletEventAccessorsProtocol, 

70 *, 

71 logger: logging.Logger | Logger, 

72) -> _ExecutionCallableRunner[P, R]: 

73 """ 

74 Run an execution callable against a task context and given arguments. 

75 

76 If the callable is a simple function, this simply calls it with the supplied 

77 arguments (including the context). If the callable is a generator function, 

78 the generator is exhausted here, with the yielded values getting fed back 

79 into the task context automatically for execution. 

80 

81 This convoluted implementation of inner class with closure is so *all* 

82 arguments passed to ``run()`` can be forwarded to the wrapped function. This 

83 is particularly important for the argument "self", which some use cases 

84 need to receive. This is not possible if this is implemented as a normal 

85 class, where "self" needs to point to the runner object, not the object 

86 bounded to the inner callable. 

87 

88 :meta private: 

89 """ 

90 

91 class _ExecutionCallableRunnerImpl(_ExecutionCallableRunner): 

92 @staticmethod 

93 def run(*args: P.args, **kwargs: P.kwargs) -> R: 

94 from airflow.sdk.definitions.asset.metadata import Metadata 

95 

96 if not inspect.isgeneratorfunction(func): 

97 return func(*args, **kwargs) 

98 

99 result: R 

100 

101 if isinstance(logger, logging.Logger): 

102 

103 def _warn_unknown(metadata): 

104 logger.warning("Ignoring unknown data of %r received from task", type(metadata)) 

105 logger.debug("Full yielded value: %r", metadata) 

106 else: 

107 

108 def _warn_unknown(metadata): 

109 logger.warning("Ignoring unknown type received from task", type=type(metadata)) 

110 logger.debug("Full yielded value", metadata=metadata) 

111 

112 def _run(): 

113 nonlocal result 

114 result = yield from func(*args, **kwargs) 

115 

116 for metadata in _run(): 

117 if isinstance(metadata, Metadata): 

118 outlet_events[metadata.asset].extra.update(metadata.extra) 

119 if metadata.alias: 

120 outlet_events[metadata.alias].add(metadata.asset, extra=metadata.extra) 

121 else: 

122 _warn_unknown(metadata) 

123 

124 return result # noqa: F821 # Ruff is not smart enough to know this is always set in _run(). 

125 

126 return cast("_ExecutionCallableRunner[P, R]", _ExecutionCallableRunnerImpl) 

127 

128 

129def create_async_executable_runner( 

130 func: Callable[P, Awaitable[R] | AsyncIterator], 

131 outlet_events: OutletEventAccessorsProtocol, 

132 *, 

133 logger: logging.Logger | Logger, 

134) -> _AsyncExecutionCallableRunner[P, R]: 

135 """ 

136 Run an async execution callable against a task context and given arguments. 

137 

138 If the callable is a simple function, this simply calls it with the supplied 

139 arguments (including the context). If the callable is a generator function, 

140 the generator is exhausted here, with the yielded values getting fed back 

141 into the task context automatically for execution. 

142 

143 This convoluted implementation of inner class with closure is so *all* 

144 arguments passed to ``run()`` can be forwarded to the wrapped function. This 

145 is particularly important for the argument "self", which some use cases 

146 need to receive. This is not possible if this is implemented as a normal 

147 class, where "self" needs to point to the runner object, not the object 

148 bounded to the inner callable. 

149 

150 :meta private: 

151 """ 

152 

153 class _AsyncExecutionCallableRunnerImpl(_AsyncExecutionCallableRunner): 

154 @staticmethod 

155 async def run(*args: P.args, **kwargs: P.kwargs) -> R: 

156 from airflow.sdk.definitions.asset.metadata import Metadata 

157 

158 if not inspect.isasyncgenfunction(func): 

159 result = cast("Awaitable[R]", func(*args, **kwargs)) 

160 return await result 

161 

162 results: list[Any] = [] 

163 

164 async for result in func(*args, **kwargs): 

165 if isinstance(result, Metadata): 

166 outlet_events[result.asset].extra.update(result.extra) 

167 if result.alias: 

168 outlet_events[result.alias].add(result.asset, extra=result.extra) 

169 

170 results.append(result) 

171 

172 return cast("R", results) 

173 

174 return cast("_AsyncExecutionCallableRunner[P, R]", _AsyncExecutionCallableRunnerImpl)