Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# orm/strategies.py
2# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
3# <see AUTHORS file>
4#
5# This module is part of SQLAlchemy and is released under
6# the MIT License: https://www.opensource.org/licenses/mit-license.php
7# mypy: ignore-errors
10"""sqlalchemy.orm.interfaces.LoaderStrategy
11implementations, and related MapperOptions."""
13from __future__ import annotations
15import collections
16from typing import Any
17from typing import Dict
18from typing import Literal
19from typing import Optional
20from typing import Tuple
21from typing import TYPE_CHECKING
22from typing import Union
24from . import attributes
25from . import exc as orm_exc
26from . import interfaces
27from . import loading
28from . import path_registry
29from . import properties
30from . import query
31from . import relationships
32from . import unitofwork
33from . import util as orm_util
34from .base import _DEFER_FOR_STATE
35from .base import _RAISE_FOR_STATE
36from .base import _SET_DEFERRED_EXPIRED
37from .base import ATTR_WAS_SET
38from .base import LoaderCallableStatus
39from .base import PASSIVE_OFF
40from .base import PassiveFlag
41from .context import _column_descriptions
42from .context import _ORMCompileState
43from .context import _ORMSelectCompileState
44from .context import QueryContext
45from .interfaces import LoaderStrategy
46from .interfaces import StrategizedProperty
47from .session import _state_session
48from .state import InstanceState
49from .strategy_options import Load
50from .util import _none_only_set
51from .util import AliasedClass
52from .. import event
53from .. import exc as sa_exc
54from .. import inspect
55from .. import log
56from .. import sql
57from .. import util
58from ..sql import util as sql_util
59from ..sql import visitors
60from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL
61from ..sql.selectable import Select
63if TYPE_CHECKING:
64 from .mapper import Mapper
65 from .relationships import RelationshipProperty
66 from ..sql.elements import ColumnElement
69def _register_attribute(
70 prop,
71 mapper,
72 useobject,
73 compare_function=None,
74 typecallable=None,
75 callable_=None,
76 proxy_property=None,
77 active_history=False,
78 impl_class=None,
79 default_scalar_value=None,
80 **kw,
81):
83 # event-hook registration functions organized into "pre validate" and "post
84 # validate" collections. Each hook registration function generates a new
85 # AttributeEvents registration for this attribute. What we are controlling
86 # here is the order in which these attribute events are established. Hook
87 # registration functions invoked in the order of first pre-validate, then
88 # user-specific validation i.e. `@validates`, then post-validate.
89 # within `@validates` we are scanning for hooks from child-most classes
90 # first (this suits the feature added in #2943).
91 pre_validate_hooks = []
92 post_validate_hooks = []
94 uselist = useobject and prop.uselist
96 if useobject and prop.single_parent:
97 pre_validate_hooks.append(_single_parent_validator)
99 if useobject:
100 post_validate_hooks.append(unitofwork._track_cascade_events)
102 # need to assemble backref listeners
103 # after the singleparentvalidator, mapper validator
104 if useobject:
105 backref = prop.back_populates
106 if backref and prop._effective_sync_backref:
107 post_validate_hooks.append(
108 lambda desc, prop: attributes._backref_listeners(
109 desc, backref, uselist
110 )
111 )
113 # a single MapperProperty is shared down a class inheritance
114 # hierarchy, so we set up attribute instrumentation and backref event
115 # for each mapper down the hierarchy.
117 # typically, "mapper" is the same as prop.parent, due to the way
118 # the configure_mappers() process runs, however this is not strongly
119 # enforced, and in the case of a second configure_mappers() run the
120 # mapper here might not be prop.parent; also, a subclass mapper may
121 # be called here before a superclass mapper. That is, can't depend
122 # on mappers not already being set up so we have to check each one.
124 for m in mapper.self_and_descendants:
125 if prop is m._props.get(
126 prop.key
127 ) and not m.class_manager._attr_has_impl(prop.key):
128 desc = attributes._register_attribute_impl(
129 m.class_,
130 prop.key,
131 parent_token=prop,
132 uselist=uselist,
133 compare_function=compare_function,
134 useobject=useobject,
135 trackparent=useobject
136 and (
137 prop.single_parent
138 or prop.direction is interfaces.ONETOMANY
139 ),
140 typecallable=typecallable,
141 callable_=callable_,
142 active_history=active_history,
143 default_scalar_value=default_scalar_value,
144 impl_class=impl_class,
145 send_modified_events=not useobject or not prop.viewonly,
146 doc=prop.doc,
147 **kw,
148 )
150 for hook in pre_validate_hooks:
151 hook(desc, prop)
153 for super_m in m.iterate_to_root():
154 if prop.key in super_m.validators:
155 fn, opts = super_m.validators[prop.key]
156 orm_util._validator_events(desc, prop.key, fn, **opts)
157 break
159 for hook in post_validate_hooks:
160 hook(desc, prop)
163@properties.ColumnProperty.strategy_for(instrument=False, deferred=False)
164class _UninstrumentedColumnLoader(LoaderStrategy):
165 """Represent a non-instrumented MapperProperty.
167 The polymorphic_on argument of mapper() often results in this,
168 if the argument is against the with_polymorphic selectable.
170 """
172 __slots__ = ("columns",)
174 def __init__(self, parent, strategy_key):
175 super().__init__(parent, strategy_key)
176 self.columns = self.parent_property.columns
178 def setup_query(
179 self,
180 compile_state,
181 query_entity,
182 path,
183 loadopt,
184 adapter,
185 column_collection=None,
186 **kwargs,
187 ):
188 for c in self.columns:
189 if adapter:
190 c = adapter.columns[c]
191 compile_state._append_dedupe_col_collection(c, column_collection)
193 def create_row_processor(
194 self,
195 context,
196 query_entity,
197 path,
198 loadopt,
199 mapper,
200 result,
201 adapter,
202 populators,
203 ):
204 pass
207@log.class_logger
208@properties.ColumnProperty.strategy_for(instrument=True, deferred=False)
209class _ColumnLoader(LoaderStrategy):
210 """Provide loading behavior for a :class:`.ColumnProperty`."""
212 __slots__ = "columns", "is_composite"
214 def __init__(self, parent, strategy_key):
215 super().__init__(parent, strategy_key)
216 self.columns = self.parent_property.columns
217 self.is_composite = hasattr(self.parent_property, "composite_class")
219 def setup_query(
220 self,
221 compile_state,
222 query_entity,
223 path,
224 loadopt,
225 adapter,
226 column_collection,
227 memoized_populators,
228 check_for_adapt=False,
229 **kwargs,
230 ):
231 for c in self.columns:
232 if adapter:
233 if check_for_adapt:
234 c = adapter.adapt_check_present(c)
235 if c is None:
236 return
237 else:
238 c = adapter.columns[c]
240 compile_state._append_dedupe_col_collection(c, column_collection)
242 fetch = self.columns[0]
243 if adapter:
244 fetch = adapter.columns[fetch]
245 if fetch is None:
246 # None happens here only for dml bulk_persistence cases
247 # when context.DMLReturningColFilter is used
248 return
250 memoized_populators[self.parent_property] = fetch
252 def init_class_attribute(self, mapper):
253 self.is_class_level = True
254 coltype = self.columns[0].type
255 # TODO: check all columns ? check for foreign key as well?
256 active_history = (
257 self.parent_property.active_history
258 or self.columns[0].primary_key
259 or (
260 mapper.version_id_col is not None
261 and mapper._columntoproperty.get(mapper.version_id_col, None)
262 is self.parent_property
263 )
264 )
266 _register_attribute(
267 self.parent_property,
268 mapper,
269 useobject=False,
270 compare_function=coltype.compare_values,
271 active_history=active_history,
272 default_scalar_value=self.parent_property._default_scalar_value,
273 )
275 def create_row_processor(
276 self,
277 context,
278 query_entity,
279 path,
280 loadopt,
281 mapper,
282 result,
283 adapter,
284 populators,
285 ):
286 # look through list of columns represented here
287 # to see which, if any, is present in the row.
289 for col in self.columns:
290 if adapter:
291 col = adapter.columns[col]
292 getter = result._getter(col, False)
293 if getter:
294 populators["quick"].append((self.key, getter))
295 break
296 else:
297 populators["expire"].append((self.key, True))
300@log.class_logger
301@properties.ColumnProperty.strategy_for(query_expression=True)
302class _ExpressionColumnLoader(_ColumnLoader):
303 def __init__(self, parent, strategy_key):
304 super().__init__(parent, strategy_key)
306 # compare to the "default" expression that is mapped in
307 # the column. If it's sql.null, we don't need to render
308 # unless an expr is passed in the options.
309 null = sql.null().label(None)
310 self._have_default_expression = any(
311 not c.compare(null) for c in self.parent_property.columns
312 )
314 def setup_query(
315 self,
316 compile_state,
317 query_entity,
318 path,
319 loadopt,
320 adapter,
321 column_collection,
322 memoized_populators,
323 **kwargs,
324 ):
325 columns = None
326 if loadopt and loadopt._extra_criteria:
327 columns = loadopt._extra_criteria
329 elif self._have_default_expression:
330 columns = self.parent_property.columns
332 if columns is None:
333 return
335 for c in columns:
336 if adapter:
337 c = adapter.columns[c]
338 compile_state._append_dedupe_col_collection(c, column_collection)
340 fetch = columns[0]
341 if adapter:
342 fetch = adapter.columns[fetch]
343 if fetch is None:
344 # None is not expected to be the result of any
345 # adapter implementation here, however there may be theoretical
346 # usages of returning() with context.DMLReturningColFilter
347 return
349 memoized_populators[self.parent_property] = fetch
351 # if the column being loaded is the polymorphic discriminator,
352 # and we have a with_expression() providing the actual column,
353 # update the query_entity to use the actual column instead of
354 # the default expression
355 if (
356 query_entity._polymorphic_discriminator is self.columns[0]
357 and loadopt
358 and loadopt._extra_criteria
359 ):
360 query_entity._polymorphic_discriminator = columns[0]
362 def create_row_processor(
363 self,
364 context,
365 query_entity,
366 path,
367 loadopt,
368 mapper,
369 result,
370 adapter,
371 populators,
372 ):
373 # look through list of columns represented here
374 # to see which, if any, is present in the row.
375 if loadopt and loadopt._extra_criteria:
376 columns = loadopt._extra_criteria
378 for col in columns:
379 if adapter:
380 col = adapter.columns[col]
381 getter = result._getter(col, False)
382 if getter:
383 populators["quick"].append((self.key, getter))
384 break
385 else:
386 populators["expire"].append((self.key, True))
388 def init_class_attribute(self, mapper):
389 self.is_class_level = True
391 _register_attribute(
392 self.parent_property,
393 mapper,
394 useobject=False,
395 compare_function=self.columns[0].type.compare_values,
396 accepts_scalar_loader=False,
397 default_scalar_value=self.parent_property._default_scalar_value,
398 )
401@log.class_logger
402@properties.ColumnProperty.strategy_for(deferred=True, instrument=True)
403@properties.ColumnProperty.strategy_for(
404 deferred=True, instrument=True, raiseload=True
405)
406@properties.ColumnProperty.strategy_for(do_nothing=True)
407class _DeferredColumnLoader(LoaderStrategy):
408 """Provide loading behavior for a deferred :class:`.ColumnProperty`."""
410 __slots__ = "columns", "group", "raiseload"
412 def __init__(self, parent, strategy_key):
413 super().__init__(parent, strategy_key)
414 if hasattr(self.parent_property, "composite_class"):
415 raise NotImplementedError(
416 "Deferred loading for composite types not implemented yet"
417 )
418 self.raiseload = self.strategy_opts.get("raiseload", False)
419 self.columns = self.parent_property.columns
420 self.group = self.parent_property.group
422 def create_row_processor(
423 self,
424 context,
425 query_entity,
426 path,
427 loadopt,
428 mapper,
429 result,
430 adapter,
431 populators,
432 ):
433 # for a DeferredColumnLoader, this method is only used during a
434 # "row processor only" query; see test_deferred.py ->
435 # tests with "rowproc_only" in their name. As of the 1.0 series,
436 # loading._instance_processor doesn't use a "row processing" function
437 # to populate columns, instead it uses data in the "populators"
438 # dictionary. Normally, the DeferredColumnLoader.setup_query()
439 # sets up that data in the "memoized_populators" dictionary
440 # and "create_row_processor()" here is never invoked.
442 if (
443 context.refresh_state
444 and context.query._compile_options._only_load_props
445 and self.key in context.query._compile_options._only_load_props
446 ):
447 self.parent_property._get_strategy(
448 (("deferred", False), ("instrument", True))
449 ).create_row_processor(
450 context,
451 query_entity,
452 path,
453 loadopt,
454 mapper,
455 result,
456 adapter,
457 populators,
458 )
460 elif not self.is_class_level:
461 if self.raiseload:
462 set_deferred_for_local_state = (
463 self.parent_property._raise_column_loader
464 )
465 else:
466 set_deferred_for_local_state = (
467 self.parent_property._deferred_column_loader
468 )
469 populators["new"].append((self.key, set_deferred_for_local_state))
470 else:
471 populators["expire"].append((self.key, False))
473 def init_class_attribute(self, mapper):
474 self.is_class_level = True
476 _register_attribute(
477 self.parent_property,
478 mapper,
479 useobject=False,
480 compare_function=self.columns[0].type.compare_values,
481 callable_=self._load_for_state,
482 load_on_unexpire=False,
483 default_scalar_value=self.parent_property._default_scalar_value,
484 )
486 def setup_query(
487 self,
488 compile_state,
489 query_entity,
490 path,
491 loadopt,
492 adapter,
493 column_collection,
494 memoized_populators,
495 only_load_props=None,
496 **kw,
497 ):
498 if (
499 (
500 compile_state.compile_options._render_for_subquery
501 and self.parent_property._renders_in_subqueries
502 )
503 or (
504 loadopt
505 and set(self.columns).intersection(
506 self.parent._should_undefer_in_wildcard
507 )
508 )
509 or (
510 loadopt
511 and self.group
512 and loadopt.local_opts.get(
513 "undefer_group_%s" % self.group, False
514 )
515 )
516 or (only_load_props and self.key in only_load_props)
517 ):
518 self.parent_property._get_strategy(
519 (("deferred", False), ("instrument", True))
520 ).setup_query(
521 compile_state,
522 query_entity,
523 path,
524 loadopt,
525 adapter,
526 column_collection,
527 memoized_populators,
528 **kw,
529 )
530 elif self.is_class_level:
531 memoized_populators[self.parent_property] = _SET_DEFERRED_EXPIRED
532 elif not self.raiseload:
533 memoized_populators[self.parent_property] = _DEFER_FOR_STATE
534 else:
535 memoized_populators[self.parent_property] = _RAISE_FOR_STATE
537 def _load_for_state(self, state, passive):
538 if not state.key:
539 return LoaderCallableStatus.ATTR_EMPTY
541 if not passive & PassiveFlag.SQL_OK:
542 return LoaderCallableStatus.PASSIVE_NO_RESULT
544 localparent = state.manager.mapper
546 if self.group:
547 toload = [
548 p.key
549 for p in localparent.iterate_properties
550 if isinstance(p, StrategizedProperty)
551 and isinstance(p.strategy, _DeferredColumnLoader)
552 and p.group == self.group
553 ]
554 else:
555 toload = [self.key]
557 # narrow the keys down to just those which have no history
558 group = [k for k in toload if k in state.unmodified]
560 session = _state_session(state)
561 if session is None:
562 raise orm_exc.DetachedInstanceError(
563 "Parent instance %s is not bound to a Session; "
564 "deferred load operation of attribute '%s' cannot proceed"
565 % (orm_util.state_str(state), self.key)
566 )
568 if self.raiseload:
569 self._invoke_raise_load(state, passive, "raise")
571 loading._load_scalar_attributes(
572 state.mapper, state, set(group), PASSIVE_OFF
573 )
575 return LoaderCallableStatus.ATTR_WAS_SET
577 def _invoke_raise_load(self, state, passive, lazy):
578 raise sa_exc.InvalidRequestError(
579 "'%s' is not available due to raiseload=True" % (self,)
580 )
583class _LoadDeferredColumns:
584 """serializable loader object used by DeferredColumnLoader"""
586 def __init__(self, key: str, raiseload: bool = False):
587 self.key = key
588 self.raiseload = raiseload
590 def __call__(self, state, passive=attributes.PASSIVE_OFF):
591 key = self.key
593 localparent = state.manager.mapper
594 prop = localparent._props[key]
595 if self.raiseload:
596 strategy_key = (
597 ("deferred", True),
598 ("instrument", True),
599 ("raiseload", True),
600 )
601 else:
602 strategy_key = (("deferred", True), ("instrument", True))
603 strategy = prop._get_strategy(strategy_key)
604 return strategy._load_for_state(state, passive)
607class _AbstractRelationshipLoader(LoaderStrategy):
608 """LoaderStratgies which deal with related objects."""
610 __slots__ = "mapper", "target", "uselist", "entity"
612 def __init__(self, parent, strategy_key):
613 super().__init__(parent, strategy_key)
614 self.mapper = self.parent_property.mapper
615 self.entity = self.parent_property.entity
616 self.target = self.parent_property.target
617 self.uselist = self.parent_property.uselist
619 def _immediateload_create_row_processor(
620 self,
621 context,
622 query_entity,
623 path,
624 loadopt,
625 mapper,
626 result,
627 adapter,
628 populators,
629 ):
630 return self.parent_property._get_strategy(
631 (("lazy", "immediate"),)
632 ).create_row_processor(
633 context,
634 query_entity,
635 path,
636 loadopt,
637 mapper,
638 result,
639 adapter,
640 populators,
641 )
644@log.class_logger
645@relationships.RelationshipProperty.strategy_for(do_nothing=True)
646class _DoNothingLoader(LoaderStrategy):
647 """Relationship loader that makes no change to the object's state.
649 Compared to NoLoader, this loader does not initialize the
650 collection/attribute to empty/none; the usual default LazyLoader will
651 take effect.
653 """
656@log.class_logger
657@relationships.RelationshipProperty.strategy_for(lazy="noload")
658@relationships.RelationshipProperty.strategy_for(lazy=None)
659class _NoLoader(_AbstractRelationshipLoader):
660 """Provide loading behavior for a :class:`.Relationship`
661 with "lazy=None".
663 """
665 __slots__ = ()
667 @util.deprecated(
668 "2.1",
669 "The ``noload`` loader strategy is deprecated and will be removed "
670 "in a future release. This option "
671 "produces incorrect results by returning ``None`` for related "
672 "items.",
673 )
674 def init_class_attribute(self, mapper):
675 self.is_class_level = True
677 _register_attribute(
678 self.parent_property,
679 mapper,
680 useobject=True,
681 typecallable=self.parent_property.collection_class,
682 )
684 def create_row_processor(
685 self,
686 context,
687 query_entity,
688 path,
689 loadopt,
690 mapper,
691 result,
692 adapter,
693 populators,
694 ):
695 def invoke_no_load(state, dict_, row):
696 if self.uselist:
697 attributes.init_state_collection(state, dict_, self.key)
698 else:
699 dict_[self.key] = None
701 populators["new"].append((self.key, invoke_no_load))
704@log.class_logger
705@relationships.RelationshipProperty.strategy_for(lazy=True)
706@relationships.RelationshipProperty.strategy_for(lazy="select")
707@relationships.RelationshipProperty.strategy_for(lazy="raise")
708@relationships.RelationshipProperty.strategy_for(lazy="raise_on_sql")
709@relationships.RelationshipProperty.strategy_for(lazy="baked_select")
710class _LazyLoader(
711 _AbstractRelationshipLoader, util.MemoizedSlots, log.Identified
712):
713 """Provide loading behavior for a :class:`.Relationship`
714 with "lazy=True", that is loads when first accessed.
716 """
718 __slots__ = (
719 "_lazywhere",
720 "_rev_lazywhere",
721 "_lazyload_reverse_option",
722 "_order_by",
723 "use_get",
724 "is_aliased_class",
725 "_bind_to_col",
726 "_equated_columns",
727 "_rev_bind_to_col",
728 "_rev_equated_columns",
729 "_simple_lazy_clause",
730 "_raise_always",
731 "_raise_on_sql",
732 )
734 _lazywhere: ColumnElement[bool]
735 _bind_to_col: Dict[str, ColumnElement[Any]]
736 _rev_lazywhere: ColumnElement[bool]
737 _rev_bind_to_col: Dict[str, ColumnElement[Any]]
739 parent_property: RelationshipProperty[Any]
741 def __init__(
742 self, parent: RelationshipProperty[Any], strategy_key: Tuple[Any, ...]
743 ):
744 super().__init__(parent, strategy_key)
745 self._raise_always = self.strategy_opts["lazy"] == "raise"
746 self._raise_on_sql = self.strategy_opts["lazy"] == "raise_on_sql"
748 self.is_aliased_class = inspect(self.entity).is_aliased_class
750 join_condition = self.parent_property._join_condition
751 (
752 self._lazywhere,
753 self._bind_to_col,
754 self._equated_columns,
755 ) = join_condition.create_lazy_clause()
757 (
758 self._rev_lazywhere,
759 self._rev_bind_to_col,
760 self._rev_equated_columns,
761 ) = join_condition.create_lazy_clause(reverse_direction=True)
763 if self.parent_property.order_by:
764 self._order_by = util.to_list(self.parent_property.order_by)
765 else:
766 self._order_by = None
768 self.logger.info("%s lazy loading clause %s", self, self._lazywhere)
770 # determine if our "lazywhere" clause is the same as the mapper's
771 # get() clause. then we can just use mapper.get()
772 #
773 # TODO: the "not self.uselist" can be taken out entirely; a m2o
774 # load that populates for a list (very unusual, but is possible with
775 # the API) can still set for "None" and the attribute system will
776 # populate as an empty list.
777 self.use_get = (
778 not self.is_aliased_class
779 and not self.uselist
780 and self.entity._get_clause[0].compare(
781 self._lazywhere,
782 use_proxies=True,
783 compare_keys=False,
784 equivalents=self.mapper._equivalent_columns,
785 )
786 )
788 if self.use_get:
789 for col in list(self._equated_columns):
790 if col in self.mapper._equivalent_columns:
791 for c in self.mapper._equivalent_columns[col]:
792 self._equated_columns[c] = self._equated_columns[col]
794 self.logger.info(
795 "%s will use Session.get() to optimize instance loads", self
796 )
798 def init_class_attribute(self, mapper):
799 self.is_class_level = True
801 _legacy_inactive_history_style = (
802 self.parent_property._legacy_inactive_history_style
803 )
805 if self.parent_property.active_history:
806 active_history = True
807 _deferred_history = False
809 elif (
810 self.parent_property.direction is not interfaces.MANYTOONE
811 or not self.use_get
812 ):
813 if _legacy_inactive_history_style:
814 active_history = True
815 _deferred_history = False
816 else:
817 active_history = False
818 _deferred_history = True
819 else:
820 active_history = _deferred_history = False
822 _register_attribute(
823 self.parent_property,
824 mapper,
825 useobject=True,
826 callable_=self._load_for_state,
827 typecallable=self.parent_property.collection_class,
828 active_history=active_history,
829 _deferred_history=_deferred_history,
830 )
832 def _memoized_attr__simple_lazy_clause(self):
833 lazywhere = self._lazywhere
835 criterion, bind_to_col = (lazywhere, self._bind_to_col)
837 params = []
839 def visit_bindparam(bindparam):
840 bindparam.unique = False
842 visitors.traverse(criterion, {}, {"bindparam": visit_bindparam})
844 def visit_bindparam(bindparam):
845 if bindparam._identifying_key in bind_to_col:
846 params.append(
847 (
848 bindparam.key,
849 bind_to_col[bindparam._identifying_key],
850 None,
851 )
852 )
853 elif bindparam.callable is None:
854 params.append((bindparam.key, None, bindparam.value))
856 criterion = visitors.cloned_traverse(
857 criterion, {}, {"bindparam": visit_bindparam}
858 )
860 return criterion, params
862 def _generate_lazy_clause(self, state, passive):
863 criterion, param_keys = self._simple_lazy_clause
865 if state is None:
866 return sql_util.adapt_criterion_to_null(
867 criterion, [key for key, ident, value in param_keys]
868 )
870 mapper = self.parent_property.parent
872 o = state.obj() # strong ref
873 dict_ = attributes.instance_dict(o)
875 if passive & PassiveFlag.INIT_OK:
876 passive ^= PassiveFlag.INIT_OK
878 params = {}
879 for key, ident, value in param_keys:
880 if ident is not None:
881 if passive and passive & PassiveFlag.LOAD_AGAINST_COMMITTED:
882 value = mapper._get_committed_state_attr_by_column(
883 state, dict_, ident, passive
884 )
885 else:
886 value = mapper._get_state_attr_by_column(
887 state, dict_, ident, passive
888 )
890 params[key] = value
892 return criterion, params
894 def _invoke_raise_load(self, state, passive, lazy):
895 raise sa_exc.InvalidRequestError(
896 "'%s' is not available due to lazy='%s'" % (self, lazy)
897 )
899 def _load_for_state(
900 self,
901 state,
902 passive,
903 loadopt=None,
904 extra_criteria=(),
905 extra_options=(),
906 alternate_effective_path=None,
907 execution_options=util.EMPTY_DICT,
908 ):
909 if not state.key and (
910 (
911 not self.parent_property.load_on_pending
912 and not state._load_pending
913 )
914 or not state.session_id
915 ):
916 return LoaderCallableStatus.ATTR_EMPTY
918 pending = not state.key
919 primary_key_identity = None
921 use_get = self.use_get and (not loadopt or not loadopt._extra_criteria)
923 if (not passive & PassiveFlag.SQL_OK and not use_get) or (
924 not passive & attributes.NON_PERSISTENT_OK and pending
925 ):
926 return LoaderCallableStatus.PASSIVE_NO_RESULT
928 if (
929 # we were given lazy="raise"
930 self._raise_always
931 # the no_raise history-related flag was not passed
932 and not passive & PassiveFlag.NO_RAISE
933 and (
934 # if we are use_get and related_object_ok is disabled,
935 # which means we are at most looking in the identity map
936 # for history purposes or otherwise returning
937 # PASSIVE_NO_RESULT, don't raise. This is also a
938 # history-related flag
939 not use_get
940 or passive & PassiveFlag.RELATED_OBJECT_OK
941 )
942 ):
943 self._invoke_raise_load(state, passive, "raise")
945 session = _state_session(state)
946 if not session:
947 if passive & PassiveFlag.NO_RAISE:
948 return LoaderCallableStatus.PASSIVE_NO_RESULT
950 raise orm_exc.DetachedInstanceError(
951 "Parent instance %s is not bound to a Session; "
952 "lazy load operation of attribute '%s' cannot proceed"
953 % (orm_util.state_str(state), self.key)
954 )
956 # if we have a simple primary key load, check the
957 # identity map without generating a Query at all
958 if use_get:
959 primary_key_identity = self._get_ident_for_use_get(
960 session, state, passive
961 )
962 if LoaderCallableStatus.PASSIVE_NO_RESULT in primary_key_identity:
963 return LoaderCallableStatus.PASSIVE_NO_RESULT
964 elif LoaderCallableStatus.NEVER_SET in primary_key_identity:
965 return LoaderCallableStatus.NEVER_SET
967 # test for None alone in primary_key_identity based on
968 # allow_partial_pks preference. PASSIVE_NO_RESULT and NEVER_SET
969 # have already been tested above
970 if not self.mapper.allow_partial_pks:
971 if _none_only_set.intersection(primary_key_identity):
972 return None
973 else:
974 if _none_only_set.issuperset(primary_key_identity):
975 return None
977 if (
978 self.key in state.dict
979 and not passive & PassiveFlag.DEFERRED_HISTORY_LOAD
980 ):
981 return LoaderCallableStatus.ATTR_WAS_SET
983 # look for this identity in the identity map. Delegate to the
984 # Query class in use, as it may have special rules for how it
985 # does this, including how it decides what the correct
986 # identity_token would be for this identity.
988 instance = session._identity_lookup(
989 self.entity,
990 primary_key_identity,
991 passive=passive,
992 lazy_loaded_from=state,
993 )
995 if instance is not None:
996 if instance is LoaderCallableStatus.PASSIVE_CLASS_MISMATCH:
997 return None
998 else:
999 return instance
1000 elif (
1001 not passive & PassiveFlag.SQL_OK
1002 or not passive & PassiveFlag.RELATED_OBJECT_OK
1003 ):
1004 return LoaderCallableStatus.PASSIVE_NO_RESULT
1006 return self._emit_lazyload(
1007 session,
1008 state,
1009 primary_key_identity,
1010 passive,
1011 loadopt,
1012 extra_criteria,
1013 extra_options,
1014 alternate_effective_path,
1015 execution_options,
1016 )
1018 def _get_ident_for_use_get(self, session, state, passive):
1019 instance_mapper = state.manager.mapper
1021 if passive & PassiveFlag.LOAD_AGAINST_COMMITTED:
1022 get_attr = instance_mapper._get_committed_state_attr_by_column
1023 else:
1024 get_attr = instance_mapper._get_state_attr_by_column
1026 dict_ = state.dict
1028 return [
1029 get_attr(state, dict_, self._equated_columns[pk], passive=passive)
1030 for pk in self.mapper.primary_key
1031 ]
1033 @util.preload_module("sqlalchemy.orm.strategy_options")
1034 def _emit_lazyload(
1035 self,
1036 session,
1037 state,
1038 primary_key_identity,
1039 passive,
1040 loadopt,
1041 extra_criteria,
1042 extra_options,
1043 alternate_effective_path,
1044 execution_options,
1045 ):
1046 strategy_options = util.preloaded.orm_strategy_options
1048 clauseelement = self.entity.__clause_element__()
1049 stmt = Select._create_raw_select(
1050 _raw_columns=[clauseelement],
1051 _propagate_attrs=clauseelement._propagate_attrs,
1052 _compile_options=_ORMCompileState.default_compile_options,
1053 )
1054 load_options = QueryContext.default_load_options
1056 load_options += {
1057 "_invoke_all_eagers": False,
1058 "_lazy_loaded_from": state,
1059 }
1061 if self.parent_property.secondary is not None:
1062 stmt = stmt.select_from(
1063 self.mapper, self.parent_property.secondary
1064 )
1066 pending = not state.key
1068 # don't autoflush on pending
1069 if pending or passive & attributes.NO_AUTOFLUSH:
1070 stmt._execution_options = util.immutabledict({"autoflush": False})
1072 use_get = self.use_get
1074 if state.load_options or (loadopt and loadopt._extra_criteria):
1075 if alternate_effective_path is None:
1076 effective_path = state.load_path[self.parent_property]
1077 else:
1078 effective_path = alternate_effective_path[self.parent_property]
1080 opts = state.load_options
1082 if loadopt and loadopt._extra_criteria:
1083 use_get = False
1084 opts += (
1085 orm_util.LoaderCriteriaOption(self.entity, extra_criteria),
1086 )
1088 stmt._with_options = opts
1089 elif alternate_effective_path is None:
1090 # this path is used if there are not already any options
1091 # in the query, but an event may want to add them
1092 effective_path = state.mapper._path_registry[self.parent_property]
1093 else:
1094 # added by immediateloader
1095 effective_path = alternate_effective_path[self.parent_property]
1097 if extra_options:
1098 stmt._with_options += extra_options
1100 stmt._compile_options += {"_current_path": effective_path}
1102 if use_get:
1103 if self._raise_on_sql and not passive & PassiveFlag.NO_RAISE:
1104 self._invoke_raise_load(state, passive, "raise_on_sql")
1106 return loading._load_on_pk_identity(
1107 session,
1108 stmt,
1109 primary_key_identity,
1110 load_options=load_options,
1111 execution_options=execution_options,
1112 )
1114 if self._order_by:
1115 stmt._order_by_clauses = self._order_by
1117 def _lazyload_reverse(compile_context):
1118 for rev in self.parent_property._reverse_property:
1119 # reverse props that are MANYTOONE are loading *this*
1120 # object from get(), so don't need to eager out to those.
1121 if (
1122 rev.direction is interfaces.MANYTOONE
1123 and rev._use_get
1124 and not isinstance(rev.strategy, _LazyLoader)
1125 ):
1126 strategy_options.Load._construct_for_existing_path(
1127 compile_context.compile_options._current_path[
1128 rev.parent
1129 ]
1130 ).lazyload(rev).process_compile_state(compile_context)
1132 stmt = stmt._add_compile_state_func(
1133 _lazyload_reverse, self.parent_property
1134 )
1136 lazy_clause, params = self._generate_lazy_clause(state, passive)
1138 if execution_options:
1139 execution_options = util.EMPTY_DICT.merge_with(
1140 execution_options, {"_sa_orm_load_options": load_options}
1141 )
1142 else:
1143 execution_options = {
1144 "_sa_orm_load_options": load_options,
1145 }
1147 if (
1148 self.key in state.dict
1149 and not passive & PassiveFlag.DEFERRED_HISTORY_LOAD
1150 ):
1151 return LoaderCallableStatus.ATTR_WAS_SET
1153 if pending:
1154 if util.has_intersection(orm_util._none_set, params.values()):
1155 return None
1157 elif util.has_intersection(orm_util._never_set, params.values()):
1158 return None
1160 if self._raise_on_sql and not passive & PassiveFlag.NO_RAISE:
1161 self._invoke_raise_load(state, passive, "raise_on_sql")
1163 stmt._where_criteria = (lazy_clause,)
1165 result = session.execute(
1166 stmt, params, execution_options=execution_options
1167 )
1169 result = result.unique().scalars().all()
1171 if self.uselist:
1172 return result
1173 else:
1174 l = len(result)
1175 if l:
1176 if l > 1:
1177 util.warn(
1178 "Multiple rows returned with "
1179 "uselist=False for lazily-loaded attribute '%s' "
1180 % self.parent_property
1181 )
1183 return result[0]
1184 else:
1185 return None
1187 def create_row_processor(
1188 self,
1189 context,
1190 query_entity,
1191 path,
1192 loadopt,
1193 mapper,
1194 result,
1195 adapter,
1196 populators,
1197 ):
1198 key = self.key
1200 if (
1201 context.load_options._is_user_refresh
1202 and context.query._compile_options._only_load_props
1203 and self.key in context.query._compile_options._only_load_props
1204 ):
1205 return self._immediateload_create_row_processor(
1206 context,
1207 query_entity,
1208 path,
1209 loadopt,
1210 mapper,
1211 result,
1212 adapter,
1213 populators,
1214 )
1216 if not self.is_class_level or (loadopt and loadopt._extra_criteria):
1217 # we are not the primary manager for this attribute
1218 # on this class - set up a
1219 # per-instance lazyloader, which will override the
1220 # class-level behavior.
1221 # this currently only happens when using a
1222 # "lazyload" option on a "no load"
1223 # attribute - "eager" attributes always have a
1224 # class-level lazyloader installed.
1225 set_lazy_callable = (
1226 InstanceState._instance_level_callable_processor
1227 )(
1228 mapper.class_manager,
1229 _LoadLazyAttribute(
1230 key,
1231 self,
1232 loadopt,
1233 (
1234 loadopt._generate_extra_criteria(context)
1235 if loadopt._extra_criteria
1236 else None
1237 ),
1238 ),
1239 key,
1240 )
1242 populators["new"].append((self.key, set_lazy_callable))
1243 elif context.populate_existing or mapper.always_refresh:
1245 def reset_for_lazy_callable(state, dict_, row):
1246 # we are the primary manager for this attribute on
1247 # this class - reset its
1248 # per-instance attribute state, so that the class-level
1249 # lazy loader is
1250 # executed when next referenced on this instance.
1251 # this is needed in
1252 # populate_existing() types of scenarios to reset
1253 # any existing state.
1254 state._reset(dict_, key)
1256 populators["new"].append((self.key, reset_for_lazy_callable))
1259class _LoadLazyAttribute:
1260 """semi-serializable loader object used by LazyLoader
1262 Historically, this object would be carried along with instances that
1263 needed to run lazyloaders, so it had to be serializable to support
1264 cached instances.
1266 this is no longer a general requirement, and the case where this object
1267 is used is exactly the case where we can't really serialize easily,
1268 which is when extra criteria in the loader option is present.
1270 We can't reliably serialize that as it refers to mapped entities and
1271 AliasedClass objects that are local to the current process, which would
1272 need to be matched up on deserialize e.g. the sqlalchemy.ext.serializer
1273 approach.
1275 """
1277 def __init__(self, key, initiating_strategy, loadopt, extra_criteria):
1278 self.key = key
1279 self.strategy_key = initiating_strategy.strategy_key
1280 self.loadopt = loadopt
1281 self.extra_criteria = extra_criteria
1283 def __getstate__(self):
1284 if self.extra_criteria is not None:
1285 util.warn(
1286 "Can't reliably serialize a lazyload() option that "
1287 "contains additional criteria; please use eager loading "
1288 "for this case"
1289 )
1290 return {
1291 "key": self.key,
1292 "strategy_key": self.strategy_key,
1293 "loadopt": self.loadopt,
1294 "extra_criteria": (),
1295 }
1297 def __call__(self, state, passive=attributes.PASSIVE_OFF):
1298 key = self.key
1299 instance_mapper = state.manager.mapper
1300 prop = instance_mapper._props[key]
1301 strategy = prop._strategies[self.strategy_key]
1303 return strategy._load_for_state(
1304 state,
1305 passive,
1306 loadopt=self.loadopt,
1307 extra_criteria=self.extra_criteria,
1308 )
1311class _PostLoader(_AbstractRelationshipLoader):
1312 """A relationship loader that emits a second SELECT statement."""
1314 __slots__ = ()
1316 def _setup_for_recursion(self, context, path, loadopt, join_depth=None):
1317 effective_path = (
1318 context.compile_state.current_path or orm_util.PathRegistry.root
1319 ) + path
1321 top_level_context = context._get_top_level_context()
1322 execution_options = util.immutabledict(
1323 {"sa_top_level_orm_context": top_level_context}
1324 )
1326 if loadopt:
1327 recursion_depth = loadopt.local_opts.get("recursion_depth", None)
1328 unlimited_recursion = recursion_depth == -1
1329 else:
1330 recursion_depth = None
1331 unlimited_recursion = False
1333 if recursion_depth is not None:
1334 if not self.parent_property._is_self_referential:
1335 raise sa_exc.InvalidRequestError(
1336 f"recursion_depth option on relationship "
1337 f"{self.parent_property} not valid for "
1338 "non-self-referential relationship"
1339 )
1340 recursion_depth = context.execution_options.get(
1341 f"_recursion_depth_{id(self)}", recursion_depth
1342 )
1344 if not unlimited_recursion and recursion_depth < 0:
1345 return (
1346 effective_path,
1347 False,
1348 execution_options,
1349 recursion_depth,
1350 )
1352 if not unlimited_recursion:
1353 execution_options = execution_options.union(
1354 {
1355 f"_recursion_depth_{id(self)}": recursion_depth - 1,
1356 }
1357 )
1359 if loading._PostLoad.path_exists(
1360 context, effective_path, self.parent_property
1361 ):
1362 return effective_path, False, execution_options, recursion_depth
1364 path_w_prop = path[self.parent_property]
1365 effective_path_w_prop = effective_path[self.parent_property]
1367 if not path_w_prop.contains(context.attributes, "loader"):
1368 if join_depth:
1369 if effective_path_w_prop.length / 2 > join_depth:
1370 return (
1371 effective_path,
1372 False,
1373 execution_options,
1374 recursion_depth,
1375 )
1376 elif effective_path_w_prop.contains_mapper(self.mapper):
1377 return (
1378 effective_path,
1379 False,
1380 execution_options,
1381 recursion_depth,
1382 )
1384 return effective_path, True, execution_options, recursion_depth
1387@relationships.RelationshipProperty.strategy_for(lazy="immediate")
1388class _ImmediateLoader(_PostLoader):
1389 __slots__ = ("join_depth",)
1391 def __init__(self, parent, strategy_key):
1392 super().__init__(parent, strategy_key)
1393 self.join_depth = self.parent_property.join_depth
1395 def init_class_attribute(self, mapper):
1396 self.parent_property._get_strategy(
1397 (("lazy", "select"),)
1398 ).init_class_attribute(mapper)
1400 def create_row_processor(
1401 self,
1402 context,
1403 query_entity,
1404 path,
1405 loadopt,
1406 mapper,
1407 result,
1408 adapter,
1409 populators,
1410 ):
1411 if not context.compile_state.compile_options._enable_eagerloads:
1412 return
1414 (
1415 effective_path,
1416 run_loader,
1417 execution_options,
1418 recursion_depth,
1419 ) = self._setup_for_recursion(context, path, loadopt, self.join_depth)
1421 if not run_loader:
1422 # this will not emit SQL and will only emit for a many-to-one
1423 # "use get" load. the "_RELATED" part means it may return
1424 # instance even if its expired, since this is a mutually-recursive
1425 # load operation.
1426 flags = attributes.PASSIVE_NO_FETCH_RELATED | PassiveFlag.NO_RAISE
1427 else:
1428 flags = attributes.PASSIVE_OFF | PassiveFlag.NO_RAISE
1430 loading._PostLoad.callable_for_path(
1431 context,
1432 effective_path,
1433 self.parent,
1434 self.parent_property,
1435 self._load_for_path,
1436 loadopt,
1437 flags,
1438 recursion_depth,
1439 execution_options,
1440 )
1442 def _load_for_path(
1443 self,
1444 context,
1445 path,
1446 states,
1447 load_only,
1448 loadopt,
1449 flags,
1450 recursion_depth,
1451 execution_options,
1452 ):
1453 if recursion_depth:
1454 new_opt = Load(loadopt.path.entity)
1455 new_opt.context = (
1456 loadopt,
1457 loadopt._recurse(),
1458 )
1459 alternate_effective_path = path._truncate_recursive()
1460 extra_options = (new_opt,)
1461 else:
1462 alternate_effective_path = path
1463 extra_options = ()
1465 key = self.key
1466 lazyloader = self.parent_property._get_strategy((("lazy", "select"),))
1467 for state, overwrite in states:
1468 dict_ = state.dict
1470 if overwrite or key not in dict_:
1471 value = lazyloader._load_for_state(
1472 state,
1473 flags,
1474 extra_options=extra_options,
1475 alternate_effective_path=alternate_effective_path,
1476 execution_options=execution_options,
1477 )
1478 if value not in (
1479 ATTR_WAS_SET,
1480 LoaderCallableStatus.PASSIVE_NO_RESULT,
1481 ):
1482 state.get_impl(key).set_committed_value(
1483 state, dict_, value
1484 )
1487@log.class_logger
1488@relationships.RelationshipProperty.strategy_for(lazy="subquery")
1489class _SubqueryLoader(_PostLoader):
1490 __slots__ = ("join_depth",)
1492 def __init__(self, parent, strategy_key):
1493 super().__init__(parent, strategy_key)
1494 self.join_depth = self.parent_property.join_depth
1496 def init_class_attribute(self, mapper):
1497 self.parent_property._get_strategy(
1498 (("lazy", "select"),)
1499 ).init_class_attribute(mapper)
1501 def _get_leftmost(
1502 self,
1503 orig_query_entity_index,
1504 subq_path,
1505 current_compile_state,
1506 is_root,
1507 ):
1508 given_subq_path = subq_path
1509 subq_path = subq_path.path
1510 subq_mapper = orm_util._class_to_mapper(subq_path[0])
1512 # determine attributes of the leftmost mapper
1513 if (
1514 self.parent.isa(subq_mapper)
1515 and self.parent_property is subq_path[1]
1516 ):
1517 leftmost_mapper, leftmost_prop = self.parent, self.parent_property
1518 else:
1519 leftmost_mapper, leftmost_prop = subq_mapper, subq_path[1]
1521 if is_root:
1522 # the subq_path is also coming from cached state, so when we start
1523 # building up this path, it has to also be converted to be in terms
1524 # of the current state. this is for the specific case of the entity
1525 # is an AliasedClass against a subquery that's not otherwise going
1526 # to adapt
1527 new_subq_path = current_compile_state._entities[
1528 orig_query_entity_index
1529 ].entity_zero._path_registry[leftmost_prop]
1530 additional = len(subq_path) - len(new_subq_path)
1531 if additional:
1532 new_subq_path += path_registry.PathRegistry.coerce(
1533 subq_path[-additional:]
1534 )
1535 else:
1536 new_subq_path = given_subq_path
1538 leftmost_cols = leftmost_prop.local_columns
1540 leftmost_attr = [
1541 getattr(
1542 new_subq_path.path[0].entity,
1543 leftmost_mapper._columntoproperty[c].key,
1544 )
1545 for c in leftmost_cols
1546 ]
1548 return leftmost_mapper, leftmost_attr, leftmost_prop, new_subq_path
1550 def _generate_from_original_query(
1551 self,
1552 orig_compile_state,
1553 orig_query,
1554 leftmost_mapper,
1555 leftmost_attr,
1556 leftmost_relationship,
1557 orig_entity,
1558 ):
1559 # reformat the original query
1560 # to look only for significant columns
1561 q = orig_query._clone().correlate(None)
1563 # LEGACY: make a Query back from the select() !!
1564 # This suits at least two legacy cases:
1565 # 1. applications which expect before_compile() to be called
1566 # below when we run .subquery() on this query (Keystone)
1567 # 2. applications which are doing subqueryload with complex
1568 # from_self() queries, as query.subquery() / .statement
1569 # has to do the full compile context for multiply-nested
1570 # from_self() (Neutron) - see test_subqload_from_self
1571 # for demo.
1572 q2 = query.Query.__new__(query.Query)
1573 q2.__dict__.update(q.__dict__)
1574 q = q2
1576 # set the query's "FROM" list explicitly to what the
1577 # FROM list would be in any case, as we will be limiting
1578 # the columns in the SELECT list which may no longer include
1579 # all entities mentioned in things like WHERE, JOIN, etc.
1580 if not q._from_obj:
1581 q._enable_assertions = False
1582 q.select_from.non_generative(
1583 q,
1584 *{
1585 ent["entity"]
1586 for ent in _column_descriptions(
1587 orig_query, compile_state=orig_compile_state
1588 )
1589 if ent["entity"] is not None
1590 },
1591 )
1593 # select from the identity columns of the outer (specifically, these
1594 # are the 'local_cols' of the property). This will remove other
1595 # columns from the query that might suggest the right entity which is
1596 # why we do set select_from above. The attributes we have are
1597 # coerced and adapted using the original query's adapter, which is
1598 # needed only for the case of adapting a subclass column to
1599 # that of a polymorphic selectable, e.g. we have
1600 # Engineer.primary_language and the entity is Person. All other
1601 # adaptations, e.g. from_self, select_entity_from(), will occur
1602 # within the new query when it compiles, as the compile_state we are
1603 # using here is only a partial one. If the subqueryload is from a
1604 # with_polymorphic() or other aliased() object, left_attr will already
1605 # be the correct attributes so no adaptation is needed.
1606 target_cols = orig_compile_state._adapt_col_list(
1607 [
1608 sql.coercions.expect(sql.roles.ColumnsClauseRole, o)
1609 for o in leftmost_attr
1610 ],
1611 orig_compile_state._get_current_adapter(),
1612 )
1613 q._raw_columns = target_cols
1615 distinct_target_key = leftmost_relationship.distinct_target_key
1617 if distinct_target_key is True:
1618 q._distinct = True
1619 elif distinct_target_key is None:
1620 # if target_cols refer to a non-primary key or only
1621 # part of a composite primary key, set the q as distinct
1622 for t in {c.table for c in target_cols}:
1623 if not set(target_cols).issuperset(t.primary_key):
1624 q._distinct = True
1625 break
1627 # don't need ORDER BY if no limit/offset
1628 if not q._has_row_limiting_clause:
1629 q._order_by_clauses = ()
1631 if q._distinct is True and q._order_by_clauses:
1632 # the logic to automatically add the order by columns to the query
1633 # when distinct is True is deprecated in the query
1634 to_add = sql_util.expand_column_list_from_order_by(
1635 target_cols, q._order_by_clauses
1636 )
1637 if to_add:
1638 q._set_entities(target_cols + to_add)
1640 # the original query now becomes a subquery
1641 # which we'll join onto.
1642 # LEGACY: as "q" is a Query, the before_compile() event is invoked
1643 # here.
1644 embed_q = q.set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL).subquery()
1645 left_alias = orm_util.AliasedClass(
1646 leftmost_mapper, embed_q, use_mapper_path=True
1647 )
1648 return left_alias
1650 def _prep_for_joins(self, left_alias, subq_path):
1651 # figure out what's being joined. a.k.a. the fun part
1652 to_join = []
1653 pairs = list(subq_path.pairs())
1655 for i, (mapper, prop) in enumerate(pairs):
1656 if i > 0:
1657 # look at the previous mapper in the chain -
1658 # if it is as or more specific than this prop's
1659 # mapper, use that instead.
1660 # note we have an assumption here that
1661 # the non-first element is always going to be a mapper,
1662 # not an AliasedClass
1664 prev_mapper = pairs[i - 1][1].mapper
1665 to_append = prev_mapper if prev_mapper.isa(mapper) else mapper
1666 else:
1667 to_append = mapper
1669 to_join.append((to_append, prop.key))
1671 # determine the immediate parent class we are joining from,
1672 # which needs to be aliased.
1674 if len(to_join) < 2:
1675 # in the case of a one level eager load, this is the
1676 # leftmost "left_alias".
1677 parent_alias = left_alias
1678 else:
1679 info = inspect(to_join[-1][0])
1680 if info.is_aliased_class:
1681 parent_alias = info.entity
1682 else:
1683 # alias a plain mapper as we may be
1684 # joining multiple times
1685 parent_alias = orm_util.AliasedClass(
1686 info.entity, use_mapper_path=True
1687 )
1689 local_cols = self.parent_property.local_columns
1691 local_attr = [
1692 getattr(parent_alias, self.parent._columntoproperty[c].key)
1693 for c in local_cols
1694 ]
1695 return to_join, local_attr, parent_alias
1697 def _apply_joins(
1698 self, q, to_join, left_alias, parent_alias, effective_entity
1699 ):
1700 ltj = len(to_join)
1701 if ltj == 1:
1702 to_join = [
1703 getattr(left_alias, to_join[0][1]).of_type(effective_entity)
1704 ]
1705 elif ltj == 2:
1706 to_join = [
1707 getattr(left_alias, to_join[0][1]).of_type(parent_alias),
1708 getattr(parent_alias, to_join[-1][1]).of_type(
1709 effective_entity
1710 ),
1711 ]
1712 elif ltj > 2:
1713 middle = [
1714 (
1715 (
1716 orm_util.AliasedClass(item[0])
1717 if not inspect(item[0]).is_aliased_class
1718 else item[0].entity
1719 ),
1720 item[1],
1721 )
1722 for item in to_join[1:-1]
1723 ]
1724 inner = []
1726 while middle:
1727 item = middle.pop(0)
1728 attr = getattr(item[0], item[1])
1729 if middle:
1730 attr = attr.of_type(middle[0][0])
1731 else:
1732 attr = attr.of_type(parent_alias)
1734 inner.append(attr)
1736 to_join = (
1737 [getattr(left_alias, to_join[0][1]).of_type(inner[0].parent)]
1738 + inner
1739 + [
1740 getattr(parent_alias, to_join[-1][1]).of_type(
1741 effective_entity
1742 )
1743 ]
1744 )
1746 for attr in to_join:
1747 q = q.join(attr)
1749 return q
1751 def _setup_options(
1752 self,
1753 context,
1754 q,
1755 subq_path,
1756 rewritten_path,
1757 orig_query,
1758 effective_entity,
1759 loadopt,
1760 ):
1761 # note that because the subqueryload object
1762 # does not reuse the cached query, instead always making
1763 # use of the current invoked query, while we have two queries
1764 # here (orig and context.query), they are both non-cached
1765 # queries and we can transfer the options as is without
1766 # adjusting for new criteria. Some work on #6881 / #6889
1767 # brought this into question.
1768 new_options = orig_query._with_options
1770 if loadopt and loadopt._extra_criteria:
1771 new_options += (
1772 orm_util.LoaderCriteriaOption(
1773 effective_entity,
1774 loadopt._generate_extra_criteria(context),
1775 ),
1776 )
1778 # propagate loader options etc. to the new query.
1779 # these will fire relative to subq_path.
1780 q = q._with_current_path(rewritten_path)
1781 q = q.options(*new_options)
1783 return q
1785 def _setup_outermost_orderby(self, q):
1786 if self.parent_property.order_by:
1788 def _setup_outermost_orderby(compile_context):
1789 compile_context.eager_order_by += tuple(
1790 util.to_list(self.parent_property.order_by)
1791 )
1793 q = q._add_compile_state_func(
1794 _setup_outermost_orderby, self.parent_property
1795 )
1797 return q
1799 class _SubqCollections:
1800 """Given a :class:`_query.Query` used to emit the "subquery load",
1801 provide a load interface that executes the query at the
1802 first moment a value is needed.
1804 """
1806 __slots__ = (
1807 "session",
1808 "execution_options",
1809 "load_options",
1810 "params",
1811 "subq",
1812 "_data",
1813 )
1815 def __init__(self, context, subq):
1816 # avoid creating a cycle by storing context
1817 # even though that's preferable
1818 self.session = context.session
1819 self.execution_options = context.execution_options
1820 self.load_options = context.load_options
1821 self.params = context.params or {}
1822 self.subq = subq
1823 self._data = None
1825 def get(self, key, default):
1826 if self._data is None:
1827 self._load()
1828 return self._data.get(key, default)
1830 def _load(self):
1831 self._data = collections.defaultdict(list)
1833 q = self.subq
1834 assert q.session is None
1836 q = q.with_session(self.session)
1838 if self.load_options._populate_existing:
1839 q = q.populate_existing()
1840 # to work with baked query, the parameters may have been
1841 # updated since this query was created, so take these into account
1843 data = self._data
1844 for row in q.params(self.params):
1845 # group plain tuples rather than Row slices, which would
1846 # incur Row construction and Row.__eq__ per row
1847 tup = row._to_tuple_instance()
1848 data[tup[1:]].append(tup[0])
1850 def loader(self, state, dict_, row):
1851 if self._data is None:
1852 self._load()
1854 def _setup_query_from_rowproc(
1855 self,
1856 context,
1857 query_entity,
1858 path,
1859 entity,
1860 loadopt,
1861 adapter,
1862 ):
1863 compile_state = context.compile_state
1864 if (
1865 not compile_state.compile_options._enable_eagerloads
1866 or compile_state.compile_options._for_refresh_state
1867 ):
1868 return
1870 orig_query_entity_index = compile_state._entities.index(query_entity)
1871 context.loaders_require_buffering = True
1873 path = path[self.parent_property]
1875 # build up a path indicating the path from the leftmost
1876 # entity to the thing we're subquery loading.
1877 with_poly_entity = path.get(
1878 compile_state.attributes, "path_with_polymorphic", None
1879 )
1880 if with_poly_entity is not None:
1881 effective_entity = with_poly_entity
1882 else:
1883 effective_entity = self.entity
1885 subq_path, rewritten_path = context.query._execution_options.get(
1886 ("subquery_paths", None),
1887 (orm_util.PathRegistry.root, orm_util.PathRegistry.root),
1888 )
1889 is_root = subq_path is orm_util.PathRegistry.root
1890 subq_path = subq_path + path
1891 rewritten_path = rewritten_path + path
1893 # use the current query being invoked, not the compile state
1894 # one. this is so that we get the current parameters. however,
1895 # it means we can't use the existing compile state, we have to make
1896 # a new one. other approaches include possibly using the
1897 # compiled query but swapping the params, seems only marginally
1898 # less time spent but more complicated
1899 orig_query = context.query._execution_options.get(
1900 ("orig_query", _SubqueryLoader), context.query
1901 )
1903 # make a new compile_state for the query that's probably cached, but
1904 # we're sort of undoing a bit of that caching :(
1905 compile_state_cls = _ORMCompileState._get_plugin_class_for_plugin(
1906 orig_query, "orm"
1907 )
1909 if orig_query._is_lambda_element:
1910 if context.load_options._lazy_loaded_from is None:
1911 util.warn(
1912 'subqueryloader for "%s" must invoke lambda callable '
1913 "at %r in "
1914 "order to produce a new query, decreasing the efficiency "
1915 "of caching for this statement. Consider using "
1916 "selectinload() for more effective full-lambda caching"
1917 % (self, orig_query)
1918 )
1919 orig_query = orig_query._resolved
1921 # this is the more "quick" version, however it's not clear how
1922 # much of this we need. in particular I can't get a test to
1923 # fail if the "set_base_alias" is missing and not sure why that is.
1924 orig_compile_state = compile_state_cls._create_entities_collection(
1925 orig_query, legacy=False
1926 )
1928 (
1929 leftmost_mapper,
1930 leftmost_attr,
1931 leftmost_relationship,
1932 rewritten_path,
1933 ) = self._get_leftmost(
1934 orig_query_entity_index,
1935 rewritten_path,
1936 orig_compile_state,
1937 is_root,
1938 )
1940 # generate a new Query from the original, then
1941 # produce a subquery from it.
1942 left_alias = self._generate_from_original_query(
1943 orig_compile_state,
1944 orig_query,
1945 leftmost_mapper,
1946 leftmost_attr,
1947 leftmost_relationship,
1948 entity,
1949 )
1951 # generate another Query that will join the
1952 # left alias to the target relationships.
1953 # basically doing a longhand
1954 # "from_self()". (from_self() itself not quite industrial
1955 # strength enough for all contingencies...but very close)
1957 q = query.Query(effective_entity)
1959 q._execution_options = context.query._execution_options.merge_with(
1960 context.execution_options,
1961 {
1962 ("orig_query", _SubqueryLoader): orig_query,
1963 ("subquery_paths", None): (subq_path, rewritten_path),
1964 },
1965 )
1967 q = q._set_enable_single_crit(False)
1968 to_join, local_attr, parent_alias = self._prep_for_joins(
1969 left_alias, subq_path
1970 )
1972 q = q.add_columns(*local_attr)
1973 q = self._apply_joins(
1974 q, to_join, left_alias, parent_alias, effective_entity
1975 )
1977 q = self._setup_options(
1978 context,
1979 q,
1980 subq_path,
1981 rewritten_path,
1982 orig_query,
1983 effective_entity,
1984 loadopt,
1985 )
1986 q = self._setup_outermost_orderby(q)
1988 return q
1990 def create_row_processor(
1991 self,
1992 context,
1993 query_entity,
1994 path,
1995 loadopt,
1996 mapper,
1997 result,
1998 adapter,
1999 populators,
2000 ):
2001 if (
2002 loadopt
2003 and context.compile_state.statement is not None
2004 and context.compile_state.statement.is_dml
2005 ):
2006 util.warn_deprecated(
2007 "The subqueryload loader option is not compatible with DML "
2008 "statements such as INSERT, UPDATE. Only SELECT may be used."
2009 "This warning will become an exception in a future release.",
2010 "2.0",
2011 )
2013 if context.refresh_state:
2014 return self._immediateload_create_row_processor(
2015 context,
2016 query_entity,
2017 path,
2018 loadopt,
2019 mapper,
2020 result,
2021 adapter,
2022 populators,
2023 )
2025 _, run_loader, _, _ = self._setup_for_recursion(
2026 context, path, loadopt, self.join_depth
2027 )
2028 if not run_loader:
2029 return
2031 if not isinstance(context.compile_state, _ORMSelectCompileState):
2032 # issue 7505 - subqueryload() in 1.3 and previous would silently
2033 # degrade for from_statement() without warning. this behavior
2034 # is restored here
2035 return
2037 if not self.parent.class_manager[self.key].impl.supports_population:
2038 raise sa_exc.InvalidRequestError(
2039 "'%s' does not support object "
2040 "population - eager loading cannot be applied." % self
2041 )
2043 # a little dance here as the "path" is still something that only
2044 # semi-tracks the exact series of things we are loading, still not
2045 # telling us about with_polymorphic() and stuff like that when it's at
2046 # the root.. the initial MapperEntity is more accurate for this case.
2047 if len(path) == 1:
2048 if not orm_util._entity_isa(query_entity.entity_zero, self.parent):
2049 return
2050 elif not orm_util._entity_isa(
2051 path[-1], self.parent
2052 ) and not self.parent.isa(path[-1].mapper):
2053 # second check accommodates a polymorphic entity where
2054 # the path has been normalized to the base mapper but
2055 # self.parent is a subclass mapper. Fixes #13209.
2056 return
2058 subq = self._setup_query_from_rowproc(
2059 context,
2060 query_entity,
2061 path,
2062 path[-1],
2063 loadopt,
2064 adapter,
2065 )
2067 if subq is None:
2068 return
2070 assert subq.session is None
2072 path = path[self.parent_property]
2074 local_cols = self.parent_property.local_columns
2076 # cache the loaded collections in the context
2077 # so that inheriting mappers don't re-load when they
2078 # call upon create_row_processor again
2079 collections = path.get(context.attributes, "collections")
2080 if collections is None:
2081 collections = self._SubqCollections(context, subq)
2082 path.set(context.attributes, "collections", collections)
2084 if adapter:
2085 local_cols = [adapter.columns[c] for c in local_cols]
2087 if self.uselist:
2088 self._create_collection_loader(
2089 context, result, collections, local_cols, populators
2090 )
2091 else:
2092 self._create_scalar_loader(
2093 context, result, collections, local_cols, populators
2094 )
2096 def _create_collection_loader(
2097 self, context, result, collections, local_cols, populators
2098 ):
2099 tuple_getter = result._tuple_getter(local_cols)
2101 def load_collection_from_subq(state, dict_, row):
2102 collection = collections.get(tuple_getter(row), ())
2103 state.get_impl(self.key).set_committed_value(
2104 state, dict_, collection
2105 )
2107 def load_collection_from_subq_existing_row(state, dict_, row):
2108 if self.key not in dict_:
2109 load_collection_from_subq(state, dict_, row)
2111 populators["new"].append((self.key, load_collection_from_subq))
2112 populators["existing"].append(
2113 (self.key, load_collection_from_subq_existing_row)
2114 )
2116 if context.invoke_all_eagers:
2117 populators["eager"].append((self.key, collections.loader))
2119 def _create_scalar_loader(
2120 self, context, result, collections, local_cols, populators
2121 ):
2122 tuple_getter = result._tuple_getter(local_cols)
2124 def load_scalar_from_subq(state, dict_, row):
2125 collection = collections.get(tuple_getter(row), (None,))
2126 if len(collection) > 1:
2127 util.warn(
2128 "Multiple rows returned with "
2129 "uselist=False for eagerly-loaded attribute '%s' " % self
2130 )
2132 scalar = collection[0]
2133 state.get_impl(self.key).set_committed_value(state, dict_, scalar)
2135 def load_scalar_from_subq_existing_row(state, dict_, row):
2136 if self.key not in dict_:
2137 load_scalar_from_subq(state, dict_, row)
2139 populators["new"].append((self.key, load_scalar_from_subq))
2140 populators["existing"].append(
2141 (self.key, load_scalar_from_subq_existing_row)
2142 )
2143 if context.invoke_all_eagers:
2144 populators["eager"].append((self.key, collections.loader))
2147@log.class_logger
2148@relationships.RelationshipProperty.strategy_for(lazy="joined")
2149@relationships.RelationshipProperty.strategy_for(lazy=False)
2150class _JoinedLoader(_AbstractRelationshipLoader):
2151 """Provide loading behavior for a :class:`.Relationship`
2152 using joined eager loading.
2154 """
2156 __slots__ = "join_depth"
2158 def __init__(self, parent, strategy_key):
2159 super().__init__(parent, strategy_key)
2160 self.join_depth = self.parent_property.join_depth
2162 def init_class_attribute(self, mapper):
2163 self.parent_property._get_strategy(
2164 (("lazy", "select"),)
2165 ).init_class_attribute(mapper)
2167 def setup_query(
2168 self,
2169 compile_state,
2170 query_entity,
2171 path,
2172 loadopt,
2173 adapter,
2174 column_collection=None,
2175 parentmapper=None,
2176 chained_from_outerjoin=False,
2177 **kwargs,
2178 ):
2179 """Add a left outer join to the statement that's being constructed."""
2181 if not compile_state.compile_options._enable_eagerloads:
2182 return
2183 elif (
2184 loadopt
2185 and compile_state.statement is not None
2186 and compile_state.statement.is_dml
2187 ):
2188 util.warn_deprecated(
2189 "The joinedload loader option is not compatible with DML "
2190 "statements such as INSERT, UPDATE. Only SELECT may be used."
2191 "This warning will become an exception in a future release.",
2192 "2.0",
2193 )
2194 elif self.uselist:
2195 compile_state.multi_row_eager_loaders = True
2197 path = path[self.parent_property]
2199 user_defined_adapter = (
2200 self._init_user_defined_eager_proc(
2201 loadopt, compile_state, compile_state.attributes
2202 )
2203 if loadopt
2204 else False
2205 )
2207 if user_defined_adapter is not False:
2208 # setup an adapter but dont create any JOIN, assume it's already
2209 # in the query
2210 (
2211 clauses,
2212 adapter,
2213 add_to_collection,
2214 ) = self._setup_query_on_user_defined_adapter(
2215 compile_state,
2216 query_entity,
2217 path,
2218 adapter,
2219 user_defined_adapter,
2220 )
2222 # don't do "wrap" for multi-row, we want to wrap
2223 # limited/distinct SELECT,
2224 # because we want to put the JOIN on the outside.
2226 else:
2227 # if not via query option, check for
2228 # a cycle
2229 if not path.contains(compile_state.attributes, "loader"):
2230 if self.join_depth:
2231 if path.length / 2 > self.join_depth:
2232 return
2233 elif path.contains_mapper(self.mapper):
2234 return
2236 # add the JOIN and create an adapter
2237 (
2238 clauses,
2239 adapter,
2240 add_to_collection,
2241 chained_from_outerjoin,
2242 ) = self._generate_row_adapter(
2243 compile_state,
2244 query_entity,
2245 path,
2246 loadopt,
2247 adapter,
2248 column_collection,
2249 parentmapper,
2250 chained_from_outerjoin,
2251 )
2253 # for multi-row, we want to wrap limited/distinct SELECT,
2254 # because we want to put the JOIN on the outside.
2255 compile_state.eager_adding_joins = True
2257 with_poly_entity = path.get(
2258 compile_state.attributes, "path_with_polymorphic", None
2259 )
2260 if with_poly_entity is not None:
2261 with_polymorphic = inspect(
2262 with_poly_entity
2263 ).with_polymorphic_mappers
2264 else:
2265 with_polymorphic = None
2267 path = path[self.entity]
2269 loading._setup_entity_query(
2270 compile_state,
2271 self.mapper,
2272 query_entity,
2273 path,
2274 clauses,
2275 add_to_collection,
2276 with_polymorphic=with_polymorphic,
2277 parentmapper=self.mapper,
2278 chained_from_outerjoin=chained_from_outerjoin,
2279 )
2281 has_nones = util.NONE_SET.intersection(compile_state.secondary_columns)
2283 if has_nones:
2284 if with_poly_entity is not None:
2285 raise sa_exc.InvalidRequestError(
2286 "Detected unaliased columns when generating joined "
2287 "load. Make sure to use aliased=True or flat=True "
2288 "when using joined loading with with_polymorphic()."
2289 )
2290 else:
2291 compile_state.secondary_columns = [
2292 c for c in compile_state.secondary_columns if c is not None
2293 ]
2295 def _init_user_defined_eager_proc(
2296 self, loadopt, compile_state, target_attributes
2297 ):
2298 # check if the opt applies at all
2299 if "eager_from_alias" not in loadopt.local_opts:
2300 # nope
2301 return False
2303 path = loadopt.path.parent
2305 # the option applies. check if the "user_defined_eager_row_processor"
2306 # has been built up.
2307 adapter = path.get(
2308 compile_state.attributes, "user_defined_eager_row_processor", False
2309 )
2310 if adapter is not False:
2311 # just return it
2312 return adapter
2314 # otherwise figure it out.
2315 alias = loadopt.local_opts["eager_from_alias"]
2316 root_mapper, prop = path[-2:]
2318 if alias is not None:
2319 if isinstance(alias, str):
2320 alias = prop.target.alias(alias)
2321 adapter = orm_util.ORMAdapter(
2322 orm_util._TraceAdaptRole.JOINEDLOAD_USER_DEFINED_ALIAS,
2323 prop.mapper,
2324 selectable=alias,
2325 equivalents=prop.mapper._equivalent_columns,
2326 limit_on_entity=False,
2327 )
2328 else:
2329 if path.contains(
2330 compile_state.attributes, "path_with_polymorphic"
2331 ):
2332 with_poly_entity = path.get(
2333 compile_state.attributes, "path_with_polymorphic"
2334 )
2335 adapter = orm_util.ORMAdapter(
2336 orm_util._TraceAdaptRole.JOINEDLOAD_PATH_WITH_POLYMORPHIC,
2337 with_poly_entity,
2338 equivalents=prop.mapper._equivalent_columns,
2339 )
2340 else:
2341 adapter = compile_state._polymorphic_adapters.get(
2342 prop.mapper, None
2343 )
2344 path.set(
2345 target_attributes,
2346 "user_defined_eager_row_processor",
2347 adapter,
2348 )
2350 return adapter
2352 def _setup_query_on_user_defined_adapter(
2353 self, context, entity, path, adapter, user_defined_adapter
2354 ):
2355 # apply some more wrapping to the "user defined adapter"
2356 # if we are setting up the query for SQL render.
2357 adapter = entity._get_entity_clauses(context)
2359 if adapter and user_defined_adapter:
2360 user_defined_adapter = user_defined_adapter.wrap(adapter)
2361 path.set(
2362 context.attributes,
2363 "user_defined_eager_row_processor",
2364 user_defined_adapter,
2365 )
2366 elif adapter:
2367 user_defined_adapter = adapter
2368 path.set(
2369 context.attributes,
2370 "user_defined_eager_row_processor",
2371 user_defined_adapter,
2372 )
2374 add_to_collection = context.primary_columns
2375 return user_defined_adapter, adapter, add_to_collection
2377 def _generate_row_adapter(
2378 self,
2379 compile_state,
2380 entity,
2381 path,
2382 loadopt,
2383 adapter,
2384 column_collection,
2385 parentmapper,
2386 chained_from_outerjoin,
2387 ):
2388 with_poly_entity = path.get(
2389 compile_state.attributes, "path_with_polymorphic", None
2390 )
2391 if with_poly_entity:
2392 to_adapt = with_poly_entity
2393 else:
2394 insp = inspect(self.entity)
2395 if insp.is_aliased_class:
2396 alt_selectable = insp.selectable
2397 else:
2398 alt_selectable = None
2400 to_adapt = orm_util.AliasedClass(
2401 self.mapper,
2402 alias=(
2403 alt_selectable._anonymous_fromclause(flat=True)
2404 if alt_selectable is not None
2405 else None
2406 ),
2407 flat=True,
2408 use_mapper_path=True,
2409 )
2411 to_adapt_insp = inspect(to_adapt)
2413 clauses = to_adapt_insp._memo(
2414 ("joinedloader_ormadapter", self),
2415 orm_util.ORMAdapter,
2416 orm_util._TraceAdaptRole.JOINEDLOAD_MEMOIZED_ADAPTER,
2417 to_adapt_insp,
2418 equivalents=self.mapper._equivalent_columns,
2419 adapt_required=True,
2420 allow_label_resolve=False,
2421 anonymize_labels=True,
2422 )
2424 assert clauses.is_aliased_class
2426 innerjoin = (
2427 loadopt.local_opts.get("innerjoin", self.parent_property.innerjoin)
2428 if loadopt is not None
2429 else self.parent_property.innerjoin
2430 )
2432 if not innerjoin:
2433 # if this is an outer join, all non-nested eager joins from
2434 # this path must also be outer joins
2435 chained_from_outerjoin = True
2437 compile_state.create_eager_joins.append(
2438 (
2439 self._create_eager_join,
2440 entity,
2441 path,
2442 adapter,
2443 parentmapper,
2444 clauses,
2445 innerjoin,
2446 chained_from_outerjoin,
2447 loadopt._extra_criteria if loadopt else (),
2448 )
2449 )
2451 add_to_collection = compile_state.secondary_columns
2452 path.set(compile_state.attributes, "eager_row_processor", clauses)
2454 return clauses, adapter, add_to_collection, chained_from_outerjoin
2456 def _create_eager_join(
2457 self,
2458 compile_state,
2459 query_entity,
2460 path,
2461 adapter,
2462 parentmapper,
2463 clauses,
2464 innerjoin,
2465 chained_from_outerjoin,
2466 extra_criteria,
2467 ):
2468 if parentmapper is None:
2469 localparent = query_entity.mapper
2470 else:
2471 localparent = parentmapper
2473 # whether or not the Query will wrap the selectable in a subquery,
2474 # and then attach eager load joins to that (i.e., in the case of
2475 # LIMIT/OFFSET etc.)
2476 should_nest_selectable = compile_state._should_nest_selectable
2478 query_entity_key = None
2480 if (
2481 query_entity not in compile_state.eager_joins
2482 and not should_nest_selectable
2483 and compile_state.from_clauses
2484 ):
2485 indexes = sql_util.find_left_clause_that_matches_given(
2486 compile_state.from_clauses, query_entity.selectable
2487 )
2489 if len(indexes) > 1:
2490 # for the eager load case, I can't reproduce this right
2491 # now. For query.join() I can.
2492 raise sa_exc.InvalidRequestError(
2493 "Can't identify which query entity in which to joined "
2494 "eager load from. Please use an exact match when "
2495 "specifying the join path."
2496 )
2498 if indexes:
2499 clause = compile_state.from_clauses[indexes[0]]
2500 # join to an existing FROM clause on the query.
2501 # key it to its list index in the eager_joins dict.
2502 # Query._compile_context will adapt as needed and
2503 # append to the FROM clause of the select().
2504 query_entity_key, default_towrap = indexes[0], clause
2506 if query_entity_key is None:
2507 query_entity_key, default_towrap = (
2508 query_entity,
2509 query_entity.selectable,
2510 )
2512 towrap = compile_state.eager_joins.setdefault(
2513 query_entity_key, default_towrap
2514 )
2516 if adapter:
2517 if getattr(adapter, "is_aliased_class", False):
2518 # joining from an adapted entity. The adapted entity
2519 # might be a "with_polymorphic", so resolve that to our
2520 # specific mapper's entity before looking for our attribute
2521 # name on it.
2522 efm = adapter.aliased_insp._entity_for_mapper(
2523 localparent
2524 if localparent.isa(self.parent)
2525 else self.parent
2526 )
2528 # look for our attribute on the adapted entity, else fall back
2529 # to our straight property
2530 onclause = getattr(efm.entity, self.key, self.parent_property)
2531 else:
2532 onclause = getattr(
2533 orm_util.AliasedClass(
2534 self.parent, adapter.selectable, use_mapper_path=True
2535 ),
2536 self.key,
2537 self.parent_property,
2538 )
2540 else:
2541 onclause = self.parent_property
2543 assert clauses.is_aliased_class
2545 attach_on_outside = (
2546 not chained_from_outerjoin
2547 or not innerjoin
2548 or innerjoin == "unnested"
2549 or query_entity.entity_zero.represents_outer_join
2550 )
2552 extra_join_criteria = extra_criteria
2553 additional_entity_criteria = compile_state.global_attributes.get(
2554 ("additional_entity_criteria", self.mapper), ()
2555 )
2556 if additional_entity_criteria:
2557 extra_join_criteria += tuple(
2558 ae._resolve_where_criteria(self.mapper)
2559 for ae in additional_entity_criteria
2560 if ae.propagate_to_loaders
2561 )
2563 if attach_on_outside:
2564 # this is the "classic" eager join case.
2565 eagerjoin = orm_util._ORMJoin(
2566 towrap,
2567 clauses.aliased_insp,
2568 onclause,
2569 isouter=not innerjoin
2570 or query_entity.entity_zero.represents_outer_join
2571 or (chained_from_outerjoin and isinstance(towrap, sql.Join)),
2572 _left_memo=self.parent,
2573 _right_memo=path[self.mapper],
2574 _extra_criteria=extra_join_criteria,
2575 )
2576 else:
2577 # all other cases are innerjoin=='nested' approach
2578 eagerjoin = self._splice_nested_inner_join(
2579 path, path[-2], towrap, clauses, onclause, extra_join_criteria
2580 )
2582 compile_state.eager_joins[query_entity_key] = eagerjoin
2584 # send a hint to the Query as to where it may "splice" this join
2585 eagerjoin.stop_on = query_entity.selectable
2587 if not parentmapper:
2588 # for parentclause that is the non-eager end of the join,
2589 # ensure all the parent cols in the primaryjoin are actually
2590 # in the
2591 # columns clause (i.e. are not deferred), so that aliasing applied
2592 # by the Query propagates those columns outward.
2593 # This has the effect
2594 # of "undefering" those columns.
2595 for col in sql_util._find_columns(
2596 self.parent_property.primaryjoin
2597 ):
2598 if localparent.persist_selectable.c.contains_column(col):
2599 if adapter:
2600 col = adapter.columns[col]
2601 compile_state._append_dedupe_col_collection(
2602 col, compile_state.primary_columns
2603 )
2605 if self.parent_property.order_by:
2606 compile_state.eager_order_by += tuple(
2607 (eagerjoin._target_adapter.copy_and_process)(
2608 util.to_list(self.parent_property.order_by)
2609 )
2610 )
2612 def _splice_nested_inner_join(
2613 self,
2614 path,
2615 entity_we_want_to_splice_onto,
2616 join_obj,
2617 clauses,
2618 onclause,
2619 extra_criteria,
2620 entity_inside_join_structure: Union[
2621 Mapper, None, Literal[False]
2622 ] = False,
2623 detected_existing_path: Optional[path_registry.PathRegistry] = None,
2624 ):
2625 # recursive fn to splice a nested join into an existing one.
2626 # entity_inside_join_structure=False means this is the outermost call,
2627 # and it should return a value. entity_inside_join_structure=<mapper>
2628 # indicates we've descended into a join and are looking at a FROM
2629 # clause representing this mapper; if this is not
2630 # entity_we_want_to_splice_onto then return None to end the recursive
2631 # branch
2633 assert entity_we_want_to_splice_onto is path[-2]
2635 if entity_inside_join_structure is False:
2636 assert isinstance(join_obj, orm_util._ORMJoin)
2638 if isinstance(join_obj, sql.selectable.FromGrouping):
2639 # FromGrouping - continue descending into the structure
2640 return self._splice_nested_inner_join(
2641 path,
2642 entity_we_want_to_splice_onto,
2643 join_obj.element,
2644 clauses,
2645 onclause,
2646 extra_criteria,
2647 entity_inside_join_structure,
2648 )
2649 elif isinstance(join_obj, orm_util._ORMJoin):
2650 # _ORMJoin - continue descending into the structure
2652 join_right_path = join_obj._right_memo
2654 # see if right side of join is viable
2655 target_join = self._splice_nested_inner_join(
2656 path,
2657 entity_we_want_to_splice_onto,
2658 join_obj.right,
2659 clauses,
2660 onclause,
2661 extra_criteria,
2662 entity_inside_join_structure=(
2663 join_right_path[-1].mapper
2664 if join_right_path is not None
2665 else None
2666 ),
2667 )
2669 if target_join is not None:
2670 # for a right splice, attempt to flatten out
2671 # a JOIN b JOIN c JOIN .. to avoid needless
2672 # parenthesis nesting
2673 if not join_obj.isouter and not target_join.isouter:
2674 eagerjoin = join_obj._splice_into_center(target_join)
2675 else:
2676 eagerjoin = orm_util._ORMJoin(
2677 join_obj.left,
2678 target_join,
2679 join_obj.onclause,
2680 isouter=join_obj.isouter,
2681 _left_memo=join_obj._left_memo,
2682 )
2684 eagerjoin._target_adapter = target_join._target_adapter
2685 return eagerjoin
2687 else:
2688 # see if left side of join is viable
2689 target_join = self._splice_nested_inner_join(
2690 path,
2691 entity_we_want_to_splice_onto,
2692 join_obj.left,
2693 clauses,
2694 onclause,
2695 extra_criteria,
2696 entity_inside_join_structure=join_obj._left_memo,
2697 detected_existing_path=join_right_path,
2698 )
2700 if target_join is not None:
2701 eagerjoin = orm_util._ORMJoin(
2702 target_join,
2703 join_obj.right,
2704 join_obj.onclause,
2705 isouter=join_obj.isouter,
2706 _right_memo=join_obj._right_memo,
2707 )
2708 eagerjoin._target_adapter = target_join._target_adapter
2709 return eagerjoin
2711 # neither side viable, return None, or fail if this was the top
2712 # most call
2713 if entity_inside_join_structure is False:
2714 assert (
2715 False
2716 ), "assertion failed attempting to produce joined eager loads"
2717 return None
2719 # reached an endpoint (e.g. a table that's mapped, or an alias of that
2720 # table). determine if we can use this endpoint to splice onto
2722 # is this the entity we want to splice onto in the first place?
2723 if not entity_we_want_to_splice_onto.isa(entity_inside_join_structure):
2724 return None
2726 # path check. if we know the path how this join endpoint got here,
2727 # lets look at our path we are satisfying and see if we're in the
2728 # wrong place. This is specifically for when our entity may
2729 # appear more than once in the path, issue #11449
2730 # updated in issue #11965.
2731 if detected_existing_path and len(detected_existing_path) > 2:
2732 # this assertion is currently based on how this call is made,
2733 # where given a join_obj, the call will have these parameters as
2734 # entity_inside_join_structure=join_obj._left_memo
2735 # and entity_inside_join_structure=join_obj._right_memo.mapper
2736 assert detected_existing_path[-3] is entity_inside_join_structure
2738 # from that, see if the path we are targeting matches the
2739 # "existing" path of this join all the way up to the midpoint
2740 # of this join object (e.g. the relationship).
2741 # if not, then this is not our target
2742 #
2743 # a test condition where this test is false looks like:
2744 #
2745 # desired splice: Node->kind->Kind
2746 # path of desired splice: NodeGroup->nodes->Node->kind
2747 # path we've located: NodeGroup->nodes->Node->common_node->Node
2748 #
2749 # above, because we want to splice kind->Kind onto
2750 # NodeGroup->nodes->Node, this is not our path because it actually
2751 # goes more steps than we want into self-referential
2752 # ->common_node->Node
2753 #
2754 # a test condition where this test is true looks like:
2755 #
2756 # desired splice: B->c2s->C2
2757 # path of desired splice: A->bs->B->c2s
2758 # path we've located: A->bs->B->c1s->C1
2759 #
2760 # above, we want to splice c2s->C2 onto B, and the located path
2761 # shows that the join ends with B->c1s->C1. so we will
2762 # add another join onto that, which would create a "branch" that
2763 # we might represent in a pseudopath as:
2764 #
2765 # B->c1s->C1
2766 # ->c2s->C2
2767 #
2768 # i.e. A JOIN B ON <bs> JOIN C1 ON <c1s>
2769 # JOIN C2 ON <c2s>
2770 #
2772 if detected_existing_path[0:-2] != path.path[0:-1]:
2773 return None
2775 return orm_util._ORMJoin(
2776 join_obj,
2777 clauses.aliased_insp,
2778 onclause,
2779 isouter=False,
2780 _left_memo=entity_inside_join_structure,
2781 _right_memo=path[path[-1].mapper],
2782 _extra_criteria=extra_criteria,
2783 )
2785 def _create_eager_adapter(self, context, result, adapter, path, loadopt):
2786 compile_state = context.compile_state
2788 user_defined_adapter = (
2789 self._init_user_defined_eager_proc(
2790 loadopt, compile_state, context.attributes
2791 )
2792 if loadopt
2793 else False
2794 )
2796 if user_defined_adapter is not False:
2797 decorator = user_defined_adapter
2798 # user defined eagerloads are part of the "primary"
2799 # portion of the load.
2800 # the adapters applied to the Query should be honored.
2801 if compile_state.compound_eager_adapter and decorator:
2802 decorator = decorator.wrap(
2803 compile_state.compound_eager_adapter
2804 )
2805 elif compile_state.compound_eager_adapter:
2806 decorator = compile_state.compound_eager_adapter
2807 else:
2808 decorator = path.get(
2809 compile_state.attributes, "eager_row_processor"
2810 )
2811 if decorator is None:
2812 return False
2814 if self.mapper._result_has_identity_key(result, decorator):
2815 return decorator
2816 else:
2817 # no identity key - don't return a row
2818 # processor, will cause a degrade to lazy
2819 return False
2821 def create_row_processor(
2822 self,
2823 context,
2824 query_entity,
2825 path,
2826 loadopt,
2827 mapper,
2828 result,
2829 adapter,
2830 populators,
2831 ):
2833 if not context.compile_state.compile_options._enable_eagerloads:
2834 return
2836 if not self.parent.class_manager[self.key].impl.supports_population:
2837 raise sa_exc.InvalidRequestError(
2838 "'%s' does not support object "
2839 "population - eager loading cannot be applied." % self
2840 )
2842 if self.uselist:
2843 context.loaders_require_uniquing = True
2845 our_path = path[self.parent_property]
2847 eager_adapter = self._create_eager_adapter(
2848 context, result, adapter, our_path, loadopt
2849 )
2851 if eager_adapter is not False:
2852 key = self.key
2854 _instance = loading._instance_processor(
2855 query_entity,
2856 self.mapper,
2857 context,
2858 result,
2859 our_path[self.entity],
2860 eager_adapter,
2861 )
2863 if not self.uselist:
2864 self._create_scalar_loader(context, key, _instance, populators)
2865 else:
2866 self._create_collection_loader(
2867 context, key, _instance, populators
2868 )
2869 else:
2870 self.parent_property._get_strategy(
2871 (("lazy", "select"),)
2872 ).create_row_processor(
2873 context,
2874 query_entity,
2875 path,
2876 loadopt,
2877 mapper,
2878 result,
2879 adapter,
2880 populators,
2881 )
2883 def _create_collection_loader(self, context, key, _instance, populators):
2884 def load_collection_from_joined_new_row(state, dict_, row):
2885 # note this must unconditionally clear out any existing collection.
2886 # an existing collection would be present only in the case of
2887 # populate_existing().
2888 collection = attributes.init_state_collection(state, dict_, key)
2889 result_list = util.UniqueAppender(
2890 collection, "append_without_event"
2891 )
2892 context.attributes[(state, key)] = result_list
2893 inst = _instance(row)
2894 if inst is not None:
2895 result_list.append(inst)
2897 def load_collection_from_joined_existing_row(state, dict_, row):
2898 if (state, key) in context.attributes:
2899 result_list = context.attributes[(state, key)]
2900 else:
2901 # appender_key can be absent from context.attributes
2902 # with isnew=False when self-referential eager loading
2903 # is used; the same instance may be present in two
2904 # distinct sets of result columns
2905 collection = attributes.init_state_collection(
2906 state, dict_, key
2907 )
2908 result_list = util.UniqueAppender(
2909 collection, "append_without_event"
2910 )
2911 context.attributes[(state, key)] = result_list
2912 inst = _instance(row)
2913 if inst is not None:
2914 result_list.append(inst)
2916 def load_collection_from_joined_exec(state, dict_, row):
2917 _instance(row)
2919 populators["new"].append(
2920 (self.key, load_collection_from_joined_new_row)
2921 )
2922 populators["existing"].append(
2923 (self.key, load_collection_from_joined_existing_row)
2924 )
2925 if context.invoke_all_eagers:
2926 populators["eager"].append(
2927 (self.key, load_collection_from_joined_exec)
2928 )
2930 def _create_scalar_loader(self, context, key, _instance, populators):
2931 def load_scalar_from_joined_new_row(state, dict_, row):
2932 # set a scalar object instance directly on the parent
2933 # object, bypassing InstrumentedAttribute event handlers.
2934 dict_[key] = _instance(row)
2936 def load_scalar_from_joined_existing_row(state, dict_, row):
2937 # call _instance on the row, even though the object has
2938 # been created, so that we further descend into properties
2939 existing = _instance(row)
2941 # conflicting value already loaded, this shouldn't happen
2942 if key in dict_:
2943 if existing is not dict_[key]:
2944 util.warn(
2945 "Multiple rows returned with "
2946 "uselist=False for eagerly-loaded attribute '%s' "
2947 % self
2948 )
2949 else:
2950 # this case is when one row has multiple loads of the
2951 # same entity (e.g. via aliasing), one has an attribute
2952 # that the other doesn't.
2953 dict_[key] = existing
2955 def load_scalar_from_joined_exec(state, dict_, row):
2956 _instance(row)
2958 populators["new"].append((self.key, load_scalar_from_joined_new_row))
2959 populators["existing"].append(
2960 (self.key, load_scalar_from_joined_existing_row)
2961 )
2962 if context.invoke_all_eagers:
2963 populators["eager"].append(
2964 (self.key, load_scalar_from_joined_exec)
2965 )
2968@log.class_logger
2969@relationships.RelationshipProperty.strategy_for(lazy="selectin")
2970class _SelectInLoader(_PostLoader, util.MemoizedSlots):
2971 __slots__ = (
2972 "join_depth",
2973 "omit_join",
2974 "_parent_alias",
2975 "_query_info",
2976 "_fallback_query_info",
2977 )
2979 query_info = collections.namedtuple(
2980 "queryinfo",
2981 [
2982 "load_only_child",
2983 "load_with_join",
2984 "in_expr",
2985 "pk_cols",
2986 "zero_idx",
2987 "n_pk",
2988 "child_lookup_cols",
2989 ],
2990 )
2992 _chunksize = 500
2994 @classmethod
2995 def _set_chunksize(cls, loadopt) -> int:
2996 if loadopt is None or hasattr(loadopt, "local_opts") is None:
2997 return cls._chunksize
2999 user_input = loadopt.local_opts.get("chunksize", None)
3000 if user_input is None:
3001 return cls._chunksize
3002 elif not isinstance(user_input, int) or user_input < 1:
3003 raise sa_exc.ArgumentError(
3004 f"'chunksize={user_input}' is not an appropriate input, "
3005 f"please use a positive non-zero integer."
3006 )
3007 return user_input
3009 def __init__(self, parent, strategy_key):
3010 super().__init__(parent, strategy_key)
3011 self.join_depth = self.parent_property.join_depth
3012 is_m2o = self.parent_property.direction is interfaces.MANYTOONE
3013 is_m2m = self.parent_property.direction is interfaces.MANYTOMANY
3015 if self.parent_property.omit_join is not None:
3016 self.omit_join = self.parent_property.omit_join
3017 else:
3018 lazyloader = self.parent_property._get_strategy(
3019 (("lazy", "select"),)
3020 )
3021 if is_m2o:
3022 self.omit_join = lazyloader.use_get
3023 elif is_m2m and not self.parent_property._is_self_referential:
3024 join_cond = self.parent_property._join_condition
3025 self.omit_join = join_cond.secondary_covers_parent_primary_key
3026 else:
3027 self.omit_join = self.parent._get_clause[0].compare(
3028 lazyloader._rev_lazywhere,
3029 use_proxies=True,
3030 compare_keys=False,
3031 equivalents=self.parent._equivalent_columns,
3032 )
3034 if self.omit_join:
3035 if is_m2o:
3036 self._query_info = self._init_for_omit_join_m2o()
3037 self._fallback_query_info = self._init_for_join()
3038 else:
3039 self._query_info = self._init_for_omit_join()
3040 else:
3041 self._query_info = self._init_for_join()
3043 def _init_for_omit_join(self):
3044 pk_to_fk = dict(
3045 self.parent_property._join_condition.local_remote_pairs
3046 )
3047 pk_to_fk.update(
3048 (equiv, pk_to_fk[k])
3049 for k in list(pk_to_fk)
3050 for equiv in self.parent._equivalent_columns.get(k, ())
3051 )
3053 pk_cols = fk_cols = [
3054 pk_to_fk[col] for col in self.parent.primary_key if col in pk_to_fk
3055 ]
3056 if len(fk_cols) > 1:
3057 in_expr = sql.tuple_(*fk_cols)
3058 zero_idx = False
3059 else:
3060 in_expr = fk_cols[0]
3061 zero_idx = True
3063 return self.query_info(
3064 False, False, in_expr, pk_cols, zero_idx, len(pk_cols), None
3065 )
3067 def _init_for_omit_join_m2o(self):
3068 pk_cols = self.mapper.primary_key
3069 if len(pk_cols) > 1:
3070 in_expr = sql.tuple_(*pk_cols)
3071 zero_idx = False
3072 else:
3073 in_expr = pk_cols[0]
3074 zero_idx = True
3076 lazyloader = self.parent_property._get_strategy((("lazy", "select"),))
3077 lookup_cols = [lazyloader._equated_columns[pk] for pk in pk_cols]
3079 return self.query_info(
3080 True, False, in_expr, pk_cols, zero_idx, len(pk_cols), lookup_cols
3081 )
3083 def _init_for_join(self):
3084 self._parent_alias = AliasedClass(self.parent.class_)
3085 pa_insp = inspect(self._parent_alias)
3086 pk_cols = [
3087 pa_insp._adapt_element(col) for col in self.parent.primary_key
3088 ]
3089 if len(pk_cols) > 1:
3090 in_expr = sql.tuple_(*pk_cols)
3091 zero_idx = False
3092 else:
3093 in_expr = pk_cols[0]
3094 zero_idx = True
3095 return self.query_info(
3096 False, True, in_expr, pk_cols, zero_idx, len(pk_cols), None
3097 )
3099 def init_class_attribute(self, mapper):
3100 self.parent_property._get_strategy(
3101 (("lazy", "select"),)
3102 ).init_class_attribute(mapper)
3104 def create_row_processor(
3105 self,
3106 context,
3107 query_entity,
3108 path,
3109 loadopt,
3110 mapper,
3111 result,
3112 adapter,
3113 populators,
3114 ):
3115 if context.refresh_state:
3116 return self._immediateload_create_row_processor(
3117 context,
3118 query_entity,
3119 path,
3120 loadopt,
3121 mapper,
3122 result,
3123 adapter,
3124 populators,
3125 )
3127 (
3128 effective_path,
3129 run_loader,
3130 execution_options,
3131 recursion_depth,
3132 ) = self._setup_for_recursion(
3133 context, path, loadopt, join_depth=self.join_depth
3134 )
3136 if not run_loader:
3137 return
3139 if not context.compile_state.compile_options._enable_eagerloads:
3140 return
3142 if not self.parent.class_manager[self.key].impl.supports_population:
3143 raise sa_exc.InvalidRequestError(
3144 "'%s' does not support object "
3145 "population - eager loading cannot be applied." % self
3146 )
3148 # a little dance here as the "path" is still something that only
3149 # semi-tracks the exact series of things we are loading, still not
3150 # telling us about with_polymorphic() and stuff like that when it's at
3151 # the root.. the initial MapperEntity is more accurate for this case.
3152 if len(path) == 1:
3153 if not orm_util._entity_isa(query_entity.entity_zero, self.parent):
3154 return
3155 elif not orm_util._entity_isa(
3156 path[-1], self.parent
3157 ) and not self.parent.isa(path[-1].mapper):
3158 # second check accommodates a polymorphic entity where
3159 # the path has been normalized to the base mapper but
3160 # self.parent is a subclass mapper, e.g.
3161 # joinedload(A.b.of_type(poly)).selectinload(poly.Sub.rel)
3162 # Fixes #13209.
3163 return
3165 selectin_path = effective_path
3167 path_w_prop = path[self.parent_property]
3169 # build up a path indicating the path from the leftmost
3170 # entity to the thing we're subquery loading.
3171 with_poly_entity = path_w_prop.get(
3172 context.attributes, "path_with_polymorphic", None
3173 )
3174 if with_poly_entity is not None:
3175 effective_entity = inspect(with_poly_entity)
3176 else:
3177 effective_entity = self.entity
3179 loading._PostLoad.callable_for_path(
3180 context,
3181 selectin_path,
3182 self.parent,
3183 self.parent_property,
3184 self._load_for_path,
3185 effective_entity,
3186 loadopt,
3187 recursion_depth,
3188 execution_options,
3189 )
3191 def _load_for_path(
3192 self,
3193 context,
3194 path,
3195 states,
3196 load_only,
3197 effective_entity,
3198 loadopt,
3199 recursion_depth,
3200 execution_options,
3201 ):
3202 if load_only and self.key not in load_only:
3203 return
3205 query_info = self._query_info
3207 if query_info.load_only_child:
3208 our_states = collections.defaultdict(list)
3209 none_states = []
3211 mapper = self.parent
3213 # attribute keys for the lookup columns; when these are
3214 # present in a state's dict, reading them directly is
3215 # equivalent to the PASSIVE_NO_FETCH attribute lookup below.
3216 # whether or not a key is present can vary per state, e.g.
3217 # individual instances may have the attribute expired or
3218 # deferred, so this is determined state-by-state
3219 get_related_ident = mapper._state_ident_getter(
3220 query_info.child_lookup_cols,
3221 passive=attributes.PASSIVE_NO_FETCH,
3222 )
3224 for state, overwrite in states:
3225 state_dict = state.dict
3226 related_ident = get_related_ident(state, state_dict)
3227 # if the loaded parent objects do not have the foreign key
3228 # to the related item loaded, then degrade into the joined
3229 # version of selectinload
3230 if LoaderCallableStatus.PASSIVE_NO_RESULT in related_ident:
3231 query_info = self._fallback_query_info
3232 break
3234 # organize states into lists keyed to particular foreign
3235 # key values.
3236 if None not in related_ident:
3237 our_states[related_ident].append(
3238 (state, state_dict, overwrite)
3239 )
3240 else:
3241 # For FK values that have None, add them to a
3242 # separate collection that will be populated separately
3243 none_states.append((state, state_dict, overwrite))
3245 # note the above conditional may have changed query_info
3246 if not query_info.load_only_child:
3247 our_states = [
3248 (state.key[1], state, state.dict, overwrite)
3249 for state, overwrite in states
3250 ]
3252 pk_cols = query_info.pk_cols
3253 in_expr = query_info.in_expr
3255 if not query_info.load_with_join:
3256 # in "omit join" mode, the primary key column and the
3257 # "in" expression are in terms of the related entity. So
3258 # if the related entity is polymorphic or otherwise aliased,
3259 # we need to adapt our "pk_cols" and "in_expr" to that
3260 # entity. in non-"omit join" mode, these are against the
3261 # parent entity and do not need adaption.
3262 if effective_entity.is_aliased_class:
3263 pk_cols = [
3264 effective_entity._adapt_element(col) for col in pk_cols
3265 ]
3266 in_expr = effective_entity._adapt_element(in_expr)
3268 entity_sql = effective_entity.__clause_element__()
3269 q = Select._create_raw_select(
3270 _raw_columns=[*pk_cols, entity_sql],
3271 _compile_options=_ORMCompileState.default_compile_options,
3272 _propagate_attrs={
3273 "compile_state_plugin": "orm",
3274 "plugin_subject": effective_entity,
3275 },
3276 )
3278 if (
3279 self.parent_property.secondary is not None
3280 and self.omit_join is True
3281 ):
3282 # The secondaryjoin condition is used to connect the
3283 # secondary table to the related entity,
3284 # and is required for composite foreign keys where SQLAlchemy
3285 # cannot determine the join condition.
3286 q = q.select_from(self.parent_property.secondary).join(
3287 entity_sql, self.parent_property._join_condition.secondaryjoin
3288 )
3289 elif not query_info.load_with_join:
3290 # the pk columns in the "omit_join" case are raw, non-annotated
3291 # columns, so to ensure the Query knows its primary entity, we
3292 # add it explicitly. Using annotated columns here would hit a
3293 # performance issue detailed in issue #4347.
3294 q = q.select_from(effective_entity)
3295 else:
3296 # in the non-omit_join case, the pk columns are against the
3297 # annotated/mapped column of the parent entity, but the #4347
3298 # issue does not occur in this case.
3299 q = q.select_from(self._parent_alias).join(
3300 getattr(self._parent_alias, self.parent_property.key).of_type(
3301 effective_entity
3302 )
3303 )
3305 q = q.filter(in_expr.in_(sql.bindparam("primary_keys")))
3307 # a test which exercises what these comments talk about is
3308 # test_selectin_relations.py -> test_twolevel_selectin_w_polymorphic
3309 #
3310 # effective_entity above is given to us in terms of the cached
3311 # statement, namely this one:
3312 orig_query = context.compile_state.select_statement
3314 # the actual statement that was requested is this one:
3315 # context_query = context.user_passed_query
3316 #
3317 # that's not the cached one, however. So while it is of the identical
3318 # structure, if it has entities like AliasedInsp, which we get from
3319 # aliased() or with_polymorphic(), the AliasedInsp will likely be a
3320 # different object identity each time, and will not match up
3321 # hashing-wise to the corresponding AliasedInsp that's in the
3322 # cached query, meaning it won't match on paths and loader lookups
3323 # and loaders like this one will be skipped if it is used in options.
3324 #
3325 # as it turns out, standard loader options like selectinload(),
3326 # lazyload() that have a path need
3327 # to come from the cached query so that the AliasedInsp etc. objects
3328 # that are in the query line up with the object that's in the path
3329 # of the strategy object. however other options like
3330 # with_loader_criteria() that doesn't have a path (has a fixed entity)
3331 # and needs to have access to the latest closure state in order to
3332 # be correct, we need to use the uncached one.
3333 #
3334 # as of #8399 we let the loader option itself figure out what it
3335 # wants to do given cached and uncached version of itself.
3337 effective_path = path[self.parent_property]
3339 if orig_query is context.user_passed_query:
3340 new_options = orig_query._with_options
3341 else:
3342 cached_options = orig_query._with_options
3343 uncached_options = context.user_passed_query._with_options
3345 # propagate compile state options from the original query,
3346 # updating their "extra_criteria" as necessary.
3347 # note this will create a different cache key than
3348 # "orig" options if extra_criteria is present, because the copy
3349 # of extra_criteria will have different boundparam than that of
3350 # the QueryableAttribute in the path
3351 new_options = [
3352 orig_opt._adapt_cached_option_to_uncached_option(
3353 context, uncached_opt
3354 )
3355 for orig_opt, uncached_opt in zip(
3356 cached_options, uncached_options
3357 )
3358 ]
3360 if loadopt and loadopt._extra_criteria:
3361 new_options += (
3362 orm_util.LoaderCriteriaOption(
3363 effective_entity,
3364 loadopt._generate_extra_criteria(context),
3365 ),
3366 )
3368 if recursion_depth is not None:
3369 effective_path = effective_path._truncate_recursive()
3371 q = q.options(*new_options)
3373 q = q._update_compile_options({"_current_path": effective_path})
3374 if context.populate_existing:
3375 q = q.execution_options(populate_existing=True)
3377 if self.parent_property.order_by:
3378 if not query_info.load_with_join:
3379 eager_order_by = self.parent_property.order_by
3380 if effective_entity.is_aliased_class:
3381 eager_order_by = [
3382 effective_entity._adapt_element(elem)
3383 for elem in eager_order_by
3384 ]
3385 q = q.order_by(*eager_order_by)
3386 else:
3388 def _setup_outermost_orderby(compile_context):
3389 compile_context.eager_order_by += tuple(
3390 util.to_list(self.parent_property.order_by)
3391 )
3393 q = q._add_compile_state_func(
3394 _setup_outermost_orderby, self.parent_property
3395 )
3397 chunksize = self._set_chunksize(loadopt)
3399 if query_info.load_only_child:
3400 self._load_via_child(
3401 our_states,
3402 none_states,
3403 query_info,
3404 q,
3405 context,
3406 execution_options,
3407 chunksize,
3408 )
3409 else:
3410 self._load_via_parent(
3411 our_states,
3412 query_info,
3413 q,
3414 context,
3415 execution_options,
3416 chunksize,
3417 )
3419 def _load_via_child(
3420 self,
3421 our_states,
3422 none_states,
3423 query_info,
3424 q,
3425 context,
3426 execution_options,
3427 chunksize,
3428 ):
3429 uselist = self.uselist
3430 n_pk = query_info.n_pk
3431 zero_idx = query_info.zero_idx
3433 # historically this used sorted instead of list to add determinism in
3434 # tests. Since dicts are now ordered it's likely no longer needed
3435 our_keys = list(our_states)
3436 while our_keys:
3437 chunk = our_keys[0:chunksize]
3438 our_keys = our_keys[chunksize:]
3439 primary_keys = [key[0] if zero_idx else key for key in chunk]
3440 result = context.session.execute(
3441 q,
3442 params={"primary_keys": primary_keys},
3443 execution_options=execution_options,
3444 )
3445 if result.context is not None and result.context.requires_uniquing:
3446 rows = result.unique()
3447 else:
3448 rows = result._raw_all_tuples()
3449 data = {row[:n_pk]: row[n_pk] for row in rows}
3451 for key in chunk:
3452 # for a real foreign key and no concurrent changes to the
3453 # DB while running this method, "key" is always present in
3454 # data. However, for primaryjoins without real foreign keys
3455 # a non-None primaryjoin condition may still refer to no
3456 # related object.
3457 related_obj = data.get(key, None)
3458 for state, dict_, overwrite in our_states[key]:
3459 if not overwrite and self.key in dict_:
3460 continue
3462 state.get_impl(self.key).set_committed_value(
3463 state,
3464 dict_,
3465 related_obj if not uselist else [related_obj],
3466 )
3467 # populate none states with empty value / collection
3468 for state, dict_, overwrite in none_states:
3469 if not overwrite and self.key in dict_:
3470 continue
3472 # note it's OK if this is a uselist=True attribute, the empty
3473 # collection will be populated
3474 state.get_impl(self.key).set_committed_value(state, dict_, None)
3476 def _load_via_parent(
3477 self, our_states, query_info, q, context, execution_options, chunksize
3478 ):
3479 uselist = self.uselist
3480 n_pk = query_info.n_pk
3481 zero_idx = query_info.zero_idx
3482 _empty_result = () if uselist else None
3484 while our_states:
3485 chunk = our_states[0:chunksize]
3486 our_states = our_states[chunksize:]
3488 primary_keys = [
3489 item[0][0] if zero_idx else item[0] for item in chunk
3490 ]
3492 result = context.session.execute(
3493 q,
3494 params={"primary_keys": primary_keys},
3495 execution_options=execution_options,
3496 )
3497 if result.context is not None and result.context.requires_uniquing:
3498 rows = result.unique()
3499 else:
3500 rows = result._raw_all_tuples()
3501 data = collections.defaultdict(list)
3502 for row in rows:
3503 data[row[:n_pk]].append(row[n_pk])
3505 for key, state, state_dict, overwrite in chunk:
3506 if not overwrite and self.key in state_dict:
3507 continue
3509 collection = data.get(key, _empty_result)
3511 if not uselist and collection:
3512 if len(collection) > 1:
3513 util.warn(
3514 "Multiple rows returned with "
3515 "uselist=False for eagerly-loaded "
3516 "attribute '%s' " % self
3517 )
3518 state.get_impl(self.key).set_committed_value(
3519 state, state_dict, collection[0]
3520 )
3521 else:
3522 # note that empty tuple set on uselist=False sets the
3523 # value to None
3524 state.get_impl(self.key).set_committed_value(
3525 state, state_dict, collection
3526 )
3529def _single_parent_validator(desc, prop):
3530 def _do_check(state, value, oldvalue, initiator):
3531 if value is not None and initiator.key == prop.key:
3532 hasparent = initiator.hasparent(attributes.instance_state(value))
3533 if hasparent and oldvalue is not value:
3534 raise sa_exc.InvalidRequestError(
3535 "Instance %s is already associated with an instance "
3536 "of %s via its %s attribute, and is only allowed a "
3537 "single parent."
3538 % (orm_util.instance_str(value), state.class_, prop),
3539 code="bbf1",
3540 )
3541 return value
3543 def append(state, value, initiator):
3544 return _do_check(state, value, None, initiator)
3546 def set_(state, value, oldvalue, initiator):
3547 return _do_check(state, value, oldvalue, initiator)
3549 event.listen(
3550 desc, "append", append, raw=True, retval=True, active_history=True
3551 )
3552 event.listen(desc, "set", set_, raw=True, retval=True, active_history=True)