Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# orm/strategies.py
2# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
3# <see AUTHORS file>
4#
5# This module is part of SQLAlchemy and is released under
6# the MIT License: https://www.opensource.org/licenses/mit-license.php
7# mypy: ignore-errors
10"""sqlalchemy.orm.interfaces.LoaderStrategy
11implementations, and related MapperOptions."""
13from __future__ import annotations
15import collections
16from typing import Any
17from typing import Dict
18from typing import Literal
19from typing import Optional
20from typing import Tuple
21from typing import TYPE_CHECKING
22from typing import Union
24from . import attributes
25from . import exc as orm_exc
26from . import interfaces
27from . import loading
28from . import path_registry
29from . import properties
30from . import query
31from . import relationships
32from . import unitofwork
33from . import util as orm_util
34from .base import _DEFER_FOR_STATE
35from .base import _RAISE_FOR_STATE
36from .base import _SET_DEFERRED_EXPIRED
37from .base import ATTR_WAS_SET
38from .base import LoaderCallableStatus
39from .base import PASSIVE_OFF
40from .base import PassiveFlag
41from .context import _column_descriptions
42from .context import _ORMCompileState
43from .context import _ORMSelectCompileState
44from .context import QueryContext
45from .interfaces import LoaderStrategy
46from .interfaces import StrategizedProperty
47from .session import _state_session
48from .state import InstanceState
49from .strategy_options import Load
50from .util import _none_only_set
51from .util import AliasedClass
52from .. import event
53from .. import exc as sa_exc
54from .. import inspect
55from .. import log
56from .. import sql
57from .. import util
58from ..sql import util as sql_util
59from ..sql import visitors
60from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL
61from ..sql.selectable import Select
63if TYPE_CHECKING:
64 from .mapper import Mapper
65 from .relationships import RelationshipProperty
66 from ..sql.elements import ColumnElement
69# sentinel for _LoadLazyAttribute.extra_criteria indicating that the loader
70# option had no additional criteria. the attribute is otherwise a single
71# element returned by Load._generate_extra_criteria(), which has no
72# meaningful boolean value of its own.
73_NO_EXTRA_CRITERIA = None
76def _register_attribute(
77 prop,
78 mapper,
79 useobject,
80 compare_function=None,
81 typecallable=None,
82 callable_=None,
83 proxy_property=None,
84 active_history=False,
85 impl_class=None,
86 default_scalar_value=None,
87 **kw,
88):
90 # event-hook registration functions organized into "pre validate" and "post
91 # validate" collections. Each hook registration function generates a new
92 # AttributeEvents registration for this attribute. What we are controlling
93 # here is the order in which these attribute events are established. Hook
94 # registration functions invoked in the order of first pre-validate, then
95 # user-specific validation i.e. `@validates`, then post-validate.
96 # within `@validates` we are scanning for hooks from child-most classes
97 # first (this suits the feature added in #2943).
98 pre_validate_hooks = []
99 post_validate_hooks = []
101 uselist = useobject and prop.uselist
103 if useobject and prop.single_parent:
104 pre_validate_hooks.append(_single_parent_validator)
106 if useobject:
107 post_validate_hooks.append(unitofwork._track_cascade_events)
109 # need to assemble backref listeners
110 # after the singleparentvalidator, mapper validator
111 if useobject:
112 backref = prop.back_populates
113 if backref and prop._effective_sync_backref:
114 post_validate_hooks.append(
115 lambda desc, prop: attributes._backref_listeners(
116 desc, backref, uselist
117 )
118 )
120 # a single MapperProperty is shared down a class inheritance
121 # hierarchy, so we set up attribute instrumentation and backref event
122 # for each mapper down the hierarchy.
124 # typically, "mapper" is the same as prop.parent, due to the way
125 # the configure_mappers() process runs, however this is not strongly
126 # enforced, and in the case of a second configure_mappers() run the
127 # mapper here might not be prop.parent; also, a subclass mapper may
128 # be called here before a superclass mapper. That is, can't depend
129 # on mappers not already being set up so we have to check each one.
131 for m in mapper.self_and_descendants:
132 if prop is m._props.get(
133 prop.key
134 ) and not m.class_manager._attr_has_impl(prop.key):
135 desc = attributes._register_attribute_impl(
136 m.class_,
137 prop.key,
138 parent_token=prop,
139 uselist=uselist,
140 compare_function=compare_function,
141 useobject=useobject,
142 trackparent=useobject
143 and (
144 prop.single_parent
145 or prop.direction is interfaces.ONETOMANY
146 ),
147 typecallable=typecallable,
148 callable_=callable_,
149 active_history=active_history,
150 default_scalar_value=default_scalar_value,
151 impl_class=impl_class,
152 send_modified_events=not useobject or not prop.viewonly,
153 doc=prop.doc,
154 **kw,
155 )
157 for hook in pre_validate_hooks:
158 hook(desc, prop)
160 for super_m in m.iterate_to_root():
161 if prop.key in super_m.validators:
162 fn, opts = super_m.validators[prop.key]
163 orm_util._validator_events(desc, prop.key, fn, **opts)
164 break
166 for hook in post_validate_hooks:
167 hook(desc, prop)
170@properties.ColumnProperty.strategy_for(instrument=False, deferred=False)
171class _UninstrumentedColumnLoader(LoaderStrategy):
172 """Represent a non-instrumented MapperProperty.
174 The polymorphic_on argument of mapper() often results in this,
175 if the argument is against the with_polymorphic selectable.
177 """
179 __slots__ = ("columns",)
181 def __init__(self, parent, strategy_key):
182 super().__init__(parent, strategy_key)
183 self.columns = self.parent_property.columns
185 def setup_query(
186 self,
187 compile_state,
188 query_entity,
189 path,
190 loadopt,
191 adapter,
192 column_collection=None,
193 **kwargs,
194 ):
195 for c in self.columns:
196 if adapter:
197 c = adapter.columns[c]
198 compile_state._append_dedupe_col_collection(c, column_collection)
200 def create_row_processor(
201 self,
202 context,
203 query_entity,
204 path,
205 loadopt,
206 mapper,
207 result,
208 adapter,
209 populators,
210 ):
211 pass
214@log.class_logger
215@properties.ColumnProperty.strategy_for(instrument=True, deferred=False)
216class _ColumnLoader(LoaderStrategy):
217 """Provide loading behavior for a :class:`.ColumnProperty`."""
219 __slots__ = "columns", "is_composite"
221 def __init__(self, parent, strategy_key):
222 super().__init__(parent, strategy_key)
223 self.columns = self.parent_property.columns
224 self.is_composite = hasattr(self.parent_property, "composite_class")
226 def setup_query(
227 self,
228 compile_state,
229 query_entity,
230 path,
231 loadopt,
232 adapter,
233 column_collection,
234 memoized_populators,
235 check_for_adapt=False,
236 **kwargs,
237 ):
238 for c in self.columns:
239 if adapter:
240 if check_for_adapt:
241 c = adapter.adapt_check_present(c)
242 if c is None:
243 return
244 else:
245 c = adapter.columns[c]
247 compile_state._append_dedupe_col_collection(c, column_collection)
249 fetch = self.columns[0]
250 if adapter:
251 fetch = adapter.columns[fetch]
252 if fetch is None:
253 # None happens here only for dml bulk_persistence cases
254 # when context.DMLReturningColFilter is used
255 return
257 memoized_populators[self.parent_property] = fetch
259 def init_class_attribute(self, mapper):
260 self.is_class_level = True
261 coltype = self.columns[0].type
262 # TODO: check all columns ? check for foreign key as well?
263 active_history = (
264 self.parent_property.active_history
265 or self.columns[0].primary_key
266 or (
267 mapper.version_id_col is not None
268 and mapper._columntoproperty.get(mapper.version_id_col, None)
269 is self.parent_property
270 )
271 )
273 _register_attribute(
274 self.parent_property,
275 mapper,
276 useobject=False,
277 compare_function=coltype.compare_values,
278 active_history=active_history,
279 default_scalar_value=self.parent_property._default_scalar_value,
280 )
282 def create_row_processor(
283 self,
284 context,
285 query_entity,
286 path,
287 loadopt,
288 mapper,
289 result,
290 adapter,
291 populators,
292 ):
293 # look through list of columns represented here
294 # to see which, if any, is present in the row.
296 for col in self.columns:
297 if adapter:
298 col = adapter.columns[col]
299 getter = result._getter(col, False)
300 if getter:
301 populators["quick"].append((self.key, getter))
302 break
303 else:
304 populators["expire"].append((self.key, True))
307@log.class_logger
308@properties.ColumnProperty.strategy_for(query_expression=True)
309class _ExpressionColumnLoader(_ColumnLoader):
310 def __init__(self, parent, strategy_key):
311 super().__init__(parent, strategy_key)
313 # compare to the "default" expression that is mapped in
314 # the column. If it's sql.null, we don't need to render
315 # unless an expr is passed in the options.
316 null = sql.null().label(None)
317 self._have_default_expression = any(
318 not c.compare(null) for c in self.parent_property.columns
319 )
321 def setup_query(
322 self,
323 compile_state,
324 query_entity,
325 path,
326 loadopt,
327 adapter,
328 column_collection,
329 memoized_populators,
330 **kwargs,
331 ):
332 columns = None
333 if loadopt and loadopt._extra_criteria:
334 columns = loadopt._extra_criteria
336 elif self._have_default_expression:
337 columns = self.parent_property.columns
339 if columns is None:
340 return
342 for c in columns:
343 if adapter:
344 c = adapter.columns[c]
345 compile_state._append_dedupe_col_collection(c, column_collection)
347 fetch = columns[0]
348 if adapter:
349 fetch = adapter.columns[fetch]
350 if fetch is None:
351 # None is not expected to be the result of any
352 # adapter implementation here, however there may be theoretical
353 # usages of returning() with context.DMLReturningColFilter
354 return
356 memoized_populators[self.parent_property] = fetch
358 # if the column being loaded is the polymorphic discriminator,
359 # and we have a with_expression() providing the actual column,
360 # update the query_entity to use the actual column instead of
361 # the default expression
362 if (
363 query_entity._polymorphic_discriminator is self.columns[0]
364 and loadopt
365 and loadopt._extra_criteria
366 ):
367 query_entity._polymorphic_discriminator = columns[0]
369 def create_row_processor(
370 self,
371 context,
372 query_entity,
373 path,
374 loadopt,
375 mapper,
376 result,
377 adapter,
378 populators,
379 ):
380 # look through list of columns represented here
381 # to see which, if any, is present in the row.
382 if loadopt and loadopt._extra_criteria:
383 columns = loadopt._extra_criteria
385 for col in columns:
386 if adapter:
387 col = adapter.columns[col]
388 getter = result._getter(col, False)
389 if getter:
390 populators["quick"].append((self.key, getter))
391 break
392 else:
393 populators["expire"].append((self.key, True))
395 def init_class_attribute(self, mapper):
396 self.is_class_level = True
398 _register_attribute(
399 self.parent_property,
400 mapper,
401 useobject=False,
402 compare_function=self.columns[0].type.compare_values,
403 accepts_scalar_loader=False,
404 default_scalar_value=self.parent_property._default_scalar_value,
405 )
408@log.class_logger
409@properties.ColumnProperty.strategy_for(deferred=True, instrument=True)
410@properties.ColumnProperty.strategy_for(
411 deferred=True, instrument=True, raiseload=True
412)
413@properties.ColumnProperty.strategy_for(do_nothing=True)
414class _DeferredColumnLoader(LoaderStrategy):
415 """Provide loading behavior for a deferred :class:`.ColumnProperty`."""
417 __slots__ = "columns", "group", "raiseload"
419 def __init__(self, parent, strategy_key):
420 super().__init__(parent, strategy_key)
421 if hasattr(self.parent_property, "composite_class"):
422 raise NotImplementedError(
423 "Deferred loading for composite types not implemented yet"
424 )
425 self.raiseload = self.strategy_opts.get("raiseload", False)
426 self.columns = self.parent_property.columns
427 self.group = self.parent_property.group
429 def create_row_processor(
430 self,
431 context,
432 query_entity,
433 path,
434 loadopt,
435 mapper,
436 result,
437 adapter,
438 populators,
439 ):
440 # for a DeferredColumnLoader, this method is only used during a
441 # "row processor only" query; see test_deferred.py ->
442 # tests with "rowproc_only" in their name. As of the 1.0 series,
443 # loading._instance_processor doesn't use a "row processing" function
444 # to populate columns, instead it uses data in the "populators"
445 # dictionary. Normally, the DeferredColumnLoader.setup_query()
446 # sets up that data in the "memoized_populators" dictionary
447 # and "create_row_processor()" here is never invoked.
449 if (
450 context.refresh_state
451 and context.query._compile_options._only_load_props
452 and self.key in context.query._compile_options._only_load_props
453 ):
454 self.parent_property._get_strategy(
455 (("deferred", False), ("instrument", True))
456 ).create_row_processor(
457 context,
458 query_entity,
459 path,
460 loadopt,
461 mapper,
462 result,
463 adapter,
464 populators,
465 )
467 elif not self.is_class_level:
468 if self.raiseload:
469 set_deferred_for_local_state = (
470 self.parent_property._raise_column_loader
471 )
472 else:
473 set_deferred_for_local_state = (
474 self.parent_property._deferred_column_loader
475 )
476 populators["new"].append((self.key, set_deferred_for_local_state))
477 else:
478 populators["expire"].append((self.key, False))
480 def init_class_attribute(self, mapper):
481 self.is_class_level = True
483 _register_attribute(
484 self.parent_property,
485 mapper,
486 useobject=False,
487 compare_function=self.columns[0].type.compare_values,
488 callable_=self._load_for_state,
489 load_on_unexpire=False,
490 default_scalar_value=self.parent_property._default_scalar_value,
491 )
493 def setup_query(
494 self,
495 compile_state,
496 query_entity,
497 path,
498 loadopt,
499 adapter,
500 column_collection,
501 memoized_populators,
502 only_load_props=None,
503 **kw,
504 ):
505 if (
506 (
507 compile_state.compile_options._render_for_subquery
508 and self.parent_property._renders_in_subqueries
509 )
510 or (
511 loadopt
512 and set(self.columns).intersection(
513 self.parent._should_undefer_in_wildcard
514 )
515 )
516 or (
517 loadopt
518 and self.group
519 and loadopt.local_opts.get(
520 "undefer_group_%s" % self.group, False
521 )
522 )
523 or (only_load_props and self.key in only_load_props)
524 ):
525 self.parent_property._get_strategy(
526 (("deferred", False), ("instrument", True))
527 ).setup_query(
528 compile_state,
529 query_entity,
530 path,
531 loadopt,
532 adapter,
533 column_collection,
534 memoized_populators,
535 **kw,
536 )
537 elif self.is_class_level:
538 memoized_populators[self.parent_property] = _SET_DEFERRED_EXPIRED
539 elif not self.raiseload:
540 memoized_populators[self.parent_property] = _DEFER_FOR_STATE
541 else:
542 memoized_populators[self.parent_property] = _RAISE_FOR_STATE
544 def _load_for_state(self, state, passive):
545 if not state.key:
546 return LoaderCallableStatus.ATTR_EMPTY
548 if not passive & PassiveFlag.SQL_OK:
549 return LoaderCallableStatus.PASSIVE_NO_RESULT
551 localparent = state.manager.mapper
553 if self.group:
554 toload = [
555 p.key
556 for p in localparent.iterate_properties
557 if isinstance(p, StrategizedProperty)
558 and isinstance(p.strategy, _DeferredColumnLoader)
559 and p.group == self.group
560 ]
561 else:
562 toload = [self.key]
564 # narrow the keys down to just those which have no history
565 group = [k for k in toload if k in state.unmodified]
567 session = _state_session(state)
568 if session is None:
569 raise orm_exc.DetachedInstanceError(
570 "Parent instance %s is not bound to a Session; "
571 "deferred load operation of attribute '%s' cannot proceed"
572 % (orm_util.state_str(state), self.key)
573 )
575 if self.raiseload:
576 self._invoke_raise_load(state, passive, "raise")
578 loading._load_scalar_attributes(
579 state.mapper, state, set(group), PASSIVE_OFF
580 )
582 return LoaderCallableStatus.ATTR_WAS_SET
584 def _invoke_raise_load(self, state, passive, lazy):
585 raise sa_exc.InvalidRequestError(
586 "'%s' is not available due to raiseload=True" % (self,)
587 )
590class _LoadDeferredColumns:
591 """serializable loader object used by DeferredColumnLoader"""
593 def __init__(self, key: str, raiseload: bool = False):
594 self.key = key
595 self.raiseload = raiseload
597 def __call__(self, state, passive=attributes.PASSIVE_OFF):
598 key = self.key
600 localparent = state.manager.mapper
601 prop = localparent._props[key]
602 if self.raiseload:
603 strategy_key = (
604 ("deferred", True),
605 ("instrument", True),
606 ("raiseload", True),
607 )
608 else:
609 strategy_key = (("deferred", True), ("instrument", True))
610 strategy = prop._get_strategy(strategy_key)
611 return strategy._load_for_state(state, passive)
614class _AbstractRelationshipLoader(LoaderStrategy):
615 """LoaderStratgies which deal with related objects."""
617 __slots__ = "mapper", "target", "uselist", "entity"
619 def __init__(self, parent, strategy_key):
620 super().__init__(parent, strategy_key)
621 self.mapper = self.parent_property.mapper
622 self.entity = self.parent_property.entity
623 self.target = self.parent_property.target
624 self.uselist = self.parent_property.uselist
626 def _immediateload_create_row_processor(
627 self,
628 context,
629 query_entity,
630 path,
631 loadopt,
632 mapper,
633 result,
634 adapter,
635 populators,
636 ):
637 return self.parent_property._get_strategy(
638 (("lazy", "immediate"),)
639 ).create_row_processor(
640 context,
641 query_entity,
642 path,
643 loadopt,
644 mapper,
645 result,
646 adapter,
647 populators,
648 )
651@log.class_logger
652@relationships.RelationshipProperty.strategy_for(do_nothing=True)
653class _DoNothingLoader(LoaderStrategy):
654 """Relationship loader that makes no change to the object's state.
656 Compared to NoLoader, this loader does not initialize the
657 collection/attribute to empty/none; the usual default LazyLoader will
658 take effect.
660 """
663@log.class_logger
664@relationships.RelationshipProperty.strategy_for(lazy="noload")
665@relationships.RelationshipProperty.strategy_for(lazy=None)
666class _NoLoader(_AbstractRelationshipLoader):
667 """Provide loading behavior for a :class:`.Relationship`
668 with "lazy=None".
670 """
672 __slots__ = ()
674 @util.deprecated(
675 "2.1",
676 "The ``noload`` loader strategy is deprecated and will be removed "
677 "in a future release. This option "
678 "produces incorrect results by returning ``None`` for related "
679 "items.",
680 )
681 def init_class_attribute(self, mapper):
682 self.is_class_level = True
684 _register_attribute(
685 self.parent_property,
686 mapper,
687 useobject=True,
688 typecallable=self.parent_property.collection_class,
689 )
691 def create_row_processor(
692 self,
693 context,
694 query_entity,
695 path,
696 loadopt,
697 mapper,
698 result,
699 adapter,
700 populators,
701 ):
702 def invoke_no_load(state, dict_, row):
703 if self.uselist:
704 attributes.init_state_collection(state, dict_, self.key)
705 else:
706 dict_[self.key] = None
708 populators["new"].append((self.key, invoke_no_load))
711@log.class_logger
712@relationships.RelationshipProperty.strategy_for(lazy=True)
713@relationships.RelationshipProperty.strategy_for(lazy="select")
714@relationships.RelationshipProperty.strategy_for(lazy="raise")
715@relationships.RelationshipProperty.strategy_for(lazy="raise_on_sql")
716@relationships.RelationshipProperty.strategy_for(lazy="baked_select")
717class _LazyLoader(
718 _AbstractRelationshipLoader, util.MemoizedSlots, log.Identified
719):
720 """Provide loading behavior for a :class:`.Relationship`
721 with "lazy=True", that is loads when first accessed.
723 """
725 __slots__ = (
726 "_lazywhere",
727 "_rev_lazywhere",
728 "_lazyload_reverse_option",
729 "_order_by",
730 "use_get",
731 "is_aliased_class",
732 "_bind_to_col",
733 "_equated_columns",
734 "_rev_bind_to_col",
735 "_rev_equated_columns",
736 "_simple_lazy_clause",
737 "_raise_always",
738 "_raise_on_sql",
739 )
741 _lazywhere: ColumnElement[bool]
742 _bind_to_col: Dict[str, ColumnElement[Any]]
743 _rev_lazywhere: ColumnElement[bool]
744 _rev_bind_to_col: Dict[str, ColumnElement[Any]]
746 parent_property: RelationshipProperty[Any]
748 def __init__(
749 self, parent: RelationshipProperty[Any], strategy_key: Tuple[Any, ...]
750 ):
751 super().__init__(parent, strategy_key)
752 self._raise_always = self.strategy_opts["lazy"] == "raise"
753 self._raise_on_sql = self.strategy_opts["lazy"] == "raise_on_sql"
755 self.is_aliased_class = inspect(self.entity).is_aliased_class
757 join_condition = self.parent_property._join_condition
758 (
759 self._lazywhere,
760 self._bind_to_col,
761 self._equated_columns,
762 ) = join_condition.create_lazy_clause()
764 (
765 self._rev_lazywhere,
766 self._rev_bind_to_col,
767 self._rev_equated_columns,
768 ) = join_condition.create_lazy_clause(reverse_direction=True)
770 if self.parent_property.order_by:
771 self._order_by = util.to_list(self.parent_property.order_by)
772 else:
773 self._order_by = None
775 self.logger.info("%s lazy loading clause %s", self, self._lazywhere)
777 # determine if our "lazywhere" clause is the same as the mapper's
778 # get() clause. then we can just use mapper.get()
779 #
780 # TODO: the "not self.uselist" can be taken out entirely; a m2o
781 # load that populates for a list (very unusual, but is possible with
782 # the API) can still set for "None" and the attribute system will
783 # populate as an empty list.
784 self.use_get = (
785 not self.is_aliased_class
786 and not self.uselist
787 and self.entity._get_clause[0].compare(
788 self._lazywhere,
789 use_proxies=True,
790 compare_keys=False,
791 equivalents=self.mapper._equivalent_columns,
792 )
793 )
795 if self.use_get:
796 for col in list(self._equated_columns):
797 if col in self.mapper._equivalent_columns:
798 for c in self.mapper._equivalent_columns[col]:
799 self._equated_columns[c] = self._equated_columns[col]
801 self.logger.info(
802 "%s will use Session.get() to optimize instance loads", self
803 )
805 def init_class_attribute(self, mapper):
806 self.is_class_level = True
808 _legacy_inactive_history_style = (
809 self.parent_property._legacy_inactive_history_style
810 )
812 if self.parent_property.active_history:
813 active_history = True
814 _deferred_history = False
816 elif (
817 self.parent_property.direction is not interfaces.MANYTOONE
818 or not self.use_get
819 ):
820 if _legacy_inactive_history_style:
821 active_history = True
822 _deferred_history = False
823 else:
824 active_history = False
825 _deferred_history = True
826 else:
827 active_history = _deferred_history = False
829 _register_attribute(
830 self.parent_property,
831 mapper,
832 useobject=True,
833 callable_=self._load_for_state,
834 typecallable=self.parent_property.collection_class,
835 active_history=active_history,
836 _deferred_history=_deferred_history,
837 )
839 def _memoized_attr__simple_lazy_clause(self):
840 lazywhere = self._lazywhere
842 criterion, bind_to_col = (lazywhere, self._bind_to_col)
844 params = []
846 def visit_bindparam(bindparam):
847 bindparam.unique = False
849 visitors.traverse(criterion, {}, {"bindparam": visit_bindparam})
851 def visit_bindparam(bindparam):
852 if bindparam._identifying_key in bind_to_col:
853 params.append(
854 (
855 bindparam.key,
856 bind_to_col[bindparam._identifying_key],
857 None,
858 )
859 )
860 elif bindparam.callable is None:
861 params.append((bindparam.key, None, bindparam.value))
863 criterion = visitors.cloned_traverse(
864 criterion, {}, {"bindparam": visit_bindparam}
865 )
867 return criterion, params
869 def _generate_lazy_clause(self, state, passive):
870 criterion, param_keys = self._simple_lazy_clause
872 if state is None:
873 return sql_util.adapt_criterion_to_null(
874 criterion, [key for key, ident, value in param_keys]
875 )
877 mapper = self.parent_property.parent
879 o = state.obj() # strong ref
880 dict_ = attributes.instance_dict(o)
882 if passive & PassiveFlag.INIT_OK:
883 passive ^= PassiveFlag.INIT_OK
885 params = {}
886 for key, ident, value in param_keys:
887 if ident is not None:
888 if passive and passive & PassiveFlag.LOAD_AGAINST_COMMITTED:
889 value = mapper._get_committed_state_attr_by_column(
890 state, dict_, ident, passive
891 )
892 else:
893 value = mapper._get_state_attr_by_column(
894 state, dict_, ident, passive
895 )
897 params[key] = value
899 return criterion, params
901 def _invoke_raise_load(self, state, passive, lazy):
902 raise sa_exc.InvalidRequestError(
903 "'%s' is not available due to lazy='%s'" % (self, lazy)
904 )
906 def _load_for_state(
907 self,
908 state,
909 passive,
910 loadopt=None,
911 extra_criteria=(),
912 extra_options=(),
913 alternate_effective_path=None,
914 execution_options=util.EMPTY_DICT,
915 ):
916 if not state.key and (
917 (
918 not self.parent_property.load_on_pending
919 and not state._load_pending
920 )
921 or not state.session_id
922 ):
923 return LoaderCallableStatus.ATTR_EMPTY
925 pending = not state.key
926 primary_key_identity = None
928 use_get = self.use_get and (not loadopt or not loadopt._extra_criteria)
930 if (not passive & PassiveFlag.SQL_OK and not use_get) or (
931 not passive & attributes.NON_PERSISTENT_OK and pending
932 ):
933 return LoaderCallableStatus.PASSIVE_NO_RESULT
935 if (
936 # we were given lazy="raise"
937 self._raise_always
938 # the no_raise history-related flag was not passed
939 and not passive & PassiveFlag.NO_RAISE
940 and (
941 # if we are use_get and related_object_ok is disabled,
942 # which means we are at most looking in the identity map
943 # for history purposes or otherwise returning
944 # PASSIVE_NO_RESULT, don't raise. This is also a
945 # history-related flag
946 not use_get
947 or passive & PassiveFlag.RELATED_OBJECT_OK
948 )
949 ):
950 self._invoke_raise_load(state, passive, "raise")
952 session = _state_session(state)
953 if not session:
954 if passive & PassiveFlag.NO_RAISE:
955 return LoaderCallableStatus.PASSIVE_NO_RESULT
957 raise orm_exc.DetachedInstanceError(
958 "Parent instance %s is not bound to a Session; "
959 "lazy load operation of attribute '%s' cannot proceed"
960 % (orm_util.state_str(state), self.key)
961 )
963 # if we have a simple primary key load, check the
964 # identity map without generating a Query at all
965 if use_get:
966 primary_key_identity = self._get_ident_for_use_get(
967 session, state, passive
968 )
969 if LoaderCallableStatus.PASSIVE_NO_RESULT in primary_key_identity:
970 return LoaderCallableStatus.PASSIVE_NO_RESULT
971 elif LoaderCallableStatus.NEVER_SET in primary_key_identity:
972 return LoaderCallableStatus.NEVER_SET
974 # test for None alone in primary_key_identity based on
975 # allow_partial_pks preference. PASSIVE_NO_RESULT and NEVER_SET
976 # have already been tested above
977 if not self.mapper.allow_partial_pks:
978 if _none_only_set.intersection(primary_key_identity):
979 return None
980 else:
981 if _none_only_set.issuperset(primary_key_identity):
982 return None
984 if (
985 self.key in state.dict
986 and not passive & PassiveFlag.DEFERRED_HISTORY_LOAD
987 ):
988 return LoaderCallableStatus.ATTR_WAS_SET
990 # look for this identity in the identity map. Delegate to the
991 # Query class in use, as it may have special rules for how it
992 # does this, including how it decides what the correct
993 # identity_token would be for this identity.
995 instance = session._identity_lookup(
996 self.entity,
997 primary_key_identity,
998 passive=passive,
999 lazy_loaded_from=state,
1000 )
1002 if instance is not None:
1003 if instance is LoaderCallableStatus.PASSIVE_CLASS_MISMATCH:
1004 return None
1005 else:
1006 return instance
1007 elif (
1008 not passive & PassiveFlag.SQL_OK
1009 or not passive & PassiveFlag.RELATED_OBJECT_OK
1010 ):
1011 return LoaderCallableStatus.PASSIVE_NO_RESULT
1013 return self._emit_lazyload(
1014 session,
1015 state,
1016 primary_key_identity,
1017 passive,
1018 loadopt,
1019 extra_criteria,
1020 extra_options,
1021 alternate_effective_path,
1022 execution_options,
1023 )
1025 def _get_ident_for_use_get(self, session, state, passive):
1026 instance_mapper = state.manager.mapper
1028 if passive & PassiveFlag.LOAD_AGAINST_COMMITTED:
1029 get_attr = instance_mapper._get_committed_state_attr_by_column
1030 else:
1031 get_attr = instance_mapper._get_state_attr_by_column
1033 dict_ = state.dict
1035 return [
1036 get_attr(state, dict_, self._equated_columns[pk], passive=passive)
1037 for pk in self.mapper.primary_key
1038 ]
1040 @util.preload_module("sqlalchemy.orm.strategy_options")
1041 def _emit_lazyload(
1042 self,
1043 session,
1044 state,
1045 primary_key_identity,
1046 passive,
1047 loadopt,
1048 extra_criteria,
1049 extra_options,
1050 alternate_effective_path,
1051 execution_options,
1052 ):
1053 strategy_options = util.preloaded.orm_strategy_options
1055 clauseelement = self.entity.__clause_element__()
1056 stmt = Select._create_raw_select(
1057 _raw_columns=[clauseelement],
1058 _propagate_attrs=clauseelement._propagate_attrs,
1059 _compile_options=_ORMCompileState.default_compile_options,
1060 )
1061 load_options = QueryContext.default_load_options
1063 load_options += {
1064 "_invoke_all_eagers": False,
1065 "_lazy_loaded_from": state,
1066 }
1068 if self.parent_property.secondary is not None:
1069 stmt = stmt.select_from(
1070 self.mapper, self.parent_property.secondary
1071 )
1073 pending = not state.key
1075 # don't autoflush on pending
1076 if pending or passive & attributes.NO_AUTOFLUSH:
1077 stmt._execution_options = util.immutabledict({"autoflush": False})
1079 use_get = self.use_get
1081 if state.load_options or (loadopt and loadopt._extra_criteria):
1082 if alternate_effective_path is None:
1083 effective_path = state.load_path[self.parent_property]
1084 else:
1085 effective_path = alternate_effective_path[self.parent_property]
1087 opts = state.load_options
1089 if loadopt and loadopt._extra_criteria:
1090 use_get = False
1091 opts += (
1092 orm_util.LoaderCriteriaOption(self.entity, extra_criteria),
1093 )
1095 stmt._with_options = opts
1096 elif alternate_effective_path is None:
1097 # this path is used if there are not already any options
1098 # in the query, but an event may want to add them
1099 effective_path = state.mapper._path_registry[self.parent_property]
1100 else:
1101 # added by immediateloader
1102 effective_path = alternate_effective_path[self.parent_property]
1104 if extra_options:
1105 stmt._with_options += extra_options
1107 stmt._compile_options += {"_current_path": effective_path}
1109 if use_get:
1110 if self._raise_on_sql and not passive & PassiveFlag.NO_RAISE:
1111 self._invoke_raise_load(state, passive, "raise_on_sql")
1113 return loading._load_on_pk_identity(
1114 session,
1115 stmt,
1116 primary_key_identity,
1117 load_options=load_options,
1118 execution_options=execution_options,
1119 )
1121 if self._order_by:
1122 stmt._order_by_clauses = self._order_by
1124 def _lazyload_reverse(compile_context):
1125 for rev in self.parent_property._reverse_property:
1126 # reverse props that are MANYTOONE are loading *this*
1127 # object from get(), so don't need to eager out to those.
1128 if (
1129 rev.direction is interfaces.MANYTOONE
1130 and rev._use_get
1131 and not isinstance(rev.strategy, _LazyLoader)
1132 ):
1133 strategy_options.Load._construct_for_existing_path(
1134 compile_context.compile_options._current_path[
1135 rev.parent
1136 ]
1137 ).lazyload(rev).process_compile_state(compile_context)
1139 stmt = stmt._add_compile_state_func(
1140 _lazyload_reverse, self.parent_property
1141 )
1143 lazy_clause, params = self._generate_lazy_clause(state, passive)
1145 if execution_options:
1146 execution_options = util.EMPTY_DICT.merge_with(
1147 execution_options, {"_sa_orm_load_options": load_options}
1148 )
1149 else:
1150 execution_options = {
1151 "_sa_orm_load_options": load_options,
1152 }
1154 if (
1155 self.key in state.dict
1156 and not passive & PassiveFlag.DEFERRED_HISTORY_LOAD
1157 ):
1158 return LoaderCallableStatus.ATTR_WAS_SET
1160 if pending:
1161 if util.has_intersection(orm_util._none_set, params.values()):
1162 return None
1164 elif util.has_intersection(orm_util._never_set, params.values()):
1165 return None
1167 if self._raise_on_sql and not passive & PassiveFlag.NO_RAISE:
1168 self._invoke_raise_load(state, passive, "raise_on_sql")
1170 stmt._where_criteria = (lazy_clause,)
1172 result = session.execute(
1173 stmt, params, execution_options=execution_options
1174 )
1176 result = result.unique().scalars().all()
1178 if self.uselist:
1179 return result
1180 else:
1181 l = len(result)
1182 if l:
1183 if l > 1:
1184 util.warn(
1185 "Multiple rows returned with "
1186 "uselist=False for lazily-loaded attribute '%s' "
1187 % self.parent_property
1188 )
1190 return result[0]
1191 else:
1192 return None
1194 def create_row_processor(
1195 self,
1196 context,
1197 query_entity,
1198 path,
1199 loadopt,
1200 mapper,
1201 result,
1202 adapter,
1203 populators,
1204 ):
1205 key = self.key
1207 if (
1208 context.load_options._is_user_refresh
1209 and context.query._compile_options._only_load_props
1210 and self.key in context.query._compile_options._only_load_props
1211 ):
1212 return self._immediateload_create_row_processor(
1213 context,
1214 query_entity,
1215 path,
1216 loadopt,
1217 mapper,
1218 result,
1219 adapter,
1220 populators,
1221 )
1223 if not self.is_class_level or (loadopt and loadopt._extra_criteria):
1224 # we are not the primary manager for this attribute
1225 # on this class - set up a
1226 # per-instance lazyloader, which will override the
1227 # class-level behavior.
1228 # this currently only happens when using a
1229 # "lazyload" option on a "no load"
1230 # attribute - "eager" attributes always have a
1231 # class-level lazyloader installed.
1232 set_lazy_callable = (
1233 InstanceState._instance_level_callable_processor
1234 )(
1235 mapper.class_manager,
1236 _LoadLazyAttribute(
1237 key,
1238 self,
1239 loadopt,
1240 (
1241 loadopt._generate_extra_criteria(context)
1242 if loadopt._extra_criteria
1243 else _NO_EXTRA_CRITERIA
1244 ),
1245 ),
1246 key,
1247 )
1249 populators["new"].append((self.key, set_lazy_callable))
1250 elif context.populate_existing or mapper.always_refresh:
1252 def reset_for_lazy_callable(state, dict_, row):
1253 # we are the primary manager for this attribute on
1254 # this class - reset its
1255 # per-instance attribute state, so that the class-level
1256 # lazy loader is
1257 # executed when next referenced on this instance.
1258 # this is needed in
1259 # populate_existing() types of scenarios to reset
1260 # any existing state.
1261 state._reset(dict_, key)
1263 populators["new"].append((self.key, reset_for_lazy_callable))
1266class _LoadLazyAttribute:
1267 """semi-serializable loader object used by LazyLoader
1269 Historically, this object would be carried along with instances that
1270 needed to run lazyloaders, so it had to be serializable to support
1271 cached instances.
1273 this is no longer a general requirement, and the case where this object
1274 is used is exactly the case where we can't really serialize easily,
1275 which is when extra criteria in the loader option is present.
1277 We can't reliably serialize that as it refers to mapped entities and
1278 AliasedClass objects that are local to the current process, which would
1279 need to be matched up on deserialize e.g. the sqlalchemy.ext.serializer
1280 approach.
1282 """
1284 def __init__(self, key, initiating_strategy, loadopt, extra_criteria):
1285 self.key = key
1286 self.strategy_key = initiating_strategy.strategy_key
1287 self.loadopt = loadopt
1288 self.extra_criteria = extra_criteria
1290 def __getstate__(self):
1291 if self.extra_criteria is not _NO_EXTRA_CRITERIA:
1292 util.warn(
1293 "Can't reliably serialize a lazyload() option that "
1294 "contains additional criteria; please use eager loading "
1295 "for this case"
1296 )
1297 return {
1298 "key": self.key,
1299 "strategy_key": self.strategy_key,
1300 "loadopt": self.loadopt,
1301 "extra_criteria": _NO_EXTRA_CRITERIA,
1302 }
1304 def __call__(self, state, passive=attributes.PASSIVE_OFF):
1305 key = self.key
1306 instance_mapper = state.manager.mapper
1307 prop = instance_mapper._props[key]
1308 strategy = prop._strategies[self.strategy_key]
1310 return strategy._load_for_state(
1311 state,
1312 passive,
1313 loadopt=self.loadopt,
1314 extra_criteria=self.extra_criteria,
1315 )
1318class _PostLoader(_AbstractRelationshipLoader):
1319 """A relationship loader that emits a second SELECT statement."""
1321 __slots__ = ()
1323 def _setup_for_recursion(self, context, path, loadopt, join_depth=None):
1324 effective_path = (
1325 context.compile_state.current_path or orm_util.PathRegistry.root
1326 ) + path
1328 top_level_context = context._get_top_level_context()
1329 execution_options = util.immutabledict(
1330 {"sa_top_level_orm_context": top_level_context}
1331 )
1333 if loadopt:
1334 recursion_depth = loadopt.local_opts.get("recursion_depth", None)
1335 unlimited_recursion = recursion_depth == -1
1336 else:
1337 recursion_depth = None
1338 unlimited_recursion = False
1340 if recursion_depth is not None:
1341 if not self.parent_property._is_self_referential:
1342 raise sa_exc.InvalidRequestError(
1343 f"recursion_depth option on relationship "
1344 f"{self.parent_property} not valid for "
1345 "non-self-referential relationship"
1346 )
1347 recursion_depth = context.execution_options.get(
1348 f"_recursion_depth_{id(self)}", recursion_depth
1349 )
1351 if not unlimited_recursion and recursion_depth < 0:
1352 return (
1353 effective_path,
1354 False,
1355 execution_options,
1356 recursion_depth,
1357 )
1359 if not unlimited_recursion:
1360 execution_options = execution_options.union(
1361 {
1362 f"_recursion_depth_{id(self)}": recursion_depth - 1,
1363 }
1364 )
1366 if loading._PostLoad.path_exists(
1367 context, effective_path, self.parent_property
1368 ):
1369 return effective_path, False, execution_options, recursion_depth
1371 path_w_prop = path[self.parent_property]
1372 effective_path_w_prop = effective_path[self.parent_property]
1374 if not path_w_prop.contains(context.attributes, "loader"):
1375 if join_depth:
1376 if effective_path_w_prop.length / 2 > join_depth:
1377 return (
1378 effective_path,
1379 False,
1380 execution_options,
1381 recursion_depth,
1382 )
1383 elif effective_path_w_prop.contains_mapper(self.mapper):
1384 return (
1385 effective_path,
1386 False,
1387 execution_options,
1388 recursion_depth,
1389 )
1391 return effective_path, True, execution_options, recursion_depth
1394@relationships.RelationshipProperty.strategy_for(lazy="immediate")
1395class _ImmediateLoader(_PostLoader):
1396 __slots__ = ("join_depth",)
1398 def __init__(self, parent, strategy_key):
1399 super().__init__(parent, strategy_key)
1400 self.join_depth = self.parent_property.join_depth
1402 def init_class_attribute(self, mapper):
1403 self.parent_property._get_strategy(
1404 (("lazy", "select"),)
1405 ).init_class_attribute(mapper)
1407 def create_row_processor(
1408 self,
1409 context,
1410 query_entity,
1411 path,
1412 loadopt,
1413 mapper,
1414 result,
1415 adapter,
1416 populators,
1417 ):
1418 if not context.compile_state.compile_options._enable_eagerloads:
1419 return
1421 (
1422 effective_path,
1423 run_loader,
1424 execution_options,
1425 recursion_depth,
1426 ) = self._setup_for_recursion(context, path, loadopt, self.join_depth)
1428 if not run_loader:
1429 # this will not emit SQL and will only emit for a many-to-one
1430 # "use get" load. the "_RELATED" part means it may return
1431 # instance even if its expired, since this is a mutually-recursive
1432 # load operation.
1433 flags = attributes.PASSIVE_NO_FETCH_RELATED | PassiveFlag.NO_RAISE
1434 else:
1435 flags = attributes.PASSIVE_OFF | PassiveFlag.NO_RAISE
1437 loading._PostLoad.callable_for_path(
1438 context,
1439 effective_path,
1440 self.parent,
1441 self.parent_property,
1442 self._load_for_path,
1443 loadopt,
1444 flags,
1445 recursion_depth,
1446 execution_options,
1447 )
1449 def _load_for_path(
1450 self,
1451 context,
1452 path,
1453 states,
1454 load_only,
1455 loadopt,
1456 flags,
1457 recursion_depth,
1458 execution_options,
1459 ):
1460 if recursion_depth:
1461 new_opt = Load(loadopt.path.entity)
1462 new_opt.context = (
1463 loadopt,
1464 loadopt._recurse(),
1465 )
1466 alternate_effective_path = path._truncate_recursive()
1467 extra_options = (new_opt,)
1468 else:
1469 alternate_effective_path = path
1470 extra_options = ()
1472 key = self.key
1473 lazyloader = self.parent_property._get_strategy((("lazy", "select"),))
1474 for state, overwrite in states:
1475 dict_ = state.dict
1477 if overwrite or key not in dict_:
1478 value = lazyloader._load_for_state(
1479 state,
1480 flags,
1481 extra_options=extra_options,
1482 alternate_effective_path=alternate_effective_path,
1483 execution_options=execution_options,
1484 )
1485 if value not in (
1486 ATTR_WAS_SET,
1487 LoaderCallableStatus.PASSIVE_NO_RESULT,
1488 ):
1489 state.get_impl(key).set_committed_value(
1490 state, dict_, value
1491 )
1494@log.class_logger
1495@relationships.RelationshipProperty.strategy_for(lazy="subquery")
1496class _SubqueryLoader(_PostLoader):
1497 __slots__ = ("join_depth",)
1499 def __init__(self, parent, strategy_key):
1500 super().__init__(parent, strategy_key)
1501 self.join_depth = self.parent_property.join_depth
1503 def init_class_attribute(self, mapper):
1504 self.parent_property._get_strategy(
1505 (("lazy", "select"),)
1506 ).init_class_attribute(mapper)
1508 def _get_leftmost(
1509 self,
1510 orig_query_entity_index,
1511 subq_path,
1512 current_compile_state,
1513 is_root,
1514 ):
1515 given_subq_path = subq_path
1516 subq_path = subq_path.path
1517 subq_mapper = orm_util._class_to_mapper(subq_path[0])
1519 # determine attributes of the leftmost mapper
1520 if (
1521 self.parent.isa(subq_mapper)
1522 and self.parent_property is subq_path[1]
1523 ):
1524 leftmost_mapper, leftmost_prop = self.parent, self.parent_property
1525 else:
1526 leftmost_mapper, leftmost_prop = subq_mapper, subq_path[1]
1528 if is_root:
1529 # the subq_path is also coming from cached state, so when we start
1530 # building up this path, it has to also be converted to be in terms
1531 # of the current state. this is for the specific case of the entity
1532 # is an AliasedClass against a subquery that's not otherwise going
1533 # to adapt
1534 new_subq_path = current_compile_state._entities[
1535 orig_query_entity_index
1536 ].entity_zero._path_registry[leftmost_prop]
1537 additional = len(subq_path) - len(new_subq_path)
1538 if additional:
1539 new_subq_path += path_registry.PathRegistry.coerce(
1540 subq_path[-additional:]
1541 )
1542 else:
1543 new_subq_path = given_subq_path
1545 leftmost_cols = leftmost_prop.local_columns
1547 leftmost_attr = [
1548 getattr(
1549 new_subq_path.path[0].entity,
1550 leftmost_mapper._columntoproperty[c].key,
1551 )
1552 for c in leftmost_cols
1553 ]
1555 return leftmost_mapper, leftmost_attr, leftmost_prop, new_subq_path
1557 def _generate_from_original_query(
1558 self,
1559 orig_compile_state,
1560 orig_query,
1561 leftmost_mapper,
1562 leftmost_attr,
1563 leftmost_relationship,
1564 orig_entity,
1565 ):
1566 # reformat the original query
1567 # to look only for significant columns
1568 q = orig_query._clone().correlate(None)
1570 # LEGACY: make a Query back from the select() !!
1571 # This suits at least two legacy cases:
1572 # 1. applications which expect before_compile() to be called
1573 # below when we run .subquery() on this query (Keystone)
1574 # 2. applications which are doing subqueryload with complex
1575 # from_self() queries, as query.subquery() / .statement
1576 # has to do the full compile context for multiply-nested
1577 # from_self() (Neutron) - see test_subqload_from_self
1578 # for demo.
1579 q2 = query.Query.__new__(query.Query)
1580 q2.__dict__.update(q.__dict__)
1581 q = q2
1583 # set the query's "FROM" list explicitly to what the
1584 # FROM list would be in any case, as we will be limiting
1585 # the columns in the SELECT list which may no longer include
1586 # all entities mentioned in things like WHERE, JOIN, etc.
1587 if not q._from_obj:
1588 q._enable_assertions = False
1589 q.select_from.non_generative(
1590 q,
1591 *{
1592 ent["entity"]
1593 for ent in _column_descriptions(
1594 orig_query, compile_state=orig_compile_state
1595 )
1596 if ent["entity"] is not None
1597 },
1598 )
1600 # select from the identity columns of the outer (specifically, these
1601 # are the 'local_cols' of the property). This will remove other
1602 # columns from the query that might suggest the right entity which is
1603 # why we do set select_from above. The attributes we have are
1604 # coerced and adapted using the original query's adapter, which is
1605 # needed only for the case of adapting a subclass column to
1606 # that of a polymorphic selectable, e.g. we have
1607 # Engineer.primary_language and the entity is Person. All other
1608 # adaptations, e.g. from_self, select_entity_from(), will occur
1609 # within the new query when it compiles, as the compile_state we are
1610 # using here is only a partial one. If the subqueryload is from a
1611 # with_polymorphic() or other aliased() object, left_attr will already
1612 # be the correct attributes so no adaptation is needed.
1613 target_cols = orig_compile_state._adapt_col_list(
1614 [
1615 sql.coercions.expect(sql.roles.ColumnsClauseRole, o)
1616 for o in leftmost_attr
1617 ],
1618 orig_compile_state._get_current_adapter(),
1619 )
1620 q._raw_columns = target_cols
1622 distinct_target_key = leftmost_relationship.distinct_target_key
1624 if distinct_target_key is True:
1625 q._distinct = True
1626 elif distinct_target_key is None:
1627 # if target_cols refer to a non-primary key or only
1628 # part of a composite primary key, set the q as distinct
1629 for t in {c.table for c in target_cols}:
1630 if not set(target_cols).issuperset(t.primary_key):
1631 q._distinct = True
1632 break
1634 # don't need ORDER BY if no limit/offset
1635 if not q._has_row_limiting_clause:
1636 q._order_by_clauses = ()
1638 if q._distinct is True and q._order_by_clauses:
1639 # the logic to automatically add the order by columns to the query
1640 # when distinct is True is deprecated in the query
1641 to_add = sql_util.expand_column_list_from_order_by(
1642 target_cols, q._order_by_clauses
1643 )
1644 if to_add:
1645 q._set_entities(target_cols + to_add)
1647 # the original query now becomes a subquery
1648 # which we'll join onto.
1649 # LEGACY: as "q" is a Query, the before_compile() event is invoked
1650 # here.
1651 embed_q = q.set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL).subquery()
1652 left_alias = orm_util.AliasedClass(
1653 leftmost_mapper, embed_q, use_mapper_path=True
1654 )
1655 return left_alias
1657 def _prep_for_joins(self, left_alias, subq_path):
1658 # figure out what's being joined. a.k.a. the fun part
1659 to_join = []
1660 pairs = list(subq_path.pairs())
1662 for i, (mapper, prop) in enumerate(pairs):
1663 if i > 0:
1664 # look at the previous mapper in the chain -
1665 # if it is as or more specific than this prop's
1666 # mapper, use that instead.
1667 # note we have an assumption here that
1668 # the non-first element is always going to be a mapper,
1669 # not an AliasedClass
1671 prev_mapper = pairs[i - 1][1].mapper
1672 to_append = prev_mapper if prev_mapper.isa(mapper) else mapper
1673 else:
1674 to_append = mapper
1676 to_join.append((to_append, prop.key))
1678 # determine the immediate parent class we are joining from,
1679 # which needs to be aliased.
1681 if len(to_join) < 2:
1682 # in the case of a one level eager load, this is the
1683 # leftmost "left_alias".
1684 parent_alias = left_alias
1685 else:
1686 info = inspect(to_join[-1][0])
1687 if info.is_aliased_class:
1688 parent_alias = info.entity
1689 else:
1690 # alias a plain mapper as we may be
1691 # joining multiple times
1692 parent_alias = orm_util.AliasedClass(
1693 info.entity, use_mapper_path=True
1694 )
1696 local_cols = self.parent_property.local_columns
1698 local_attr = [
1699 getattr(parent_alias, self.parent._columntoproperty[c].key)
1700 for c in local_cols
1701 ]
1702 return to_join, local_attr, parent_alias
1704 def _apply_joins(
1705 self, q, to_join, left_alias, parent_alias, effective_entity
1706 ):
1707 ltj = len(to_join)
1708 if ltj == 1:
1709 to_join = [
1710 getattr(left_alias, to_join[0][1]).of_type(effective_entity)
1711 ]
1712 elif ltj == 2:
1713 to_join = [
1714 getattr(left_alias, to_join[0][1]).of_type(parent_alias),
1715 getattr(parent_alias, to_join[-1][1]).of_type(
1716 effective_entity
1717 ),
1718 ]
1719 elif ltj > 2:
1720 middle = [
1721 (
1722 (
1723 orm_util.AliasedClass(item[0])
1724 if not inspect(item[0]).is_aliased_class
1725 else item[0].entity
1726 ),
1727 item[1],
1728 )
1729 for item in to_join[1:-1]
1730 ]
1731 inner = []
1733 while middle:
1734 item = middle.pop(0)
1735 attr = getattr(item[0], item[1])
1736 if middle:
1737 attr = attr.of_type(middle[0][0])
1738 else:
1739 attr = attr.of_type(parent_alias)
1741 inner.append(attr)
1743 to_join = (
1744 [getattr(left_alias, to_join[0][1]).of_type(inner[0].parent)]
1745 + inner
1746 + [
1747 getattr(parent_alias, to_join[-1][1]).of_type(
1748 effective_entity
1749 )
1750 ]
1751 )
1753 for attr in to_join:
1754 q = q.join(attr)
1756 return q
1758 def _setup_options(
1759 self,
1760 context,
1761 q,
1762 subq_path,
1763 rewritten_path,
1764 orig_query,
1765 effective_entity,
1766 loadopt,
1767 ):
1768 # note that because the subqueryload object
1769 # does not reuse the cached query, instead always making
1770 # use of the current invoked query, while we have two queries
1771 # here (orig and context.query), they are both non-cached
1772 # queries and we can transfer the options as is without
1773 # adjusting for new criteria. Some work on #6881 / #6889
1774 # brought this into question.
1775 new_options = orig_query._with_options
1777 if loadopt and loadopt._extra_criteria:
1778 new_options += (
1779 orm_util.LoaderCriteriaOption(
1780 effective_entity,
1781 loadopt._generate_extra_criteria(context),
1782 ),
1783 )
1785 # propagate loader options etc. to the new query.
1786 # these will fire relative to subq_path.
1787 q = q._with_current_path(rewritten_path)
1788 q = q.options(*new_options)
1790 return q
1792 def _setup_outermost_orderby(self, q):
1793 if self.parent_property.order_by:
1795 def _setup_outermost_orderby(compile_context):
1796 compile_context.eager_order_by += tuple(
1797 util.to_list(self.parent_property.order_by)
1798 )
1800 q = q._add_compile_state_func(
1801 _setup_outermost_orderby, self.parent_property
1802 )
1804 return q
1806 class _SubqCollections:
1807 """Given a :class:`_query.Query` used to emit the "subquery load",
1808 provide a load interface that executes the query at the
1809 first moment a value is needed.
1811 """
1813 __slots__ = (
1814 "session",
1815 "execution_options",
1816 "load_options",
1817 "params",
1818 "subq",
1819 "_data",
1820 )
1822 def __init__(self, context, subq):
1823 # avoid creating a cycle by storing context
1824 # even though that's preferable
1825 self.session = context.session
1826 self.execution_options = context.execution_options
1827 self.load_options = context.load_options
1828 self.params = context.params or {}
1829 self.subq = subq
1830 self._data = None
1832 def get(self, key, default):
1833 if self._data is None:
1834 self._load()
1835 return self._data.get(key, default)
1837 def _load(self):
1838 self._data = collections.defaultdict(list)
1840 q = self.subq
1841 assert q.session is None
1843 q = q.with_session(self.session)
1845 if self.load_options._populate_existing:
1846 q = q.populate_existing()
1847 # to work with baked query, the parameters may have been
1848 # updated since this query was created, so take these into account
1850 data = self._data
1851 for row in q.params(self.params):
1852 # group plain tuples rather than Row slices, which would
1853 # incur Row construction and Row.__eq__ per row
1854 tup = row._to_tuple_instance()
1855 data[tup[1:]].append(tup[0])
1857 def loader(self, state, dict_, row):
1858 if self._data is None:
1859 self._load()
1861 def _setup_query_from_rowproc(
1862 self,
1863 context,
1864 query_entity,
1865 path,
1866 entity,
1867 loadopt,
1868 adapter,
1869 ):
1870 compile_state = context.compile_state
1871 if (
1872 not compile_state.compile_options._enable_eagerloads
1873 or compile_state.compile_options._for_refresh_state
1874 ):
1875 return
1877 orig_query_entity_index = compile_state._entities.index(query_entity)
1878 context.loaders_require_buffering = True
1880 path = path[self.parent_property]
1882 # build up a path indicating the path from the leftmost
1883 # entity to the thing we're subquery loading.
1884 with_poly_entity = path.get(
1885 compile_state.attributes, "path_with_polymorphic", None
1886 )
1887 if with_poly_entity is not None:
1888 effective_entity = with_poly_entity
1889 else:
1890 effective_entity = self.entity
1892 subq_path, rewritten_path = context.query._execution_options.get(
1893 ("subquery_paths", None),
1894 (orm_util.PathRegistry.root, orm_util.PathRegistry.root),
1895 )
1896 is_root = subq_path is orm_util.PathRegistry.root
1897 subq_path = subq_path + path
1898 rewritten_path = rewritten_path + path
1900 # use the current query being invoked, not the compile state
1901 # one. this is so that we get the current parameters. however,
1902 # it means we can't use the existing compile state, we have to make
1903 # a new one. other approaches include possibly using the
1904 # compiled query but swapping the params, seems only marginally
1905 # less time spent but more complicated
1906 orig_query = context.query._execution_options.get(
1907 ("orig_query", _SubqueryLoader), context.query
1908 )
1910 # make a new compile_state for the query that's probably cached, but
1911 # we're sort of undoing a bit of that caching :(
1912 compile_state_cls = _ORMCompileState._get_plugin_class_for_plugin(
1913 orig_query, "orm"
1914 )
1916 if orig_query._is_lambda_element:
1917 if context.load_options._lazy_loaded_from is None:
1918 util.warn(
1919 'subqueryloader for "%s" must invoke lambda callable '
1920 "at %r in "
1921 "order to produce a new query, decreasing the efficiency "
1922 "of caching for this statement. Consider using "
1923 "selectinload() for more effective full-lambda caching"
1924 % (self, orig_query)
1925 )
1926 orig_query = orig_query._resolved
1928 # this is the more "quick" version, however it's not clear how
1929 # much of this we need. in particular I can't get a test to
1930 # fail if the "set_base_alias" is missing and not sure why that is.
1931 orig_compile_state = compile_state_cls._create_entities_collection(
1932 orig_query, legacy=False
1933 )
1935 (
1936 leftmost_mapper,
1937 leftmost_attr,
1938 leftmost_relationship,
1939 rewritten_path,
1940 ) = self._get_leftmost(
1941 orig_query_entity_index,
1942 rewritten_path,
1943 orig_compile_state,
1944 is_root,
1945 )
1947 # generate a new Query from the original, then
1948 # produce a subquery from it.
1949 left_alias = self._generate_from_original_query(
1950 orig_compile_state,
1951 orig_query,
1952 leftmost_mapper,
1953 leftmost_attr,
1954 leftmost_relationship,
1955 entity,
1956 )
1958 # generate another Query that will join the
1959 # left alias to the target relationships.
1960 # basically doing a longhand
1961 # "from_self()". (from_self() itself not quite industrial
1962 # strength enough for all contingencies...but very close)
1964 q = query.Query(effective_entity)
1966 q._execution_options = context.query._execution_options.merge_with(
1967 context.execution_options,
1968 {
1969 ("orig_query", _SubqueryLoader): orig_query,
1970 ("subquery_paths", None): (subq_path, rewritten_path),
1971 },
1972 )
1974 q = q._set_enable_single_crit(False)
1975 to_join, local_attr, parent_alias = self._prep_for_joins(
1976 left_alias, subq_path
1977 )
1979 q = q.add_columns(*local_attr)
1980 q = self._apply_joins(
1981 q, to_join, left_alias, parent_alias, effective_entity
1982 )
1984 q = self._setup_options(
1985 context,
1986 q,
1987 subq_path,
1988 rewritten_path,
1989 orig_query,
1990 effective_entity,
1991 loadopt,
1992 )
1993 q = self._setup_outermost_orderby(q)
1995 return q
1997 def create_row_processor(
1998 self,
1999 context,
2000 query_entity,
2001 path,
2002 loadopt,
2003 mapper,
2004 result,
2005 adapter,
2006 populators,
2007 ):
2008 if (
2009 loadopt
2010 and context.compile_state.statement is not None
2011 and context.compile_state.statement.is_dml
2012 ):
2013 util.warn_deprecated(
2014 "The subqueryload loader option is not compatible with DML "
2015 "statements such as INSERT, UPDATE. Only SELECT may be used."
2016 "This warning will become an exception in a future release.",
2017 "2.0",
2018 )
2020 if context.refresh_state:
2021 return self._immediateload_create_row_processor(
2022 context,
2023 query_entity,
2024 path,
2025 loadopt,
2026 mapper,
2027 result,
2028 adapter,
2029 populators,
2030 )
2032 _, run_loader, _, _ = self._setup_for_recursion(
2033 context, path, loadopt, self.join_depth
2034 )
2035 if not run_loader:
2036 return
2038 if not isinstance(context.compile_state, _ORMSelectCompileState):
2039 # issue 7505 - subqueryload() in 1.3 and previous would silently
2040 # degrade for from_statement() without warning. this behavior
2041 # is restored here
2042 return
2044 if not self.parent.class_manager[self.key].impl.supports_population:
2045 raise sa_exc.InvalidRequestError(
2046 "'%s' does not support object "
2047 "population - eager loading cannot be applied." % self
2048 )
2050 # a little dance here as the "path" is still something that only
2051 # semi-tracks the exact series of things we are loading, still not
2052 # telling us about with_polymorphic() and stuff like that when it's at
2053 # the root.. the initial MapperEntity is more accurate for this case.
2054 if len(path) == 1:
2055 if not orm_util._entity_isa(query_entity.entity_zero, self.parent):
2056 return
2057 elif not orm_util._entity_isa(
2058 path[-1], self.parent
2059 ) and not self.parent.isa(path[-1].mapper):
2060 # second check accommodates a polymorphic entity where
2061 # the path has been normalized to the base mapper but
2062 # self.parent is a subclass mapper. Fixes #13209.
2063 return
2065 subq = self._setup_query_from_rowproc(
2066 context,
2067 query_entity,
2068 path,
2069 path[-1],
2070 loadopt,
2071 adapter,
2072 )
2074 if subq is None:
2075 return
2077 assert subq.session is None
2079 path = path[self.parent_property]
2081 local_cols = self.parent_property.local_columns
2083 # cache the loaded collections in the context
2084 # so that inheriting mappers don't re-load when they
2085 # call upon create_row_processor again
2086 collections = path.get(context.attributes, "collections")
2087 if collections is None:
2088 collections = self._SubqCollections(context, subq)
2089 path.set(context.attributes, "collections", collections)
2091 if adapter:
2092 local_cols = [adapter.columns[c] for c in local_cols]
2094 if self.uselist:
2095 self._create_collection_loader(
2096 context, result, collections, local_cols, populators
2097 )
2098 else:
2099 self._create_scalar_loader(
2100 context, result, collections, local_cols, populators
2101 )
2103 def _create_collection_loader(
2104 self, context, result, collections, local_cols, populators
2105 ):
2106 tuple_getter = result._tuple_getter(local_cols)
2108 def load_collection_from_subq(state, dict_, row):
2109 collection = collections.get(tuple_getter(row), ())
2110 state.get_impl(self.key).set_committed_value(
2111 state, dict_, collection
2112 )
2114 def load_collection_from_subq_existing_row(state, dict_, row):
2115 if self.key not in dict_:
2116 load_collection_from_subq(state, dict_, row)
2118 populators["new"].append((self.key, load_collection_from_subq))
2119 populators["existing"].append(
2120 (self.key, load_collection_from_subq_existing_row)
2121 )
2123 if context.invoke_all_eagers:
2124 populators["eager"].append((self.key, collections.loader))
2126 def _create_scalar_loader(
2127 self, context, result, collections, local_cols, populators
2128 ):
2129 tuple_getter = result._tuple_getter(local_cols)
2131 def load_scalar_from_subq(state, dict_, row):
2132 collection = collections.get(tuple_getter(row), (None,))
2133 if len(collection) > 1:
2134 util.warn(
2135 "Multiple rows returned with "
2136 "uselist=False for eagerly-loaded attribute '%s' " % self
2137 )
2139 scalar = collection[0]
2140 state.get_impl(self.key).set_committed_value(state, dict_, scalar)
2142 def load_scalar_from_subq_existing_row(state, dict_, row):
2143 if self.key not in dict_:
2144 load_scalar_from_subq(state, dict_, row)
2146 populators["new"].append((self.key, load_scalar_from_subq))
2147 populators["existing"].append(
2148 (self.key, load_scalar_from_subq_existing_row)
2149 )
2150 if context.invoke_all_eagers:
2151 populators["eager"].append((self.key, collections.loader))
2154@log.class_logger
2155@relationships.RelationshipProperty.strategy_for(lazy="joined")
2156@relationships.RelationshipProperty.strategy_for(lazy=False)
2157class _JoinedLoader(_AbstractRelationshipLoader):
2158 """Provide loading behavior for a :class:`.Relationship`
2159 using joined eager loading.
2161 """
2163 __slots__ = "join_depth"
2165 def __init__(self, parent, strategy_key):
2166 super().__init__(parent, strategy_key)
2167 self.join_depth = self.parent_property.join_depth
2169 def init_class_attribute(self, mapper):
2170 self.parent_property._get_strategy(
2171 (("lazy", "select"),)
2172 ).init_class_attribute(mapper)
2174 def setup_query(
2175 self,
2176 compile_state,
2177 query_entity,
2178 path,
2179 loadopt,
2180 adapter,
2181 column_collection=None,
2182 parentmapper=None,
2183 chained_from_outerjoin=False,
2184 **kwargs,
2185 ):
2186 """Add a left outer join to the statement that's being constructed."""
2188 if not compile_state.compile_options._enable_eagerloads:
2189 return
2190 elif (
2191 loadopt
2192 and compile_state.statement is not None
2193 and compile_state.statement.is_dml
2194 ):
2195 util.warn_deprecated(
2196 "The joinedload loader option is not compatible with DML "
2197 "statements such as INSERT, UPDATE. Only SELECT may be used."
2198 "This warning will become an exception in a future release.",
2199 "2.0",
2200 )
2201 elif self.uselist:
2202 compile_state.multi_row_eager_loaders = True
2204 path = path[self.parent_property]
2206 user_defined_adapter = (
2207 self._init_user_defined_eager_proc(
2208 loadopt, compile_state, compile_state.attributes
2209 )
2210 if loadopt
2211 else False
2212 )
2214 if user_defined_adapter is not False:
2215 # setup an adapter but dont create any JOIN, assume it's already
2216 # in the query
2217 (
2218 clauses,
2219 adapter,
2220 add_to_collection,
2221 ) = self._setup_query_on_user_defined_adapter(
2222 compile_state,
2223 query_entity,
2224 path,
2225 adapter,
2226 user_defined_adapter,
2227 )
2229 # don't do "wrap" for multi-row, we want to wrap
2230 # limited/distinct SELECT,
2231 # because we want to put the JOIN on the outside.
2233 else:
2234 # if not via query option, check for
2235 # a cycle
2236 if not path.contains(compile_state.attributes, "loader"):
2237 if self.join_depth:
2238 if path.length / 2 > self.join_depth:
2239 return
2240 elif path.contains_mapper(self.mapper):
2241 return
2243 # add the JOIN and create an adapter
2244 (
2245 clauses,
2246 adapter,
2247 add_to_collection,
2248 chained_from_outerjoin,
2249 ) = self._generate_row_adapter(
2250 compile_state,
2251 query_entity,
2252 path,
2253 loadopt,
2254 adapter,
2255 column_collection,
2256 parentmapper,
2257 chained_from_outerjoin,
2258 )
2260 # for multi-row, we want to wrap limited/distinct SELECT,
2261 # because we want to put the JOIN on the outside.
2262 compile_state.eager_adding_joins = True
2264 with_poly_entity = path.get(
2265 compile_state.attributes, "path_with_polymorphic", None
2266 )
2267 if with_poly_entity is not None:
2268 with_polymorphic = inspect(
2269 with_poly_entity
2270 ).with_polymorphic_mappers
2271 else:
2272 with_polymorphic = None
2274 path = path[self.entity]
2276 loading._setup_entity_query(
2277 compile_state,
2278 self.mapper,
2279 query_entity,
2280 path,
2281 clauses,
2282 add_to_collection,
2283 with_polymorphic=with_polymorphic,
2284 parentmapper=self.mapper,
2285 chained_from_outerjoin=chained_from_outerjoin,
2286 )
2288 has_nones = util.NONE_SET.intersection(compile_state.secondary_columns)
2290 if has_nones:
2291 if with_poly_entity is not None:
2292 raise sa_exc.InvalidRequestError(
2293 "Detected unaliased columns when generating joined "
2294 "load. Make sure to use aliased=True or flat=True "
2295 "when using joined loading with with_polymorphic()."
2296 )
2297 else:
2298 compile_state.secondary_columns = [
2299 c for c in compile_state.secondary_columns if c is not None
2300 ]
2302 def _init_user_defined_eager_proc(
2303 self, loadopt, compile_state, target_attributes
2304 ):
2305 # check if the opt applies at all
2306 if "eager_from_alias" not in loadopt.local_opts:
2307 # nope
2308 return False
2310 path = loadopt.path.parent
2312 # the option applies. check if the "user_defined_eager_row_processor"
2313 # has been built up.
2314 adapter = path.get(
2315 compile_state.attributes, "user_defined_eager_row_processor", False
2316 )
2317 if adapter is not False:
2318 # just return it
2319 return adapter
2321 # otherwise figure it out.
2322 alias = loadopt.local_opts["eager_from_alias"]
2323 root_mapper, prop = path[-2:]
2325 if alias is not None:
2326 if isinstance(alias, str):
2327 alias = prop.target.alias(alias)
2328 adapter = orm_util.ORMAdapter(
2329 orm_util._TraceAdaptRole.JOINEDLOAD_USER_DEFINED_ALIAS,
2330 prop.mapper,
2331 selectable=alias,
2332 equivalents=prop.mapper._equivalent_columns,
2333 limit_on_entity=False,
2334 )
2335 else:
2336 if path.contains(
2337 compile_state.attributes, "path_with_polymorphic"
2338 ):
2339 with_poly_entity = path.get(
2340 compile_state.attributes, "path_with_polymorphic"
2341 )
2342 adapter = orm_util.ORMAdapter(
2343 orm_util._TraceAdaptRole.JOINEDLOAD_PATH_WITH_POLYMORPHIC,
2344 with_poly_entity,
2345 equivalents=prop.mapper._equivalent_columns,
2346 )
2347 else:
2348 adapter = compile_state._polymorphic_adapters.get(
2349 prop.mapper, None
2350 )
2351 path.set(
2352 target_attributes,
2353 "user_defined_eager_row_processor",
2354 adapter,
2355 )
2357 return adapter
2359 def _setup_query_on_user_defined_adapter(
2360 self, context, entity, path, adapter, user_defined_adapter
2361 ):
2362 # apply some more wrapping to the "user defined adapter"
2363 # if we are setting up the query for SQL render.
2364 adapter = entity._get_entity_clauses(context)
2366 if adapter and user_defined_adapter:
2367 user_defined_adapter = user_defined_adapter.wrap(adapter)
2368 path.set(
2369 context.attributes,
2370 "user_defined_eager_row_processor",
2371 user_defined_adapter,
2372 )
2373 elif adapter:
2374 user_defined_adapter = adapter
2375 path.set(
2376 context.attributes,
2377 "user_defined_eager_row_processor",
2378 user_defined_adapter,
2379 )
2381 add_to_collection = context.primary_columns
2382 return user_defined_adapter, adapter, add_to_collection
2384 def _generate_row_adapter(
2385 self,
2386 compile_state,
2387 entity,
2388 path,
2389 loadopt,
2390 adapter,
2391 column_collection,
2392 parentmapper,
2393 chained_from_outerjoin,
2394 ):
2395 with_poly_entity = path.get(
2396 compile_state.attributes, "path_with_polymorphic", None
2397 )
2398 if with_poly_entity:
2399 to_adapt = with_poly_entity
2400 else:
2401 insp = inspect(self.entity)
2402 if insp.is_aliased_class:
2403 alt_selectable = insp.selectable
2404 else:
2405 alt_selectable = None
2407 to_adapt = orm_util.AliasedClass(
2408 self.mapper,
2409 alias=(
2410 alt_selectable._anonymous_fromclause(flat=True)
2411 if alt_selectable is not None
2412 else None
2413 ),
2414 flat=True,
2415 use_mapper_path=True,
2416 )
2418 to_adapt_insp = inspect(to_adapt)
2420 clauses = to_adapt_insp._memo(
2421 ("joinedloader_ormadapter", self),
2422 orm_util.ORMAdapter,
2423 orm_util._TraceAdaptRole.JOINEDLOAD_MEMOIZED_ADAPTER,
2424 to_adapt_insp,
2425 equivalents=self.mapper._equivalent_columns,
2426 adapt_required=True,
2427 allow_label_resolve=False,
2428 anonymize_labels=True,
2429 )
2431 assert clauses.is_aliased_class
2433 innerjoin = (
2434 loadopt.local_opts.get("innerjoin", self.parent_property.innerjoin)
2435 if loadopt is not None
2436 else self.parent_property.innerjoin
2437 )
2439 if not innerjoin:
2440 # if this is an outer join, all non-nested eager joins from
2441 # this path must also be outer joins
2442 chained_from_outerjoin = True
2444 compile_state.create_eager_joins.append(
2445 (
2446 self._create_eager_join,
2447 entity,
2448 path,
2449 adapter,
2450 parentmapper,
2451 clauses,
2452 innerjoin,
2453 chained_from_outerjoin,
2454 loadopt._extra_criteria if loadopt else (),
2455 )
2456 )
2458 add_to_collection = compile_state.secondary_columns
2459 path.set(compile_state.attributes, "eager_row_processor", clauses)
2461 return clauses, adapter, add_to_collection, chained_from_outerjoin
2463 def _create_eager_join(
2464 self,
2465 compile_state,
2466 query_entity,
2467 path,
2468 adapter,
2469 parentmapper,
2470 clauses,
2471 innerjoin,
2472 chained_from_outerjoin,
2473 extra_criteria,
2474 ):
2475 if parentmapper is None:
2476 localparent = query_entity.mapper
2477 else:
2478 localparent = parentmapper
2480 # whether or not the Query will wrap the selectable in a subquery,
2481 # and then attach eager load joins to that (i.e., in the case of
2482 # LIMIT/OFFSET etc.)
2483 should_nest_selectable = compile_state._should_nest_selectable
2485 query_entity_key = None
2487 if (
2488 query_entity not in compile_state.eager_joins
2489 and not should_nest_selectable
2490 and compile_state.from_clauses
2491 ):
2492 indexes = sql_util.find_left_clause_that_matches_given(
2493 compile_state.from_clauses, query_entity.selectable
2494 )
2496 if len(indexes) > 1:
2497 # for the eager load case, I can't reproduce this right
2498 # now. For query.join() I can.
2499 raise sa_exc.InvalidRequestError(
2500 "Can't identify which query entity in which to joined "
2501 "eager load from. Please use an exact match when "
2502 "specifying the join path."
2503 )
2505 if indexes:
2506 clause = compile_state.from_clauses[indexes[0]]
2507 # join to an existing FROM clause on the query.
2508 # key it to its list index in the eager_joins dict.
2509 # Query._compile_context will adapt as needed and
2510 # append to the FROM clause of the select().
2511 query_entity_key, default_towrap = indexes[0], clause
2513 if query_entity_key is None:
2514 query_entity_key, default_towrap = (
2515 query_entity,
2516 query_entity.selectable,
2517 )
2519 towrap = compile_state.eager_joins.setdefault(
2520 query_entity_key, default_towrap
2521 )
2523 if adapter:
2524 if getattr(adapter, "is_aliased_class", False):
2525 # joining from an adapted entity. The adapted entity
2526 # might be a "with_polymorphic", so resolve that to our
2527 # specific mapper's entity before looking for our attribute
2528 # name on it.
2529 efm = adapter.aliased_insp._entity_for_mapper(
2530 localparent
2531 if localparent.isa(self.parent)
2532 else self.parent
2533 )
2535 # look for our attribute on the adapted entity, else fall back
2536 # to our straight property
2537 onclause = getattr(efm.entity, self.key, self.parent_property)
2538 else:
2539 onclause = getattr(
2540 orm_util.AliasedClass(
2541 self.parent, adapter.selectable, use_mapper_path=True
2542 ),
2543 self.key,
2544 self.parent_property,
2545 )
2547 else:
2548 onclause = self.parent_property
2550 assert clauses.is_aliased_class
2552 attach_on_outside = (
2553 not chained_from_outerjoin
2554 or not innerjoin
2555 or innerjoin == "unnested"
2556 or query_entity.entity_zero.represents_outer_join
2557 )
2559 extra_join_criteria = extra_criteria
2560 additional_entity_criteria = compile_state.global_attributes.get(
2561 ("additional_entity_criteria", self.mapper), ()
2562 )
2563 if additional_entity_criteria:
2564 extra_join_criteria += tuple(
2565 ae._resolve_where_criteria(self.mapper)
2566 for ae in additional_entity_criteria
2567 if ae.propagate_to_loaders
2568 )
2570 if attach_on_outside:
2571 # this is the "classic" eager join case.
2572 eagerjoin = orm_util._ORMJoin(
2573 towrap,
2574 clauses.aliased_insp,
2575 onclause,
2576 isouter=not innerjoin
2577 or query_entity.entity_zero.represents_outer_join
2578 or (chained_from_outerjoin and isinstance(towrap, sql.Join)),
2579 _left_memo=self.parent,
2580 _right_memo=path[self.mapper],
2581 _extra_criteria=extra_join_criteria,
2582 )
2583 else:
2584 # all other cases are innerjoin=='nested' approach
2585 eagerjoin = self._splice_nested_inner_join(
2586 path, path[-2], towrap, clauses, onclause, extra_join_criteria
2587 )
2589 compile_state.eager_joins[query_entity_key] = eagerjoin
2591 # send a hint to the Query as to where it may "splice" this join
2592 eagerjoin.stop_on = query_entity.selectable
2594 if not parentmapper:
2595 # for parentclause that is the non-eager end of the join,
2596 # ensure all the parent cols in the primaryjoin are actually
2597 # in the
2598 # columns clause (i.e. are not deferred), so that aliasing applied
2599 # by the Query propagates those columns outward.
2600 # This has the effect
2601 # of "undefering" those columns.
2602 for col in sql_util._find_columns(
2603 self.parent_property.primaryjoin
2604 ):
2605 if localparent.persist_selectable.c.contains_column(col):
2606 if adapter:
2607 col = adapter.columns[col]
2608 compile_state._append_dedupe_col_collection(
2609 col, compile_state.primary_columns
2610 )
2612 if self.parent_property.order_by:
2613 compile_state.eager_order_by += tuple(
2614 (eagerjoin._target_adapter.copy_and_process)(
2615 util.to_list(self.parent_property.order_by)
2616 )
2617 )
2619 def _splice_nested_inner_join(
2620 self,
2621 path,
2622 entity_we_want_to_splice_onto,
2623 join_obj,
2624 clauses,
2625 onclause,
2626 extra_criteria,
2627 entity_inside_join_structure: Union[
2628 Mapper, None, Literal[False]
2629 ] = False,
2630 detected_existing_path: Optional[path_registry.PathRegistry] = None,
2631 ):
2632 # recursive fn to splice a nested join into an existing one.
2633 # entity_inside_join_structure=False means this is the outermost call,
2634 # and it should return a value. entity_inside_join_structure=<mapper>
2635 # indicates we've descended into a join and are looking at a FROM
2636 # clause representing this mapper; if this is not
2637 # entity_we_want_to_splice_onto then return None to end the recursive
2638 # branch
2640 assert entity_we_want_to_splice_onto is path[-2]
2642 if entity_inside_join_structure is False:
2643 assert isinstance(join_obj, orm_util._ORMJoin)
2645 if isinstance(join_obj, sql.selectable.FromGrouping):
2646 # FromGrouping - continue descending into the structure
2647 return self._splice_nested_inner_join(
2648 path,
2649 entity_we_want_to_splice_onto,
2650 join_obj.element,
2651 clauses,
2652 onclause,
2653 extra_criteria,
2654 entity_inside_join_structure,
2655 )
2656 elif isinstance(join_obj, orm_util._ORMJoin):
2657 # _ORMJoin - continue descending into the structure
2659 join_right_path = join_obj._right_memo
2661 # see if right side of join is viable
2662 target_join = self._splice_nested_inner_join(
2663 path,
2664 entity_we_want_to_splice_onto,
2665 join_obj.right,
2666 clauses,
2667 onclause,
2668 extra_criteria,
2669 entity_inside_join_structure=(
2670 join_right_path[-1].mapper
2671 if join_right_path is not None
2672 else None
2673 ),
2674 )
2676 if target_join is not None:
2677 # for a right splice, attempt to flatten out
2678 # a JOIN b JOIN c JOIN .. to avoid needless
2679 # parenthesis nesting
2680 if not join_obj.isouter and not target_join.isouter:
2681 eagerjoin = join_obj._splice_into_center(target_join)
2682 else:
2683 eagerjoin = orm_util._ORMJoin(
2684 join_obj.left,
2685 target_join,
2686 join_obj.onclause,
2687 isouter=join_obj.isouter,
2688 _left_memo=join_obj._left_memo,
2689 )
2691 eagerjoin._target_adapter = target_join._target_adapter
2692 return eagerjoin
2694 else:
2695 # see if left side of join is viable
2696 target_join = self._splice_nested_inner_join(
2697 path,
2698 entity_we_want_to_splice_onto,
2699 join_obj.left,
2700 clauses,
2701 onclause,
2702 extra_criteria,
2703 entity_inside_join_structure=join_obj._left_memo,
2704 detected_existing_path=join_right_path,
2705 )
2707 if target_join is not None:
2708 eagerjoin = orm_util._ORMJoin(
2709 target_join,
2710 join_obj.right,
2711 join_obj.onclause,
2712 isouter=join_obj.isouter,
2713 _right_memo=join_obj._right_memo,
2714 )
2715 eagerjoin._target_adapter = target_join._target_adapter
2716 return eagerjoin
2718 # neither side viable, return None, or fail if this was the top
2719 # most call
2720 if entity_inside_join_structure is False:
2721 assert (
2722 False
2723 ), "assertion failed attempting to produce joined eager loads"
2724 return None
2726 # reached an endpoint (e.g. a table that's mapped, or an alias of that
2727 # table). determine if we can use this endpoint to splice onto
2729 # is this the entity we want to splice onto in the first place?
2730 if not entity_we_want_to_splice_onto.isa(entity_inside_join_structure):
2731 return None
2733 # path check. if we know the path how this join endpoint got here,
2734 # lets look at our path we are satisfying and see if we're in the
2735 # wrong place. This is specifically for when our entity may
2736 # appear more than once in the path, issue #11449
2737 # updated in issue #11965.
2738 if detected_existing_path and len(detected_existing_path) > 2:
2739 # this assertion is currently based on how this call is made,
2740 # where given a join_obj, the call will have these parameters as
2741 # entity_inside_join_structure=join_obj._left_memo
2742 # and entity_inside_join_structure=join_obj._right_memo.mapper
2743 assert detected_existing_path[-3] is entity_inside_join_structure
2745 # from that, see if the path we are targeting matches the
2746 # "existing" path of this join all the way up to the midpoint
2747 # of this join object (e.g. the relationship).
2748 # if not, then this is not our target
2749 #
2750 # a test condition where this test is false looks like:
2751 #
2752 # desired splice: Node->kind->Kind
2753 # path of desired splice: NodeGroup->nodes->Node->kind
2754 # path we've located: NodeGroup->nodes->Node->common_node->Node
2755 #
2756 # above, because we want to splice kind->Kind onto
2757 # NodeGroup->nodes->Node, this is not our path because it actually
2758 # goes more steps than we want into self-referential
2759 # ->common_node->Node
2760 #
2761 # a test condition where this test is true looks like:
2762 #
2763 # desired splice: B->c2s->C2
2764 # path of desired splice: A->bs->B->c2s
2765 # path we've located: A->bs->B->c1s->C1
2766 #
2767 # above, we want to splice c2s->C2 onto B, and the located path
2768 # shows that the join ends with B->c1s->C1. so we will
2769 # add another join onto that, which would create a "branch" that
2770 # we might represent in a pseudopath as:
2771 #
2772 # B->c1s->C1
2773 # ->c2s->C2
2774 #
2775 # i.e. A JOIN B ON <bs> JOIN C1 ON <c1s>
2776 # JOIN C2 ON <c2s>
2777 #
2779 if detected_existing_path[0:-2] != path.path[0:-1]:
2780 return None
2782 return orm_util._ORMJoin(
2783 join_obj,
2784 clauses.aliased_insp,
2785 onclause,
2786 isouter=False,
2787 _left_memo=entity_inside_join_structure,
2788 _right_memo=path[path[-1].mapper],
2789 _extra_criteria=extra_criteria,
2790 )
2792 def _create_eager_adapter(self, context, result, adapter, path, loadopt):
2793 compile_state = context.compile_state
2795 user_defined_adapter = (
2796 self._init_user_defined_eager_proc(
2797 loadopt, compile_state, context.attributes
2798 )
2799 if loadopt
2800 else False
2801 )
2803 if user_defined_adapter is not False:
2804 decorator = user_defined_adapter
2805 # user defined eagerloads are part of the "primary"
2806 # portion of the load.
2807 # the adapters applied to the Query should be honored.
2808 if compile_state.compound_eager_adapter and decorator:
2809 decorator = decorator.wrap(
2810 compile_state.compound_eager_adapter
2811 )
2812 elif compile_state.compound_eager_adapter:
2813 decorator = compile_state.compound_eager_adapter
2814 else:
2815 decorator = path.get(
2816 compile_state.attributes, "eager_row_processor"
2817 )
2818 if decorator is None:
2819 return False
2821 if self.mapper._result_has_identity_key(result, decorator):
2822 return decorator
2823 else:
2824 # no identity key - don't return a row
2825 # processor, will cause a degrade to lazy
2826 return False
2828 def create_row_processor(
2829 self,
2830 context,
2831 query_entity,
2832 path,
2833 loadopt,
2834 mapper,
2835 result,
2836 adapter,
2837 populators,
2838 ):
2840 if not context.compile_state.compile_options._enable_eagerloads:
2841 return
2843 if not self.parent.class_manager[self.key].impl.supports_population:
2844 raise sa_exc.InvalidRequestError(
2845 "'%s' does not support object "
2846 "population - eager loading cannot be applied." % self
2847 )
2849 if self.uselist:
2850 context.loaders_require_uniquing = True
2852 our_path = path[self.parent_property]
2854 eager_adapter = self._create_eager_adapter(
2855 context, result, adapter, our_path, loadopt
2856 )
2858 if eager_adapter is not False:
2859 key = self.key
2861 _instance = loading._instance_processor(
2862 query_entity,
2863 self.mapper,
2864 context,
2865 result,
2866 our_path[self.entity],
2867 eager_adapter,
2868 )
2870 if not self.uselist:
2871 self._create_scalar_loader(context, key, _instance, populators)
2872 else:
2873 self._create_collection_loader(
2874 context, key, _instance, populators
2875 )
2876 else:
2877 self.parent_property._get_strategy(
2878 (("lazy", "select"),)
2879 ).create_row_processor(
2880 context,
2881 query_entity,
2882 path,
2883 loadopt,
2884 mapper,
2885 result,
2886 adapter,
2887 populators,
2888 )
2890 def _create_collection_loader(self, context, key, _instance, populators):
2891 def load_collection_from_joined_new_row(state, dict_, row):
2892 # note this must unconditionally clear out any existing collection.
2893 # an existing collection would be present only in the case of
2894 # populate_existing().
2895 collection = attributes.init_state_collection(state, dict_, key)
2896 result_list = util.UniqueAppender(
2897 collection, "append_without_event"
2898 )
2899 context.attributes[(state, key)] = result_list
2900 inst = _instance(row)
2901 if inst is not None:
2902 result_list.append(inst)
2904 def load_collection_from_joined_existing_row(state, dict_, row):
2905 if (state, key) in context.attributes:
2906 result_list = context.attributes[(state, key)]
2907 else:
2908 # appender_key can be absent from context.attributes
2909 # with isnew=False when self-referential eager loading
2910 # is used; the same instance may be present in two
2911 # distinct sets of result columns
2912 collection = attributes.init_state_collection(
2913 state, dict_, key
2914 )
2915 result_list = util.UniqueAppender(
2916 collection, "append_without_event"
2917 )
2918 context.attributes[(state, key)] = result_list
2919 inst = _instance(row)
2920 if inst is not None:
2921 result_list.append(inst)
2923 def load_collection_from_joined_exec(state, dict_, row):
2924 _instance(row)
2926 populators["new"].append(
2927 (self.key, load_collection_from_joined_new_row)
2928 )
2929 populators["existing"].append(
2930 (self.key, load_collection_from_joined_existing_row)
2931 )
2932 if context.invoke_all_eagers:
2933 populators["eager"].append(
2934 (self.key, load_collection_from_joined_exec)
2935 )
2937 def _create_scalar_loader(self, context, key, _instance, populators):
2938 def load_scalar_from_joined_new_row(state, dict_, row):
2939 # set a scalar object instance directly on the parent
2940 # object, bypassing InstrumentedAttribute event handlers.
2941 dict_[key] = _instance(row)
2943 def load_scalar_from_joined_existing_row(state, dict_, row):
2944 # call _instance on the row, even though the object has
2945 # been created, so that we further descend into properties
2946 existing = _instance(row)
2948 # conflicting value already loaded, this shouldn't happen
2949 if key in dict_:
2950 if existing is not dict_[key]:
2951 util.warn(
2952 "Multiple rows returned with "
2953 "uselist=False for eagerly-loaded attribute '%s' "
2954 % self
2955 )
2956 else:
2957 # this case is when one row has multiple loads of the
2958 # same entity (e.g. via aliasing), one has an attribute
2959 # that the other doesn't.
2960 dict_[key] = existing
2962 def load_scalar_from_joined_exec(state, dict_, row):
2963 _instance(row)
2965 populators["new"].append((self.key, load_scalar_from_joined_new_row))
2966 populators["existing"].append(
2967 (self.key, load_scalar_from_joined_existing_row)
2968 )
2969 if context.invoke_all_eagers:
2970 populators["eager"].append(
2971 (self.key, load_scalar_from_joined_exec)
2972 )
2975@log.class_logger
2976@relationships.RelationshipProperty.strategy_for(lazy="selectin")
2977class _SelectInLoader(_PostLoader, util.MemoizedSlots):
2978 __slots__ = (
2979 "join_depth",
2980 "omit_join",
2981 "_parent_alias",
2982 "_query_info",
2983 "_fallback_query_info",
2984 )
2986 query_info = collections.namedtuple(
2987 "queryinfo",
2988 [
2989 "load_only_child",
2990 "load_with_join",
2991 "in_expr",
2992 "pk_cols",
2993 "zero_idx",
2994 "n_pk",
2995 "child_lookup_cols",
2996 ],
2997 )
2999 _chunksize = 500
3001 @classmethod
3002 def _set_chunksize(cls, loadopt) -> int:
3003 if loadopt is None or hasattr(loadopt, "local_opts") is None:
3004 return cls._chunksize
3006 user_input = loadopt.local_opts.get("chunksize", None)
3007 if user_input is None:
3008 return cls._chunksize
3009 elif not isinstance(user_input, int) or user_input < 1:
3010 raise sa_exc.ArgumentError(
3011 f"'chunksize={user_input}' is not an appropriate input, "
3012 f"please use a positive non-zero integer."
3013 )
3014 return user_input
3016 def __init__(self, parent, strategy_key):
3017 super().__init__(parent, strategy_key)
3018 self.join_depth = self.parent_property.join_depth
3019 is_m2o = self.parent_property.direction is interfaces.MANYTOONE
3020 is_m2m = self.parent_property.direction is interfaces.MANYTOMANY
3022 if self.parent_property.omit_join is not None:
3023 self.omit_join = self.parent_property.omit_join
3024 else:
3025 lazyloader = self.parent_property._get_strategy(
3026 (("lazy", "select"),)
3027 )
3028 if is_m2o:
3029 self.omit_join = lazyloader.use_get
3030 elif is_m2m and not self.parent_property._is_self_referential:
3031 join_cond = self.parent_property._join_condition
3032 self.omit_join = join_cond.secondary_covers_parent_primary_key
3033 else:
3034 self.omit_join = self.parent._get_clause[0].compare(
3035 lazyloader._rev_lazywhere,
3036 use_proxies=True,
3037 compare_keys=False,
3038 equivalents=self.parent._equivalent_columns,
3039 )
3041 if self.omit_join:
3042 if is_m2o:
3043 self._query_info = self._init_for_omit_join_m2o()
3044 self._fallback_query_info = self._init_for_join()
3045 else:
3046 self._query_info = self._init_for_omit_join()
3047 else:
3048 self._query_info = self._init_for_join()
3050 def _init_for_omit_join(self):
3051 pk_to_fk = dict(
3052 self.parent_property._join_condition.local_remote_pairs
3053 )
3054 pk_to_fk.update(
3055 (equiv, pk_to_fk[k])
3056 for k in list(pk_to_fk)
3057 for equiv in self.parent._equivalent_columns.get(k, ())
3058 )
3060 pk_cols = fk_cols = [
3061 pk_to_fk[col] for col in self.parent.primary_key if col in pk_to_fk
3062 ]
3063 if len(fk_cols) > 1:
3064 in_expr = sql.tuple_(*fk_cols)
3065 zero_idx = False
3066 else:
3067 in_expr = fk_cols[0]
3068 zero_idx = True
3070 return self.query_info(
3071 False, False, in_expr, pk_cols, zero_idx, len(pk_cols), None
3072 )
3074 def _init_for_omit_join_m2o(self):
3075 pk_cols = self.mapper.primary_key
3076 if len(pk_cols) > 1:
3077 in_expr = sql.tuple_(*pk_cols)
3078 zero_idx = False
3079 else:
3080 in_expr = pk_cols[0]
3081 zero_idx = True
3083 lazyloader = self.parent_property._get_strategy((("lazy", "select"),))
3084 lookup_cols = [lazyloader._equated_columns[pk] for pk in pk_cols]
3086 return self.query_info(
3087 True, False, in_expr, pk_cols, zero_idx, len(pk_cols), lookup_cols
3088 )
3090 def _init_for_join(self):
3091 self._parent_alias = AliasedClass(self.parent.class_)
3092 pa_insp = inspect(self._parent_alias)
3093 pk_cols = [
3094 pa_insp._adapt_element(col) for col in self.parent.primary_key
3095 ]
3096 if len(pk_cols) > 1:
3097 in_expr = sql.tuple_(*pk_cols)
3098 zero_idx = False
3099 else:
3100 in_expr = pk_cols[0]
3101 zero_idx = True
3102 return self.query_info(
3103 False, True, in_expr, pk_cols, zero_idx, len(pk_cols), None
3104 )
3106 def init_class_attribute(self, mapper):
3107 self.parent_property._get_strategy(
3108 (("lazy", "select"),)
3109 ).init_class_attribute(mapper)
3111 def create_row_processor(
3112 self,
3113 context,
3114 query_entity,
3115 path,
3116 loadopt,
3117 mapper,
3118 result,
3119 adapter,
3120 populators,
3121 ):
3122 if context.refresh_state:
3123 return self._immediateload_create_row_processor(
3124 context,
3125 query_entity,
3126 path,
3127 loadopt,
3128 mapper,
3129 result,
3130 adapter,
3131 populators,
3132 )
3134 (
3135 effective_path,
3136 run_loader,
3137 execution_options,
3138 recursion_depth,
3139 ) = self._setup_for_recursion(
3140 context, path, loadopt, join_depth=self.join_depth
3141 )
3143 if not run_loader:
3144 return
3146 if not context.compile_state.compile_options._enable_eagerloads:
3147 return
3149 if not self.parent.class_manager[self.key].impl.supports_population:
3150 raise sa_exc.InvalidRequestError(
3151 "'%s' does not support object "
3152 "population - eager loading cannot be applied." % self
3153 )
3155 # a little dance here as the "path" is still something that only
3156 # semi-tracks the exact series of things we are loading, still not
3157 # telling us about with_polymorphic() and stuff like that when it's at
3158 # the root.. the initial MapperEntity is more accurate for this case.
3159 if len(path) == 1:
3160 if not orm_util._entity_isa(query_entity.entity_zero, self.parent):
3161 return
3162 elif not orm_util._entity_isa(
3163 path[-1], self.parent
3164 ) and not self.parent.isa(path[-1].mapper):
3165 # second check accommodates a polymorphic entity where
3166 # the path has been normalized to the base mapper but
3167 # self.parent is a subclass mapper, e.g.
3168 # joinedload(A.b.of_type(poly)).selectinload(poly.Sub.rel)
3169 # Fixes #13209.
3170 return
3172 selectin_path = effective_path
3174 path_w_prop = path[self.parent_property]
3176 # build up a path indicating the path from the leftmost
3177 # entity to the thing we're subquery loading.
3178 with_poly_entity = path_w_prop.get(
3179 context.attributes, "path_with_polymorphic", None
3180 )
3181 if with_poly_entity is not None:
3182 effective_entity = inspect(with_poly_entity)
3183 else:
3184 effective_entity = self.entity
3186 loading._PostLoad.callable_for_path(
3187 context,
3188 selectin_path,
3189 self.parent,
3190 self.parent_property,
3191 self._load_for_path,
3192 effective_entity,
3193 loadopt,
3194 recursion_depth,
3195 execution_options,
3196 )
3198 def _load_for_path(
3199 self,
3200 context,
3201 path,
3202 states,
3203 load_only,
3204 effective_entity,
3205 loadopt,
3206 recursion_depth,
3207 execution_options,
3208 ):
3209 if load_only and self.key not in load_only:
3210 return
3212 query_info = self._query_info
3214 if query_info.load_only_child:
3215 our_states = collections.defaultdict(list)
3216 none_states = []
3218 mapper = self.parent
3220 # attribute keys for the lookup columns; when these are
3221 # present in a state's dict, reading them directly is
3222 # equivalent to the PASSIVE_NO_FETCH attribute lookup below.
3223 # whether or not a key is present can vary per state, e.g.
3224 # individual instances may have the attribute expired or
3225 # deferred, so this is determined state-by-state
3226 get_related_ident = mapper._state_ident_getter(
3227 query_info.child_lookup_cols,
3228 passive=attributes.PASSIVE_NO_FETCH,
3229 )
3231 for state, overwrite in states:
3232 state_dict = state.dict
3233 related_ident = get_related_ident(state, state_dict)
3234 # if the loaded parent objects do not have the foreign key
3235 # to the related item loaded, then degrade into the joined
3236 # version of selectinload
3237 if LoaderCallableStatus.PASSIVE_NO_RESULT in related_ident:
3238 query_info = self._fallback_query_info
3239 break
3241 # organize states into lists keyed to particular foreign
3242 # key values.
3243 if None not in related_ident:
3244 our_states[related_ident].append(
3245 (state, state_dict, overwrite)
3246 )
3247 else:
3248 # For FK values that have None, add them to a
3249 # separate collection that will be populated separately
3250 none_states.append((state, state_dict, overwrite))
3252 # note the above conditional may have changed query_info
3253 if not query_info.load_only_child:
3254 our_states = [
3255 (state.key[1], state, state.dict, overwrite)
3256 for state, overwrite in states
3257 ]
3259 pk_cols = query_info.pk_cols
3260 in_expr = query_info.in_expr
3262 if not query_info.load_with_join:
3263 # in "omit join" mode, the primary key column and the
3264 # "in" expression are in terms of the related entity. So
3265 # if the related entity is polymorphic or otherwise aliased,
3266 # we need to adapt our "pk_cols" and "in_expr" to that
3267 # entity. in non-"omit join" mode, these are against the
3268 # parent entity and do not need adaption.
3269 if effective_entity.is_aliased_class:
3270 pk_cols = [
3271 effective_entity._adapt_element(col) for col in pk_cols
3272 ]
3273 in_expr = effective_entity._adapt_element(in_expr)
3275 entity_sql = effective_entity.__clause_element__()
3276 q = Select._create_raw_select(
3277 _raw_columns=[*pk_cols, entity_sql],
3278 _compile_options=_ORMCompileState.default_compile_options,
3279 _propagate_attrs={
3280 "compile_state_plugin": "orm",
3281 "plugin_subject": effective_entity,
3282 },
3283 )
3285 if (
3286 self.parent_property.secondary is not None
3287 and self.omit_join is True
3288 ):
3289 # The secondaryjoin condition is used to connect the
3290 # secondary table to the related entity,
3291 # and is required for composite foreign keys where SQLAlchemy
3292 # cannot determine the join condition.
3293 q = q.select_from(self.parent_property.secondary).join(
3294 entity_sql, self.parent_property._join_condition.secondaryjoin
3295 )
3296 elif not query_info.load_with_join:
3297 # the pk columns in the "omit_join" case are raw, non-annotated
3298 # columns, so to ensure the Query knows its primary entity, we
3299 # add it explicitly. Using annotated columns here would hit a
3300 # performance issue detailed in issue #4347.
3301 q = q.select_from(effective_entity)
3302 else:
3303 # in the non-omit_join case, the pk columns are against the
3304 # annotated/mapped column of the parent entity, but the #4347
3305 # issue does not occur in this case.
3306 q = q.select_from(self._parent_alias).join(
3307 getattr(self._parent_alias, self.parent_property.key).of_type(
3308 effective_entity
3309 )
3310 )
3312 q = q.filter(in_expr.in_(sql.bindparam("primary_keys")))
3314 # a test which exercises what these comments talk about is
3315 # test_selectin_relations.py -> test_twolevel_selectin_w_polymorphic
3316 #
3317 # effective_entity above is given to us in terms of the cached
3318 # statement, namely this one:
3319 orig_query = context.compile_state.select_statement
3321 # the actual statement that was requested is this one:
3322 # context_query = context.user_passed_query
3323 #
3324 # that's not the cached one, however. So while it is of the identical
3325 # structure, if it has entities like AliasedInsp, which we get from
3326 # aliased() or with_polymorphic(), the AliasedInsp will likely be a
3327 # different object identity each time, and will not match up
3328 # hashing-wise to the corresponding AliasedInsp that's in the
3329 # cached query, meaning it won't match on paths and loader lookups
3330 # and loaders like this one will be skipped if it is used in options.
3331 #
3332 # as it turns out, standard loader options like selectinload(),
3333 # lazyload() that have a path need
3334 # to come from the cached query so that the AliasedInsp etc. objects
3335 # that are in the query line up with the object that's in the path
3336 # of the strategy object. however other options like
3337 # with_loader_criteria() that doesn't have a path (has a fixed entity)
3338 # and needs to have access to the latest closure state in order to
3339 # be correct, we need to use the uncached one.
3340 #
3341 # as of #8399 we let the loader option itself figure out what it
3342 # wants to do given cached and uncached version of itself.
3344 effective_path = path[self.parent_property]
3346 if orig_query is context.user_passed_query:
3347 new_options = orig_query._with_options
3348 else:
3349 cached_options = orig_query._with_options
3350 uncached_options = context.user_passed_query._with_options
3352 # propagate compile state options from the original query,
3353 # updating their "extra_criteria" as necessary.
3354 # note this will create a different cache key than
3355 # "orig" options if extra_criteria is present, because the copy
3356 # of extra_criteria will have different boundparam than that of
3357 # the QueryableAttribute in the path
3358 new_options = [
3359 orig_opt._adapt_cached_option_to_uncached_option(
3360 context, uncached_opt
3361 )
3362 for orig_opt, uncached_opt in zip(
3363 cached_options, uncached_options
3364 )
3365 ]
3367 if loadopt and loadopt._extra_criteria:
3368 new_options += (
3369 orm_util.LoaderCriteriaOption(
3370 effective_entity,
3371 loadopt._generate_extra_criteria(context),
3372 ),
3373 )
3375 if recursion_depth is not None:
3376 effective_path = effective_path._truncate_recursive()
3378 q = q.options(*new_options)
3380 q = q._update_compile_options({"_current_path": effective_path})
3381 if context.populate_existing:
3382 q = q.execution_options(populate_existing=True)
3384 if self.parent_property.order_by:
3385 if not query_info.load_with_join:
3386 eager_order_by = self.parent_property.order_by
3387 if effective_entity.is_aliased_class:
3388 eager_order_by = [
3389 effective_entity._adapt_element(elem)
3390 for elem in eager_order_by
3391 ]
3392 q = q.order_by(*eager_order_by)
3393 else:
3395 def _setup_outermost_orderby(compile_context):
3396 compile_context.eager_order_by += tuple(
3397 util.to_list(self.parent_property.order_by)
3398 )
3400 q = q._add_compile_state_func(
3401 _setup_outermost_orderby, self.parent_property
3402 )
3404 chunksize = self._set_chunksize(loadopt)
3406 if query_info.load_only_child:
3407 self._load_via_child(
3408 our_states,
3409 none_states,
3410 query_info,
3411 q,
3412 context,
3413 execution_options,
3414 chunksize,
3415 )
3416 else:
3417 self._load_via_parent(
3418 our_states,
3419 query_info,
3420 q,
3421 context,
3422 execution_options,
3423 chunksize,
3424 )
3426 def _load_via_child(
3427 self,
3428 our_states,
3429 none_states,
3430 query_info,
3431 q,
3432 context,
3433 execution_options,
3434 chunksize,
3435 ):
3436 uselist = self.uselist
3437 n_pk = query_info.n_pk
3438 zero_idx = query_info.zero_idx
3440 # historically this used sorted instead of list to add determinism in
3441 # tests. Since dicts are now ordered it's likely no longer needed
3442 our_keys = list(our_states)
3443 while our_keys:
3444 chunk = our_keys[0:chunksize]
3445 our_keys = our_keys[chunksize:]
3446 primary_keys = [key[0] if zero_idx else key for key in chunk]
3447 result = context.session.execute(
3448 q,
3449 params={"primary_keys": primary_keys},
3450 execution_options=execution_options,
3451 )
3452 if result.context is not None and result.context.requires_uniquing:
3453 rows = result.unique()
3454 else:
3455 rows = result._raw_all_tuples()
3456 data = {row[:n_pk]: row[n_pk] for row in rows}
3458 for key in chunk:
3459 # for a real foreign key and no concurrent changes to the
3460 # DB while running this method, "key" is always present in
3461 # data. However, for primaryjoins without real foreign keys
3462 # a non-None primaryjoin condition may still refer to no
3463 # related object.
3464 related_obj = data.get(key, None)
3465 for state, dict_, overwrite in our_states[key]:
3466 if not overwrite and self.key in dict_:
3467 continue
3469 state.get_impl(self.key).set_committed_value(
3470 state,
3471 dict_,
3472 related_obj if not uselist else [related_obj],
3473 )
3474 # populate none states with empty value / collection
3475 for state, dict_, overwrite in none_states:
3476 if not overwrite and self.key in dict_:
3477 continue
3479 # note it's OK if this is a uselist=True attribute, the empty
3480 # collection will be populated
3481 state.get_impl(self.key).set_committed_value(state, dict_, None)
3483 def _load_via_parent(
3484 self, our_states, query_info, q, context, execution_options, chunksize
3485 ):
3486 uselist = self.uselist
3487 n_pk = query_info.n_pk
3488 zero_idx = query_info.zero_idx
3489 _empty_result = () if uselist else None
3491 while our_states:
3492 chunk = our_states[0:chunksize]
3493 our_states = our_states[chunksize:]
3495 primary_keys = [
3496 item[0][0] if zero_idx else item[0] for item in chunk
3497 ]
3499 result = context.session.execute(
3500 q,
3501 params={"primary_keys": primary_keys},
3502 execution_options=execution_options,
3503 )
3504 if result.context is not None and result.context.requires_uniquing:
3505 rows = result.unique()
3506 else:
3507 rows = result._raw_all_tuples()
3508 data = collections.defaultdict(list)
3509 for row in rows:
3510 data[row[:n_pk]].append(row[n_pk])
3512 for key, state, state_dict, overwrite in chunk:
3513 if not overwrite and self.key in state_dict:
3514 continue
3516 collection = data.get(key, _empty_result)
3518 if not uselist and collection:
3519 if len(collection) > 1:
3520 util.warn(
3521 "Multiple rows returned with "
3522 "uselist=False for eagerly-loaded "
3523 "attribute '%s' " % self
3524 )
3525 state.get_impl(self.key).set_committed_value(
3526 state, state_dict, collection[0]
3527 )
3528 else:
3529 # note that empty tuple set on uselist=False sets the
3530 # value to None
3531 state.get_impl(self.key).set_committed_value(
3532 state, state_dict, collection
3533 )
3536def _single_parent_validator(desc, prop):
3537 def _do_check(state, value, oldvalue, initiator):
3538 if value is not None and initiator.key == prop.key:
3539 hasparent = initiator.hasparent(attributes.instance_state(value))
3540 if hasparent and oldvalue is not value:
3541 raise sa_exc.InvalidRequestError(
3542 "Instance %s is already associated with an instance "
3543 "of %s via its %s attribute, and is only allowed a "
3544 "single parent."
3545 % (orm_util.instance_str(value), state.class_, prop),
3546 code="bbf1",
3547 )
3548 return value
3550 def append(state, value, initiator):
3551 return _do_check(state, value, None, initiator)
3553 def set_(state, value, oldvalue, initiator):
3554 return _do_check(state, value, oldvalue, initiator)
3556 event.listen(
3557 desc, "append", append, raw=True, retval=True, active_history=True
3558 )
3559 event.listen(desc, "set", set_, raw=True, retval=True, active_history=True)