Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/django/db/models/sql/query.py: 14%
1277 statements
« prev ^ index » next coverage.py v7.0.5, created at 2023-01-17 06:13 +0000
1"""
2Create SQL statements for QuerySets.
4The code in here encapsulates all of the SQL construction so that QuerySets
5themselves do not have to (and could be backed by things other than SQL
6databases). The abstraction barrier only works one way: this module has to know
7all about the internals of models in order to get the information it needs.
8"""
9import copy
10import difflib
11import functools
12import sys
13from collections import Counter, namedtuple
14from collections.abc import Iterator, Mapping
15from itertools import chain, count, product
16from string import ascii_uppercase
18from django.core.exceptions import FieldDoesNotExist, FieldError
19from django.db import DEFAULT_DB_ALIAS, NotSupportedError, connections
20from django.db.models.aggregates import Count
21from django.db.models.constants import LOOKUP_SEP
22from django.db.models.expressions import (
23 BaseExpression,
24 Col,
25 Exists,
26 F,
27 OuterRef,
28 Ref,
29 ResolvedOuterRef,
30 Value,
31)
32from django.db.models.fields import Field
33from django.db.models.fields.related_lookups import MultiColSource
34from django.db.models.lookups import Lookup
35from django.db.models.query_utils import (
36 Q,
37 check_rel_lookup_compatibility,
38 refs_expression,
39)
40from django.db.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE
41from django.db.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin
42from django.db.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode
43from django.utils.functional import cached_property
44from django.utils.regex_helper import _lazy_re_compile
45from django.utils.tree import Node
47__all__ = ["Query", "RawQuery"]
49# Quotation marks ('"`[]), whitespace characters, semicolons, or inline
50# SQL comments are forbidden in column aliases.
51FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(r"['`\"\]\[;\s]|--|/\*|\*/")
53# Inspired from
54# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
55EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w\-]+")
58def get_field_names_from_opts(opts):
59 if opts is None:
60 return set()
61 return set(
62 chain.from_iterable(
63 (f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields()
64 )
65 )
68def get_children_from_q(q):
69 for child in q.children:
70 if isinstance(child, Node):
71 yield from get_children_from_q(child)
72 else:
73 yield child
76JoinInfo = namedtuple(
77 "JoinInfo",
78 ("final_field", "targets", "opts", "joins", "path", "transform_function"),
79)
82class RawQuery:
83 """A single raw SQL query."""
85 def __init__(self, sql, using, params=()):
86 self.params = params
87 self.sql = sql
88 self.using = using
89 self.cursor = None
91 # Mirror some properties of a normal query so that
92 # the compiler can be used to process results.
93 self.low_mark, self.high_mark = 0, None # Used for offset/limit
94 self.extra_select = {}
95 self.annotation_select = {}
97 def chain(self, using):
98 return self.clone(using)
100 def clone(self, using):
101 return RawQuery(self.sql, using, params=self.params)
103 def get_columns(self):
104 if self.cursor is None:
105 self._execute_query()
106 converter = connections[self.using].introspection.identifier_converter
107 return [converter(column_meta[0]) for column_meta in self.cursor.description]
109 def __iter__(self):
110 # Always execute a new query for a new iterator.
111 # This could be optimized with a cache at the expense of RAM.
112 self._execute_query()
113 if not connections[self.using].features.can_use_chunked_reads:
114 # If the database can't use chunked reads we need to make sure we
115 # evaluate the entire query up front.
116 result = list(self.cursor)
117 else:
118 result = self.cursor
119 return iter(result)
121 def __repr__(self):
122 return "<%s: %s>" % (self.__class__.__name__, self)
124 @property
125 def params_type(self):
126 if self.params is None:
127 return None
128 return dict if isinstance(self.params, Mapping) else tuple
130 def __str__(self):
131 if self.params_type is None:
132 return self.sql
133 return self.sql % self.params_type(self.params)
135 def _execute_query(self):
136 connection = connections[self.using]
138 # Adapt parameters to the database, as much as possible considering
139 # that the target type isn't known. See #17755.
140 params_type = self.params_type
141 adapter = connection.ops.adapt_unknown_value
142 if params_type is tuple:
143 params = tuple(adapter(val) for val in self.params)
144 elif params_type is dict:
145 params = {key: adapter(val) for key, val in self.params.items()}
146 elif params_type is None:
147 params = None
148 else:
149 raise RuntimeError("Unexpected params type: %s" % params_type)
151 self.cursor = connection.cursor()
152 self.cursor.execute(self.sql, params)
155ExplainInfo = namedtuple("ExplainInfo", ("format", "options"))
158class Query(BaseExpression):
159 """A single SQL query."""
161 alias_prefix = "T"
162 empty_result_set_value = None
163 subq_aliases = frozenset([alias_prefix])
165 compiler = "SQLCompiler"
167 base_table_class = BaseTable
168 join_class = Join
170 default_cols = True
171 default_ordering = True
172 standard_ordering = True
174 filter_is_sticky = False
175 subquery = False
177 # SQL-related attributes.
178 # Select and related select clauses are expressions to use in the SELECT
179 # clause of the query. The select is used for cases where we want to set up
180 # the select clause to contain other than default fields (values(),
181 # subqueries...). Note that annotations go to annotations dictionary.
182 select = ()
183 # The group_by attribute can have one of the following forms:
184 # - None: no group by at all in the query
185 # - A tuple of expressions: group by (at least) those expressions.
186 # String refs are also allowed for now.
187 # - True: group by all select fields of the model
188 # See compiler.get_group_by() for details.
189 group_by = None
190 order_by = ()
191 low_mark = 0 # Used for offset/limit.
192 high_mark = None # Used for offset/limit.
193 distinct = False
194 distinct_fields = ()
195 select_for_update = False
196 select_for_update_nowait = False
197 select_for_update_skip_locked = False
198 select_for_update_of = ()
199 select_for_no_key_update = False
200 select_related = False
201 has_select_fields = False
202 # Arbitrary limit for select_related to prevents infinite recursion.
203 max_depth = 5
204 # Holds the selects defined by a call to values() or values_list()
205 # excluding annotation_select and extra_select.
206 values_select = ()
208 # SQL annotation-related attributes.
209 annotation_select_mask = None
210 _annotation_select_cache = None
212 # Set combination attributes.
213 combinator = None
214 combinator_all = False
215 combined_queries = ()
217 # These are for extensions. The contents are more or less appended verbatim
218 # to the appropriate clause.
219 extra_select_mask = None
220 _extra_select_cache = None
222 extra_tables = ()
223 extra_order_by = ()
225 # A tuple that is a set of model field names and either True, if these are
226 # the fields to defer, or False if these are the only fields to load.
227 deferred_loading = (frozenset(), True)
229 explain_info = None
231 def __init__(self, model, alias_cols=True):
232 self.model = model
233 self.alias_refcount = {}
234 # alias_map is the most important data structure regarding joins.
235 # It's used for recording which joins exist in the query and what
236 # types they are. The key is the alias of the joined table (possibly
237 # the table name) and the value is a Join-like object (see
238 # sql.datastructures.Join for more information).
239 self.alias_map = {}
240 # Whether to provide alias to columns during reference resolving.
241 self.alias_cols = alias_cols
242 # Sometimes the query contains references to aliases in outer queries (as
243 # a result of split_exclude). Correct alias quoting needs to know these
244 # aliases too.
245 # Map external tables to whether they are aliased.
246 self.external_aliases = {}
247 self.table_map = {} # Maps table names to list of aliases.
248 self.used_aliases = set()
250 self.where = WhereNode()
251 # Maps alias -> Annotation Expression.
252 self.annotations = {}
253 # These are for extensions. The contents are more or less appended
254 # verbatim to the appropriate clause.
255 self.extra = {} # Maps col_alias -> (col_sql, params).
257 self._filtered_relations = {}
259 @property
260 def output_field(self):
261 if len(self.select) == 1:
262 select = self.select[0]
263 return getattr(select, "target", None) or select.field
264 elif len(self.annotation_select) == 1:
265 return next(iter(self.annotation_select.values())).output_field
267 @cached_property
268 def base_table(self):
269 for alias in self.alias_map:
270 return alias
272 def __str__(self):
273 """
274 Return the query as a string of SQL with the parameter values
275 substituted in (use sql_with_params() to see the unsubstituted string).
277 Parameter values won't necessarily be quoted correctly, since that is
278 done by the database interface at execution time.
279 """
280 sql, params = self.sql_with_params()
281 return sql % params
283 def sql_with_params(self):
284 """
285 Return the query as an SQL string and the parameters that will be
286 substituted into the query.
287 """
288 return self.get_compiler(DEFAULT_DB_ALIAS).as_sql()
290 def __deepcopy__(self, memo):
291 """Limit the amount of work when a Query is deepcopied."""
292 result = self.clone()
293 memo[id(self)] = result
294 return result
296 def get_compiler(self, using=None, connection=None, elide_empty=True):
297 if using is None and connection is None:
298 raise ValueError("Need either using or connection")
299 if using:
300 connection = connections[using]
301 return connection.ops.compiler(self.compiler)(
302 self, connection, using, elide_empty
303 )
305 def get_meta(self):
306 """
307 Return the Options instance (the model._meta) from which to start
308 processing. Normally, this is self.model._meta, but it can be changed
309 by subclasses.
310 """
311 if self.model:
312 return self.model._meta
314 def clone(self):
315 """
316 Return a copy of the current Query. A lightweight alternative to
317 deepcopy().
318 """
319 obj = Empty()
320 obj.__class__ = self.__class__
321 # Copy references to everything.
322 obj.__dict__ = self.__dict__.copy()
323 # Clone attributes that can't use shallow copy.
324 obj.alias_refcount = self.alias_refcount.copy()
325 obj.alias_map = self.alias_map.copy()
326 obj.external_aliases = self.external_aliases.copy()
327 obj.table_map = self.table_map.copy()
328 obj.where = self.where.clone()
329 obj.annotations = self.annotations.copy()
330 if self.annotation_select_mask is not None:
331 obj.annotation_select_mask = self.annotation_select_mask.copy()
332 if self.combined_queries:
333 obj.combined_queries = tuple(
334 [query.clone() for query in self.combined_queries]
335 )
336 # _annotation_select_cache cannot be copied, as doing so breaks the
337 # (necessary) state in which both annotations and
338 # _annotation_select_cache point to the same underlying objects.
339 # It will get re-populated in the cloned queryset the next time it's
340 # used.
341 obj._annotation_select_cache = None
342 obj.extra = self.extra.copy()
343 if self.extra_select_mask is not None:
344 obj.extra_select_mask = self.extra_select_mask.copy()
345 if self._extra_select_cache is not None:
346 obj._extra_select_cache = self._extra_select_cache.copy()
347 if self.select_related is not False:
348 # Use deepcopy because select_related stores fields in nested
349 # dicts.
350 obj.select_related = copy.deepcopy(obj.select_related)
351 if "subq_aliases" in self.__dict__:
352 obj.subq_aliases = self.subq_aliases.copy()
353 obj.used_aliases = self.used_aliases.copy()
354 obj._filtered_relations = self._filtered_relations.copy()
355 # Clear the cached_property, if it exists.
356 obj.__dict__.pop("base_table", None)
357 return obj
359 def chain(self, klass=None):
360 """
361 Return a copy of the current Query that's ready for another operation.
362 The klass argument changes the type of the Query, e.g. UpdateQuery.
363 """
364 obj = self.clone()
365 if klass and obj.__class__ != klass:
366 obj.__class__ = klass
367 if not obj.filter_is_sticky:
368 obj.used_aliases = set()
369 obj.filter_is_sticky = False
370 if hasattr(obj, "_setup_query"):
371 obj._setup_query()
372 return obj
374 def relabeled_clone(self, change_map):
375 clone = self.clone()
376 clone.change_aliases(change_map)
377 return clone
379 def _get_col(self, target, field, alias):
380 if not self.alias_cols:
381 alias = None
382 return target.get_col(alias, field)
384 def get_aggregation(self, using, aggregate_exprs):
385 """
386 Return the dictionary with the values of the existing aggregations.
387 """
388 if not aggregate_exprs:
389 return {}
390 aggregates = {}
391 for alias, aggregate_expr in aggregate_exprs.items():
392 self.check_alias(alias)
393 aggregate = aggregate_expr.resolve_expression(
394 self, allow_joins=True, reuse=None, summarize=True
395 )
396 if not aggregate.contains_aggregate:
397 raise TypeError("%s is not an aggregate expression" % alias)
398 aggregates[alias] = aggregate
399 # Existing usage of aggregation can be determined by the presence of
400 # selected aggregates but also by filters against aliased aggregates.
401 _, having, qualify = self.where.split_having_qualify()
402 has_existing_aggregation = (
403 any(
404 getattr(annotation, "contains_aggregate", True)
405 for annotation in self.annotations.values()
406 )
407 or having
408 )
409 # Decide if we need to use a subquery.
410 #
411 # Existing aggregations would cause incorrect results as
412 # get_aggregation() must produce just one result and thus must not use
413 # GROUP BY.
414 #
415 # If the query has limit or distinct, or uses set operations, then
416 # those operations must be done in a subquery so that the query
417 # aggregates on the limit and/or distinct results instead of applying
418 # the distinct and limit after the aggregation.
419 if (
420 isinstance(self.group_by, tuple)
421 or self.is_sliced
422 or has_existing_aggregation
423 or qualify
424 or self.distinct
425 or self.combinator
426 ):
427 from django.db.models.sql.subqueries import AggregateQuery
429 inner_query = self.clone()
430 inner_query.subquery = True
431 outer_query = AggregateQuery(self.model, inner_query)
432 inner_query.select_for_update = False
433 inner_query.select_related = False
434 inner_query.set_annotation_mask(self.annotation_select)
435 # Queries with distinct_fields need ordering and when a limit is
436 # applied we must take the slice from the ordered query. Otherwise
437 # no need for ordering.
438 inner_query.clear_ordering(force=False)
439 if not inner_query.distinct:
440 # If the inner query uses default select and it has some
441 # aggregate annotations, then we must make sure the inner
442 # query is grouped by the main model's primary key. However,
443 # clearing the select clause can alter results if distinct is
444 # used.
445 if inner_query.default_cols and has_existing_aggregation:
446 inner_query.group_by = (
447 self.model._meta.pk.get_col(inner_query.get_initial_alias()),
448 )
449 inner_query.default_cols = False
450 if not qualify:
451 # Mask existing annotations that are not referenced by
452 # aggregates to be pushed to the outer query unless
453 # filtering against window functions is involved as it
454 # requires complex realising.
455 annotation_mask = set()
456 for aggregate in aggregates.values():
457 annotation_mask |= aggregate.get_refs()
458 inner_query.set_annotation_mask(annotation_mask)
460 # Add aggregates to the outer AggregateQuery. This requires making
461 # sure all columns referenced by the aggregates are selected in the
462 # inner query. It is achieved by retrieving all column references
463 # by the aggregates, explicitly selecting them in the inner query,
464 # and making sure the aggregates are repointed to them.
465 col_refs = {}
466 for alias, aggregate in aggregates.items():
467 replacements = {}
468 for col in self._gen_cols([aggregate], resolve_refs=False):
469 if not (col_ref := col_refs.get(col)):
470 index = len(col_refs) + 1
471 col_alias = f"__col{index}"
472 col_ref = Ref(col_alias, col)
473 col_refs[col] = col_ref
474 inner_query.annotations[col_alias] = col
475 inner_query.append_annotation_mask([col_alias])
476 replacements[col] = col_ref
477 outer_query.annotations[alias] = aggregate.replace_expressions(
478 replacements
479 )
480 if (
481 inner_query.select == ()
482 and not inner_query.default_cols
483 and not inner_query.annotation_select_mask
484 ):
485 # In case of Model.objects[0:3].count(), there would be no
486 # field selected in the inner query, yet we must use a subquery.
487 # So, make sure at least one field is selected.
488 inner_query.select = (
489 self.model._meta.pk.get_col(inner_query.get_initial_alias()),
490 )
491 else:
492 outer_query = self
493 self.select = ()
494 self.default_cols = False
495 self.extra = {}
496 if self.annotations:
497 # Inline reference to existing annotations and mask them as
498 # they are unnecessary given only the summarized aggregations
499 # are requested.
500 replacements = {
501 Ref(alias, annotation): annotation
502 for alias, annotation in self.annotations.items()
503 }
504 self.annotations = {
505 alias: aggregate.replace_expressions(replacements)
506 for alias, aggregate in aggregates.items()
507 }
508 else:
509 self.annotations = aggregates
510 self.set_annotation_mask(aggregates)
512 empty_set_result = [
513 expression.empty_result_set_value
514 for expression in outer_query.annotation_select.values()
515 ]
516 elide_empty = not any(result is NotImplemented for result in empty_set_result)
517 outer_query.clear_ordering(force=True)
518 outer_query.clear_limits()
519 outer_query.select_for_update = False
520 outer_query.select_related = False
521 compiler = outer_query.get_compiler(using, elide_empty=elide_empty)
522 result = compiler.execute_sql(SINGLE)
523 if result is None:
524 result = empty_set_result
525 else:
526 converters = compiler.get_converters(outer_query.annotation_select.values())
527 result = next(compiler.apply_converters((result,), converters))
529 return dict(zip(outer_query.annotation_select, result))
531 def get_count(self, using):
532 """
533 Perform a COUNT() query using the current filter constraints.
534 """
535 obj = self.clone()
536 return obj.get_aggregation(using, {"__count": Count("*")})["__count"]
538 def has_filters(self):
539 return self.where
541 def exists(self, limit=True):
542 q = self.clone()
543 if not (q.distinct and q.is_sliced):
544 if q.group_by is True:
545 q.add_fields(
546 (f.attname for f in self.model._meta.concrete_fields), False
547 )
548 # Disable GROUP BY aliases to avoid orphaning references to the
549 # SELECT clause which is about to be cleared.
550 q.set_group_by(allow_aliases=False)
551 q.clear_select_clause()
552 if q.combined_queries and q.combinator == "union":
553 q.combined_queries = tuple(
554 combined_query.exists(limit=False)
555 for combined_query in q.combined_queries
556 )
557 q.clear_ordering(force=True)
558 if limit:
559 q.set_limits(high=1)
560 q.add_annotation(Value(1), "a")
561 return q
563 def has_results(self, using):
564 q = self.exists(using)
565 compiler = q.get_compiler(using=using)
566 return compiler.has_results()
568 def explain(self, using, format=None, **options):
569 q = self.clone()
570 for option_name in options:
571 if (
572 not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name)
573 or "--" in option_name
574 ):
575 raise ValueError(f"Invalid option name: {option_name!r}.")
576 q.explain_info = ExplainInfo(format, options)
577 compiler = q.get_compiler(using=using)
578 return "\n".join(compiler.explain_query())
580 def combine(self, rhs, connector):
581 """
582 Merge the 'rhs' query into the current one (with any 'rhs' effects
583 being applied *after* (that is, "to the right of") anything in the
584 current query. 'rhs' is not modified during a call to this function.
586 The 'connector' parameter describes how to connect filters from the
587 'rhs' query.
588 """
589 if self.model != rhs.model:
590 raise TypeError("Cannot combine queries on two different base models.")
591 if self.is_sliced:
592 raise TypeError("Cannot combine queries once a slice has been taken.")
593 if self.distinct != rhs.distinct:
594 raise TypeError("Cannot combine a unique query with a non-unique query.")
595 if self.distinct_fields != rhs.distinct_fields:
596 raise TypeError("Cannot combine queries with different distinct fields.")
598 # If lhs and rhs shares the same alias prefix, it is possible to have
599 # conflicting alias changes like T4 -> T5, T5 -> T6, which might end up
600 # as T4 -> T6 while combining two querysets. To prevent this, change an
601 # alias prefix of the rhs and update current aliases accordingly,
602 # except if the alias is the base table since it must be present in the
603 # query on both sides.
604 initial_alias = self.get_initial_alias()
605 rhs.bump_prefix(self, exclude={initial_alias})
607 # Work out how to relabel the rhs aliases, if necessary.
608 change_map = {}
609 conjunction = connector == AND
611 # Determine which existing joins can be reused. When combining the
612 # query with AND we must recreate all joins for m2m filters. When
613 # combining with OR we can reuse joins. The reason is that in AND
614 # case a single row can't fulfill a condition like:
615 # revrel__col=1 & revrel__col=2
616 # But, there might be two different related rows matching this
617 # condition. In OR case a single True is enough, so single row is
618 # enough, too.
619 #
620 # Note that we will be creating duplicate joins for non-m2m joins in
621 # the AND case. The results will be correct but this creates too many
622 # joins. This is something that could be fixed later on.
623 reuse = set() if conjunction else set(self.alias_map)
624 joinpromoter = JoinPromoter(connector, 2, False)
625 joinpromoter.add_votes(
626 j for j in self.alias_map if self.alias_map[j].join_type == INNER
627 )
628 rhs_votes = set()
629 # Now, add the joins from rhs query into the new query (skipping base
630 # table).
631 rhs_tables = list(rhs.alias_map)[1:]
632 for alias in rhs_tables:
633 join = rhs.alias_map[alias]
634 # If the left side of the join was already relabeled, use the
635 # updated alias.
636 join = join.relabeled_clone(change_map)
637 new_alias = self.join(join, reuse=reuse)
638 if join.join_type == INNER:
639 rhs_votes.add(new_alias)
640 # We can't reuse the same join again in the query. If we have two
641 # distinct joins for the same connection in rhs query, then the
642 # combined query must have two joins, too.
643 reuse.discard(new_alias)
644 if alias != new_alias:
645 change_map[alias] = new_alias
646 if not rhs.alias_refcount[alias]:
647 # The alias was unused in the rhs query. Unref it so that it
648 # will be unused in the new query, too. We have to add and
649 # unref the alias so that join promotion has information of
650 # the join type for the unused alias.
651 self.unref_alias(new_alias)
652 joinpromoter.add_votes(rhs_votes)
653 joinpromoter.update_join_types(self)
655 # Combine subqueries aliases to ensure aliases relabelling properly
656 # handle subqueries when combining where and select clauses.
657 self.subq_aliases |= rhs.subq_aliases
659 # Now relabel a copy of the rhs where-clause and add it to the current
660 # one.
661 w = rhs.where.clone()
662 w.relabel_aliases(change_map)
663 self.where.add(w, connector)
665 # Selection columns and extra extensions are those provided by 'rhs'.
666 if rhs.select:
667 self.set_select([col.relabeled_clone(change_map) for col in rhs.select])
668 else:
669 self.select = ()
671 if connector == OR:
672 # It would be nice to be able to handle this, but the queries don't
673 # really make sense (or return consistent value sets). Not worth
674 # the extra complexity when you can write a real query instead.
675 if self.extra and rhs.extra:
676 raise ValueError(
677 "When merging querysets using 'or', you cannot have "
678 "extra(select=...) on both sides."
679 )
680 self.extra.update(rhs.extra)
681 extra_select_mask = set()
682 if self.extra_select_mask is not None:
683 extra_select_mask.update(self.extra_select_mask)
684 if rhs.extra_select_mask is not None:
685 extra_select_mask.update(rhs.extra_select_mask)
686 if extra_select_mask:
687 self.set_extra_mask(extra_select_mask)
688 self.extra_tables += rhs.extra_tables
690 # Ordering uses the 'rhs' ordering, unless it has none, in which case
691 # the current ordering is used.
692 self.order_by = rhs.order_by or self.order_by
693 self.extra_order_by = rhs.extra_order_by or self.extra_order_by
695 def _get_defer_select_mask(self, opts, mask, select_mask=None):
696 if select_mask is None:
697 select_mask = {}
698 select_mask[opts.pk] = {}
699 # All concrete fields that are not part of the defer mask must be
700 # loaded. If a relational field is encountered it gets added to the
701 # mask for it be considered if `select_related` and the cycle continues
702 # by recursively caling this function.
703 for field in opts.concrete_fields:
704 field_mask = mask.pop(field.name, None)
705 if field_mask is None:
706 select_mask.setdefault(field, {})
707 elif field_mask:
708 if not field.is_relation:
709 raise FieldError(next(iter(field_mask)))
710 field_select_mask = select_mask.setdefault(field, {})
711 related_model = field.remote_field.model._meta.concrete_model
712 self._get_defer_select_mask(
713 related_model._meta, field_mask, field_select_mask
714 )
715 # Remaining defer entries must be references to reverse relationships.
716 # The following code is expected to raise FieldError if it encounters
717 # a malformed defer entry.
718 for field_name, field_mask in mask.items():
719 if filtered_relation := self._filtered_relations.get(field_name):
720 relation = opts.get_field(filtered_relation.relation_name)
721 field_select_mask = select_mask.setdefault((field_name, relation), {})
722 field = relation.field
723 else:
724 field = opts.get_field(field_name).field
725 field_select_mask = select_mask.setdefault(field, {})
726 related_model = field.model._meta.concrete_model
727 self._get_defer_select_mask(
728 related_model._meta, field_mask, field_select_mask
729 )
730 return select_mask
732 def _get_only_select_mask(self, opts, mask, select_mask=None):
733 if select_mask is None:
734 select_mask = {}
735 select_mask[opts.pk] = {}
736 # Only include fields mentioned in the mask.
737 for field_name, field_mask in mask.items():
738 field = opts.get_field(field_name)
739 field_select_mask = select_mask.setdefault(field, {})
740 if field_mask:
741 if not field.is_relation:
742 raise FieldError(next(iter(field_mask)))
743 related_model = field.remote_field.model._meta.concrete_model
744 self._get_only_select_mask(
745 related_model._meta, field_mask, field_select_mask
746 )
747 return select_mask
749 def get_select_mask(self):
750 """
751 Convert the self.deferred_loading data structure to an alternate data
752 structure, describing the field that *will* be loaded. This is used to
753 compute the columns to select from the database and also by the
754 QuerySet class to work out which fields are being initialized on each
755 model. Models that have all their fields included aren't mentioned in
756 the result, only those that have field restrictions in place.
757 """
758 field_names, defer = self.deferred_loading
759 if not field_names:
760 return {}
761 mask = {}
762 for field_name in field_names:
763 part_mask = mask
764 for part in field_name.split(LOOKUP_SEP):
765 part_mask = part_mask.setdefault(part, {})
766 opts = self.get_meta()
767 if defer:
768 return self._get_defer_select_mask(opts, mask)
769 return self._get_only_select_mask(opts, mask)
771 def table_alias(self, table_name, create=False, filtered_relation=None):
772 """
773 Return a table alias for the given table_name and whether this is a
774 new alias or not.
776 If 'create' is true, a new alias is always created. Otherwise, the
777 most recently created alias for the table (if one exists) is reused.
778 """
779 alias_list = self.table_map.get(table_name)
780 if not create and alias_list:
781 alias = alias_list[0]
782 self.alias_refcount[alias] += 1
783 return alias, False
785 # Create a new alias for this table.
786 if alias_list:
787 alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1)
788 alias_list.append(alias)
789 else:
790 # The first occurrence of a table uses the table name directly.
791 alias = (
792 filtered_relation.alias if filtered_relation is not None else table_name
793 )
794 self.table_map[table_name] = [alias]
795 self.alias_refcount[alias] = 1
796 return alias, True
798 def ref_alias(self, alias):
799 """Increases the reference count for this alias."""
800 self.alias_refcount[alias] += 1
802 def unref_alias(self, alias, amount=1):
803 """Decreases the reference count for this alias."""
804 self.alias_refcount[alias] -= amount
806 def promote_joins(self, aliases):
807 """
808 Promote recursively the join type of given aliases and its children to
809 an outer join. If 'unconditional' is False, only promote the join if
810 it is nullable or the parent join is an outer join.
812 The children promotion is done to avoid join chains that contain a LOUTER
813 b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted,
814 then we must also promote b->c automatically, or otherwise the promotion
815 of a->b doesn't actually change anything in the query results.
816 """
817 aliases = list(aliases)
818 while aliases:
819 alias = aliases.pop(0)
820 if self.alias_map[alias].join_type is None:
821 # This is the base table (first FROM entry) - this table
822 # isn't really joined at all in the query, so we should not
823 # alter its join type.
824 continue
825 # Only the first alias (skipped above) should have None join_type
826 assert self.alias_map[alias].join_type is not None
827 parent_alias = self.alias_map[alias].parent_alias
828 parent_louter = (
829 parent_alias and self.alias_map[parent_alias].join_type == LOUTER
830 )
831 already_louter = self.alias_map[alias].join_type == LOUTER
832 if (self.alias_map[alias].nullable or parent_louter) and not already_louter:
833 self.alias_map[alias] = self.alias_map[alias].promote()
834 # Join type of 'alias' changed, so re-examine all aliases that
835 # refer to this one.
836 aliases.extend(
837 join
838 for join in self.alias_map
839 if self.alias_map[join].parent_alias == alias
840 and join not in aliases
841 )
843 def demote_joins(self, aliases):
844 """
845 Change join type from LOUTER to INNER for all joins in aliases.
847 Similarly to promote_joins(), this method must ensure no join chains
848 containing first an outer, then an inner join are generated. If we
849 are demoting b->c join in chain a LOUTER b LOUTER c then we must
850 demote a->b automatically, or otherwise the demotion of b->c doesn't
851 actually change anything in the query results. .
852 """
853 aliases = list(aliases)
854 while aliases:
855 alias = aliases.pop(0)
856 if self.alias_map[alias].join_type == LOUTER:
857 self.alias_map[alias] = self.alias_map[alias].demote()
858 parent_alias = self.alias_map[alias].parent_alias
859 if self.alias_map[parent_alias].join_type == INNER:
860 aliases.append(parent_alias)
862 def reset_refcounts(self, to_counts):
863 """
864 Reset reference counts for aliases so that they match the value passed
865 in `to_counts`.
866 """
867 for alias, cur_refcount in self.alias_refcount.copy().items():
868 unref_amount = cur_refcount - to_counts.get(alias, 0)
869 self.unref_alias(alias, unref_amount)
871 def change_aliases(self, change_map):
872 """
873 Change the aliases in change_map (which maps old-alias -> new-alias),
874 relabelling any references to them in select columns and the where
875 clause.
876 """
877 # If keys and values of change_map were to intersect, an alias might be
878 # updated twice (e.g. T4 -> T5, T5 -> T6, so also T4 -> T6) depending
879 # on their order in change_map.
880 assert set(change_map).isdisjoint(change_map.values())
882 # 1. Update references in "select" (normal columns plus aliases),
883 # "group by" and "where".
884 self.where.relabel_aliases(change_map)
885 if isinstance(self.group_by, tuple):
886 self.group_by = tuple(
887 [col.relabeled_clone(change_map) for col in self.group_by]
888 )
889 self.select = tuple([col.relabeled_clone(change_map) for col in self.select])
890 self.annotations = self.annotations and {
891 key: col.relabeled_clone(change_map)
892 for key, col in self.annotations.items()
893 }
895 # 2. Rename the alias in the internal table/alias datastructures.
896 for old_alias, new_alias in change_map.items():
897 if old_alias not in self.alias_map:
898 continue
899 alias_data = self.alias_map[old_alias].relabeled_clone(change_map)
900 self.alias_map[new_alias] = alias_data
901 self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
902 del self.alias_refcount[old_alias]
903 del self.alias_map[old_alias]
905 table_aliases = self.table_map[alias_data.table_name]
906 for pos, alias in enumerate(table_aliases):
907 if alias == old_alias:
908 table_aliases[pos] = new_alias
909 break
910 self.external_aliases = {
911 # Table is aliased or it's being changed and thus is aliased.
912 change_map.get(alias, alias): (aliased or alias in change_map)
913 for alias, aliased in self.external_aliases.items()
914 }
916 def bump_prefix(self, other_query, exclude=None):
917 """
918 Change the alias prefix to the next letter in the alphabet in a way
919 that the other query's aliases and this query's aliases will not
920 conflict. Even tables that previously had no alias will get an alias
921 after this call. To prevent changing aliases use the exclude parameter.
922 """
924 def prefix_gen():
925 """
926 Generate a sequence of characters in alphabetical order:
927 -> 'A', 'B', 'C', ...
929 When the alphabet is finished, the sequence will continue with the
930 Cartesian product:
931 -> 'AA', 'AB', 'AC', ...
932 """
933 alphabet = ascii_uppercase
934 prefix = chr(ord(self.alias_prefix) + 1)
935 yield prefix
936 for n in count(1):
937 seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet
938 for s in product(seq, repeat=n):
939 yield "".join(s)
940 prefix = None
942 if self.alias_prefix != other_query.alias_prefix:
943 # No clashes between self and outer query should be possible.
944 return
946 # Explicitly avoid infinite loop. The constant divider is based on how
947 # much depth recursive subquery references add to the stack. This value
948 # might need to be adjusted when adding or removing function calls from
949 # the code path in charge of performing these operations.
950 local_recursion_limit = sys.getrecursionlimit() // 16
951 for pos, prefix in enumerate(prefix_gen()):
952 if prefix not in self.subq_aliases:
953 self.alias_prefix = prefix
954 break
955 if pos > local_recursion_limit:
956 raise RecursionError(
957 "Maximum recursion depth exceeded: too many subqueries."
958 )
959 self.subq_aliases = self.subq_aliases.union([self.alias_prefix])
960 other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases)
961 if exclude is None:
962 exclude = {}
963 self.change_aliases(
964 {
965 alias: "%s%d" % (self.alias_prefix, pos)
966 for pos, alias in enumerate(self.alias_map)
967 if alias not in exclude
968 }
969 )
971 def get_initial_alias(self):
972 """
973 Return the first alias for this query, after increasing its reference
974 count.
975 """
976 if self.alias_map:
977 alias = self.base_table
978 self.ref_alias(alias)
979 elif self.model:
980 alias = self.join(self.base_table_class(self.get_meta().db_table, None))
981 else:
982 alias = None
983 return alias
985 def count_active_tables(self):
986 """
987 Return the number of tables in this query with a non-zero reference
988 count. After execution, the reference counts are zeroed, so tables
989 added in compiler will not be seen by this method.
990 """
991 return len([1 for count in self.alias_refcount.values() if count])
993 def join(self, join, reuse=None, reuse_with_filtered_relation=False):
994 """
995 Return an alias for the 'join', either reusing an existing alias for
996 that join or creating a new one. 'join' is either a base_table_class or
997 join_class.
999 The 'reuse' parameter can be either None which means all joins are
1000 reusable, or it can be a set containing the aliases that can be reused.
1002 The 'reuse_with_filtered_relation' parameter is used when computing
1003 FilteredRelation instances.
1005 A join is always created as LOUTER if the lhs alias is LOUTER to make
1006 sure chains like t1 LOUTER t2 INNER t3 aren't generated. All new
1007 joins are created as LOUTER if the join is nullable.
1008 """
1009 if reuse_with_filtered_relation and reuse:
1010 reuse_aliases = [
1011 a for a, j in self.alias_map.items() if a in reuse and j.equals(join)
1012 ]
1013 else:
1014 reuse_aliases = [
1015 a
1016 for a, j in self.alias_map.items()
1017 if (reuse is None or a in reuse) and j == join
1018 ]
1019 if reuse_aliases:
1020 if join.table_alias in reuse_aliases:
1021 reuse_alias = join.table_alias
1022 else:
1023 # Reuse the most recent alias of the joined table
1024 # (a many-to-many relation may be joined multiple times).
1025 reuse_alias = reuse_aliases[-1]
1026 self.ref_alias(reuse_alias)
1027 return reuse_alias
1029 # No reuse is possible, so we need a new alias.
1030 alias, _ = self.table_alias(
1031 join.table_name, create=True, filtered_relation=join.filtered_relation
1032 )
1033 if join.join_type:
1034 if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable:
1035 join_type = LOUTER
1036 else:
1037 join_type = INNER
1038 join.join_type = join_type
1039 join.table_alias = alias
1040 self.alias_map[alias] = join
1041 return alias
1043 def join_parent_model(self, opts, model, alias, seen):
1044 """
1045 Make sure the given 'model' is joined in the query. If 'model' isn't
1046 a parent of 'opts' or if it is None this method is a no-op.
1048 The 'alias' is the root alias for starting the join, 'seen' is a dict
1049 of model -> alias of existing joins. It must also contain a mapping
1050 of None -> some alias. This will be returned in the no-op case.
1051 """
1052 if model in seen:
1053 return seen[model]
1054 chain = opts.get_base_chain(model)
1055 if not chain:
1056 return alias
1057 curr_opts = opts
1058 for int_model in chain:
1059 if int_model in seen:
1060 curr_opts = int_model._meta
1061 alias = seen[int_model]
1062 continue
1063 # Proxy model have elements in base chain
1064 # with no parents, assign the new options
1065 # object and skip to the next base in that
1066 # case
1067 if not curr_opts.parents[int_model]:
1068 curr_opts = int_model._meta
1069 continue
1070 link_field = curr_opts.get_ancestor_link(int_model)
1071 join_info = self.setup_joins([link_field.name], curr_opts, alias)
1072 curr_opts = int_model._meta
1073 alias = seen[int_model] = join_info.joins[-1]
1074 return alias or seen[None]
1076 def check_alias(self, alias):
1077 if FORBIDDEN_ALIAS_PATTERN.search(alias):
1078 raise ValueError(
1079 "Column aliases cannot contain whitespace characters, quotation marks, "
1080 "semicolons, or SQL comments."
1081 )
1083 def add_annotation(self, annotation, alias, select=True):
1084 """Add a single annotation expression to the Query."""
1085 self.check_alias(alias)
1086 annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None)
1087 if select:
1088 self.append_annotation_mask([alias])
1089 else:
1090 self.set_annotation_mask(set(self.annotation_select).difference({alias}))
1091 self.annotations[alias] = annotation
1093 def resolve_expression(self, query, *args, **kwargs):
1094 clone = self.clone()
1095 # Subqueries need to use a different set of aliases than the outer query.
1096 clone.bump_prefix(query)
1097 clone.subquery = True
1098 clone.where.resolve_expression(query, *args, **kwargs)
1099 # Resolve combined queries.
1100 if clone.combinator:
1101 clone.combined_queries = tuple(
1102 [
1103 combined_query.resolve_expression(query, *args, **kwargs)
1104 for combined_query in clone.combined_queries
1105 ]
1106 )
1107 for key, value in clone.annotations.items():
1108 resolved = value.resolve_expression(query, *args, **kwargs)
1109 if hasattr(resolved, "external_aliases"):
1110 resolved.external_aliases.update(clone.external_aliases)
1111 clone.annotations[key] = resolved
1112 # Outer query's aliases are considered external.
1113 for alias, table in query.alias_map.items():
1114 clone.external_aliases[alias] = (
1115 isinstance(table, Join)
1116 and table.join_field.related_model._meta.db_table != alias
1117 ) or (
1118 isinstance(table, BaseTable) and table.table_name != table.table_alias
1119 )
1120 return clone
1122 def get_external_cols(self):
1123 exprs = chain(self.annotations.values(), self.where.children)
1124 return [
1125 col
1126 for col in self._gen_cols(exprs, include_external=True)
1127 if col.alias in self.external_aliases
1128 ]
1130 def get_group_by_cols(self, wrapper=None):
1131 # If wrapper is referenced by an alias for an explicit GROUP BY through
1132 # values() a reference to this expression and not the self must be
1133 # returned to ensure external column references are not grouped against
1134 # as well.
1135 external_cols = self.get_external_cols()
1136 if any(col.possibly_multivalued for col in external_cols):
1137 return [wrapper or self]
1138 return external_cols
1140 def as_sql(self, compiler, connection):
1141 # Some backends (e.g. Oracle) raise an error when a subquery contains
1142 # unnecessary ORDER BY clause.
1143 if (
1144 self.subquery
1145 and not connection.features.ignores_unnecessary_order_by_in_subqueries
1146 ):
1147 self.clear_ordering(force=False)
1148 for query in self.combined_queries:
1149 query.clear_ordering(force=False)
1150 sql, params = self.get_compiler(connection=connection).as_sql()
1151 if self.subquery:
1152 sql = "(%s)" % sql
1153 return sql, params
1155 def resolve_lookup_value(self, value, can_reuse, allow_joins):
1156 if hasattr(value, "resolve_expression"):
1157 value = value.resolve_expression(
1158 self,
1159 reuse=can_reuse,
1160 allow_joins=allow_joins,
1161 )
1162 elif isinstance(value, (list, tuple)):
1163 # The items of the iterable may be expressions and therefore need
1164 # to be resolved independently.
1165 values = (
1166 self.resolve_lookup_value(sub_value, can_reuse, allow_joins)
1167 for sub_value in value
1168 )
1169 type_ = type(value)
1170 if hasattr(type_, "_make"): # namedtuple
1171 return type_(*values)
1172 return type_(values)
1173 return value
1175 def solve_lookup_type(self, lookup, summarize=False):
1176 """
1177 Solve the lookup type from the lookup (e.g.: 'foobar__id__icontains').
1178 """
1179 lookup_splitted = lookup.split(LOOKUP_SEP)
1180 if self.annotations:
1181 annotation, expression_lookups = refs_expression(
1182 lookup_splitted, self.annotations
1183 )
1184 if annotation:
1185 expression = self.annotations[annotation]
1186 if summarize:
1187 expression = Ref(annotation, expression)
1188 return expression_lookups, (), expression
1189 _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
1190 field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)]
1191 if len(lookup_parts) > 1 and not field_parts:
1192 raise FieldError(
1193 'Invalid lookup "%s" for model %s".'
1194 % (lookup, self.get_meta().model.__name__)
1195 )
1196 return lookup_parts, field_parts, False
1198 def check_query_object_type(self, value, opts, field):
1199 """
1200 Check whether the object passed while querying is of the correct type.
1201 If not, raise a ValueError specifying the wrong object.
1202 """
1203 if hasattr(value, "_meta"):
1204 if not check_rel_lookup_compatibility(value._meta.model, opts, field):
1205 raise ValueError(
1206 'Cannot query "%s": Must be "%s" instance.'
1207 % (value, opts.object_name)
1208 )
1210 def check_related_objects(self, field, value, opts):
1211 """Check the type of object passed to query relations."""
1212 if field.is_relation:
1213 # Check that the field and the queryset use the same model in a
1214 # query like .filter(author=Author.objects.all()). For example, the
1215 # opts would be Author's (from the author field) and value.model
1216 # would be Author.objects.all() queryset's .model (Author also).
1217 # The field is the related field on the lhs side.
1218 if (
1219 isinstance(value, Query)
1220 and not value.has_select_fields
1221 and not check_rel_lookup_compatibility(value.model, opts, field)
1222 ):
1223 raise ValueError(
1224 'Cannot use QuerySet for "%s": Use a QuerySet for "%s".'
1225 % (value.model._meta.object_name, opts.object_name)
1226 )
1227 elif hasattr(value, "_meta"):
1228 self.check_query_object_type(value, opts, field)
1229 elif hasattr(value, "__iter__"):
1230 for v in value:
1231 self.check_query_object_type(v, opts, field)
1233 def check_filterable(self, expression):
1234 """Raise an error if expression cannot be used in a WHERE clause."""
1235 if hasattr(expression, "resolve_expression") and not getattr(
1236 expression, "filterable", True
1237 ):
1238 raise NotSupportedError(
1239 expression.__class__.__name__ + " is disallowed in the filter "
1240 "clause."
1241 )
1242 if hasattr(expression, "get_source_expressions"):
1243 for expr in expression.get_source_expressions():
1244 self.check_filterable(expr)
1246 def build_lookup(self, lookups, lhs, rhs):
1247 """
1248 Try to extract transforms and lookup from given lhs.
1250 The lhs value is something that works like SQLExpression.
1251 The rhs value is what the lookup is going to compare against.
1252 The lookups is a list of names to extract using get_lookup()
1253 and get_transform().
1254 """
1255 # __exact is the default lookup if one isn't given.
1256 *transforms, lookup_name = lookups or ["exact"]
1257 for name in transforms:
1258 lhs = self.try_transform(lhs, name)
1259 # First try get_lookup() so that the lookup takes precedence if the lhs
1260 # supports both transform and lookup for the name.
1261 lookup_class = lhs.get_lookup(lookup_name)
1262 if not lookup_class:
1263 # A lookup wasn't found. Try to interpret the name as a transform
1264 # and do an Exact lookup against it.
1265 lhs = self.try_transform(lhs, lookup_name)
1266 lookup_name = "exact"
1267 lookup_class = lhs.get_lookup(lookup_name)
1268 if not lookup_class:
1269 return
1271 lookup = lookup_class(lhs, rhs)
1272 # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all
1273 # uses of None as a query value unless the lookup supports it.
1274 if lookup.rhs is None and not lookup.can_use_none_as_rhs:
1275 if lookup_name not in ("exact", "iexact"):
1276 raise ValueError("Cannot use None as a query value")
1277 return lhs.get_lookup("isnull")(lhs, True)
1279 # For Oracle '' is equivalent to null. The check must be done at this
1280 # stage because join promotion can't be done in the compiler. Using
1281 # DEFAULT_DB_ALIAS isn't nice but it's the best that can be done here.
1282 # A similar thing is done in is_nullable(), too.
1283 if (
1284 lookup_name == "exact"
1285 and lookup.rhs == ""
1286 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
1287 ):
1288 return lhs.get_lookup("isnull")(lhs, True)
1290 return lookup
1292 def try_transform(self, lhs, name):
1293 """
1294 Helper method for build_lookup(). Try to fetch and initialize
1295 a transform for name parameter from lhs.
1296 """
1297 transform_class = lhs.get_transform(name)
1298 if transform_class:
1299 return transform_class(lhs)
1300 else:
1301 output_field = lhs.output_field.__class__
1302 suggested_lookups = difflib.get_close_matches(
1303 name, output_field.get_lookups()
1304 )
1305 if suggested_lookups:
1306 suggestion = ", perhaps you meant %s?" % " or ".join(suggested_lookups)
1307 else:
1308 suggestion = "."
1309 raise FieldError(
1310 "Unsupported lookup '%s' for %s or join on the field not "
1311 "permitted%s" % (name, output_field.__name__, suggestion)
1312 )
1314 def build_filter(
1315 self,
1316 filter_expr,
1317 branch_negated=False,
1318 current_negated=False,
1319 can_reuse=None,
1320 allow_joins=True,
1321 split_subq=True,
1322 reuse_with_filtered_relation=False,
1323 check_filterable=True,
1324 summarize=False,
1325 ):
1326 """
1327 Build a WhereNode for a single filter clause but don't add it
1328 to this Query. Query.add_q() will then add this filter to the where
1329 Node.
1331 The 'branch_negated' tells us if the current branch contains any
1332 negations. This will be used to determine if subqueries are needed.
1334 The 'current_negated' is used to determine if the current filter is
1335 negated or not and this will be used to determine if IS NULL filtering
1336 is needed.
1338 The difference between current_negated and branch_negated is that
1339 branch_negated is set on first negation, but current_negated is
1340 flipped for each negation.
1342 Note that add_filter will not do any negating itself, that is done
1343 upper in the code by add_q().
1345 The 'can_reuse' is a set of reusable joins for multijoins.
1347 If 'reuse_with_filtered_relation' is True, then only joins in can_reuse
1348 will be reused.
1350 The method will create a filter clause that can be added to the current
1351 query. However, if the filter isn't added to the query then the caller
1352 is responsible for unreffing the joins used.
1353 """
1354 if isinstance(filter_expr, dict):
1355 raise FieldError("Cannot parse keyword query as dict")
1356 if isinstance(filter_expr, Q):
1357 return self._add_q(
1358 filter_expr,
1359 branch_negated=branch_negated,
1360 current_negated=current_negated,
1361 used_aliases=can_reuse,
1362 allow_joins=allow_joins,
1363 split_subq=split_subq,
1364 check_filterable=check_filterable,
1365 summarize=summarize,
1366 )
1367 if hasattr(filter_expr, "resolve_expression"):
1368 if not getattr(filter_expr, "conditional", False):
1369 raise TypeError("Cannot filter against a non-conditional expression.")
1370 condition = filter_expr.resolve_expression(
1371 self, allow_joins=allow_joins, summarize=summarize
1372 )
1373 if not isinstance(condition, Lookup):
1374 condition = self.build_lookup(["exact"], condition, True)
1375 return WhereNode([condition], connector=AND), []
1376 arg, value = filter_expr
1377 if not arg:
1378 raise FieldError("Cannot parse keyword query %r" % arg)
1379 lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize)
1381 if check_filterable:
1382 self.check_filterable(reffed_expression)
1384 if not allow_joins and len(parts) > 1:
1385 raise FieldError("Joined field references are not permitted in this query")
1387 pre_joins = self.alias_refcount.copy()
1388 value = self.resolve_lookup_value(value, can_reuse, allow_joins)
1389 used_joins = {
1390 k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)
1391 }
1393 if check_filterable:
1394 self.check_filterable(value)
1396 if reffed_expression:
1397 condition = self.build_lookup(lookups, reffed_expression, value)
1398 return WhereNode([condition], connector=AND), []
1400 opts = self.get_meta()
1401 alias = self.get_initial_alias()
1402 allow_many = not branch_negated or not split_subq
1404 try:
1405 join_info = self.setup_joins(
1406 parts,
1407 opts,
1408 alias,
1409 can_reuse=can_reuse,
1410 allow_many=allow_many,
1411 reuse_with_filtered_relation=reuse_with_filtered_relation,
1412 )
1414 # Prevent iterator from being consumed by check_related_objects()
1415 if isinstance(value, Iterator):
1416 value = list(value)
1417 self.check_related_objects(join_info.final_field, value, join_info.opts)
1419 # split_exclude() needs to know which joins were generated for the
1420 # lookup parts
1421 self._lookup_joins = join_info.joins
1422 except MultiJoin as e:
1423 return self.split_exclude(filter_expr, can_reuse, e.names_with_path)
1425 # Update used_joins before trimming since they are reused to determine
1426 # which joins could be later promoted to INNER.
1427 used_joins.update(join_info.joins)
1428 targets, alias, join_list = self.trim_joins(
1429 join_info.targets, join_info.joins, join_info.path
1430 )
1431 if can_reuse is not None:
1432 can_reuse.update(join_list)
1434 if join_info.final_field.is_relation:
1435 if len(targets) == 1:
1436 col = self._get_col(targets[0], join_info.final_field, alias)
1437 else:
1438 col = MultiColSource(
1439 alias, targets, join_info.targets, join_info.final_field
1440 )
1441 else:
1442 col = self._get_col(targets[0], join_info.final_field, alias)
1444 condition = self.build_lookup(lookups, col, value)
1445 lookup_type = condition.lookup_name
1446 clause = WhereNode([condition], connector=AND)
1448 require_outer = (
1449 lookup_type == "isnull" and condition.rhs is True and not current_negated
1450 )
1451 if (
1452 current_negated
1453 and (lookup_type != "isnull" or condition.rhs is False)
1454 and condition.rhs is not None
1455 ):
1456 require_outer = True
1457 if lookup_type != "isnull":
1458 # The condition added here will be SQL like this:
1459 # NOT (col IS NOT NULL), where the first NOT is added in
1460 # upper layers of code. The reason for addition is that if col
1461 # is null, then col != someval will result in SQL "unknown"
1462 # which isn't the same as in Python. The Python None handling
1463 # is wanted, and it can be gotten by
1464 # (col IS NULL OR col != someval)
1465 # <=>
1466 # NOT (col IS NOT NULL AND col = someval).
1467 if (
1468 self.is_nullable(targets[0])
1469 or self.alias_map[join_list[-1]].join_type == LOUTER
1470 ):
1471 lookup_class = targets[0].get_lookup("isnull")
1472 col = self._get_col(targets[0], join_info.targets[0], alias)
1473 clause.add(lookup_class(col, False), AND)
1474 # If someval is a nullable column, someval IS NOT NULL is
1475 # added.
1476 if isinstance(value, Col) and self.is_nullable(value.target):
1477 lookup_class = value.target.get_lookup("isnull")
1478 clause.add(lookup_class(value, False), AND)
1479 return clause, used_joins if not require_outer else ()
1481 def add_filter(self, filter_lhs, filter_rhs):
1482 self.add_q(Q((filter_lhs, filter_rhs)))
1484 def add_q(self, q_object):
1485 """
1486 A preprocessor for the internal _add_q(). Responsible for doing final
1487 join promotion.
1488 """
1489 # For join promotion this case is doing an AND for the added q_object
1490 # and existing conditions. So, any existing inner join forces the join
1491 # type to remain inner. Existing outer joins can however be demoted.
1492 # (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if
1493 # rel_a doesn't produce any rows, then the whole condition must fail.
1494 # So, demotion is OK.
1495 existing_inner = {
1496 a for a in self.alias_map if self.alias_map[a].join_type == INNER
1497 }
1498 clause, _ = self._add_q(q_object, self.used_aliases)
1499 if clause:
1500 self.where.add(clause, AND)
1501 self.demote_joins(existing_inner)
1503 def build_where(self, filter_expr):
1504 return self.build_filter(filter_expr, allow_joins=False)[0]
1506 def clear_where(self):
1507 self.where = WhereNode()
1509 def _add_q(
1510 self,
1511 q_object,
1512 used_aliases,
1513 branch_negated=False,
1514 current_negated=False,
1515 allow_joins=True,
1516 split_subq=True,
1517 check_filterable=True,
1518 summarize=False,
1519 ):
1520 """Add a Q-object to the current filter."""
1521 connector = q_object.connector
1522 current_negated ^= q_object.negated
1523 branch_negated = branch_negated or q_object.negated
1524 target_clause = WhereNode(connector=connector, negated=q_object.negated)
1525 joinpromoter = JoinPromoter(
1526 q_object.connector, len(q_object.children), current_negated
1527 )
1528 for child in q_object.children:
1529 child_clause, needed_inner = self.build_filter(
1530 child,
1531 can_reuse=used_aliases,
1532 branch_negated=branch_negated,
1533 current_negated=current_negated,
1534 allow_joins=allow_joins,
1535 split_subq=split_subq,
1536 check_filterable=check_filterable,
1537 summarize=summarize,
1538 )
1539 joinpromoter.add_votes(needed_inner)
1540 if child_clause:
1541 target_clause.add(child_clause, connector)
1542 needed_inner = joinpromoter.update_join_types(self)
1543 return target_clause, needed_inner
1545 def build_filtered_relation_q(
1546 self, q_object, reuse, branch_negated=False, current_negated=False
1547 ):
1548 """Add a FilteredRelation object to the current filter."""
1549 connector = q_object.connector
1550 current_negated ^= q_object.negated
1551 branch_negated = branch_negated or q_object.negated
1552 target_clause = WhereNode(connector=connector, negated=q_object.negated)
1553 for child in q_object.children:
1554 if isinstance(child, Node):
1555 child_clause = self.build_filtered_relation_q(
1556 child,
1557 reuse=reuse,
1558 branch_negated=branch_negated,
1559 current_negated=current_negated,
1560 )
1561 else:
1562 child_clause, _ = self.build_filter(
1563 child,
1564 can_reuse=reuse,
1565 branch_negated=branch_negated,
1566 current_negated=current_negated,
1567 allow_joins=True,
1568 split_subq=False,
1569 reuse_with_filtered_relation=True,
1570 )
1571 target_clause.add(child_clause, connector)
1572 return target_clause
1574 def add_filtered_relation(self, filtered_relation, alias):
1575 filtered_relation.alias = alias
1576 lookups = dict(get_children_from_q(filtered_relation.condition))
1577 relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type(
1578 filtered_relation.relation_name
1579 )
1580 if relation_lookup_parts:
1581 raise ValueError(
1582 "FilteredRelation's relation_name cannot contain lookups "
1583 "(got %r)." % filtered_relation.relation_name
1584 )
1585 for lookup in chain(lookups):
1586 lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup)
1587 shift = 2 if not lookup_parts else 1
1588 lookup_field_path = lookup_field_parts[:-shift]
1589 for idx, lookup_field_part in enumerate(lookup_field_path):
1590 if len(relation_field_parts) > idx:
1591 if relation_field_parts[idx] != lookup_field_part:
1592 raise ValueError(
1593 "FilteredRelation's condition doesn't support "
1594 "relations outside the %r (got %r)."
1595 % (filtered_relation.relation_name, lookup)
1596 )
1597 else:
1598 raise ValueError(
1599 "FilteredRelation's condition doesn't support nested "
1600 "relations deeper than the relation_name (got %r for "
1601 "%r)." % (lookup, filtered_relation.relation_name)
1602 )
1603 self._filtered_relations[filtered_relation.alias] = filtered_relation
1605 def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False):
1606 """
1607 Walk the list of names and turns them into PathInfo tuples. A single
1608 name in 'names' can generate multiple PathInfos (m2m, for example).
1610 'names' is the path of names to travel, 'opts' is the model Options we
1611 start the name resolving from, 'allow_many' is as for setup_joins().
1612 If fail_on_missing is set to True, then a name that can't be resolved
1613 will generate a FieldError.
1615 Return a list of PathInfo tuples. In addition return the final field
1616 (the last used join field) and target (which is a field guaranteed to
1617 contain the same value as the final field). Finally, return those names
1618 that weren't found (which are likely transforms and the final lookup).
1619 """
1620 path, names_with_path = [], []
1621 for pos, name in enumerate(names):
1622 cur_names_with_path = (name, [])
1623 if name == "pk":
1624 name = opts.pk.name
1626 field = None
1627 filtered_relation = None
1628 try:
1629 if opts is None:
1630 raise FieldDoesNotExist
1631 field = opts.get_field(name)
1632 except FieldDoesNotExist:
1633 if name in self.annotation_select:
1634 field = self.annotation_select[name].output_field
1635 elif name in self._filtered_relations and pos == 0:
1636 filtered_relation = self._filtered_relations[name]
1637 if LOOKUP_SEP in filtered_relation.relation_name:
1638 parts = filtered_relation.relation_name.split(LOOKUP_SEP)
1639 filtered_relation_path, field, _, _ = self.names_to_path(
1640 parts,
1641 opts,
1642 allow_many,
1643 fail_on_missing,
1644 )
1645 path.extend(filtered_relation_path[:-1])
1646 else:
1647 field = opts.get_field(filtered_relation.relation_name)
1648 if field is not None:
1649 # Fields that contain one-to-many relations with a generic
1650 # model (like a GenericForeignKey) cannot generate reverse
1651 # relations and therefore cannot be used for reverse querying.
1652 if field.is_relation and not field.related_model:
1653 raise FieldError(
1654 "Field %r does not generate an automatic reverse "
1655 "relation and therefore cannot be used for reverse "
1656 "querying. If it is a GenericForeignKey, consider "
1657 "adding a GenericRelation." % name
1658 )
1659 try:
1660 model = field.model._meta.concrete_model
1661 except AttributeError:
1662 # QuerySet.annotate() may introduce fields that aren't
1663 # attached to a model.
1664 model = None
1665 else:
1666 # We didn't find the current field, so move position back
1667 # one step.
1668 pos -= 1
1669 if pos == -1 or fail_on_missing:
1670 available = sorted(
1671 [
1672 *get_field_names_from_opts(opts),
1673 *self.annotation_select,
1674 *self._filtered_relations,
1675 ]
1676 )
1677 raise FieldError(
1678 "Cannot resolve keyword '%s' into field. "
1679 "Choices are: %s" % (name, ", ".join(available))
1680 )
1681 break
1682 # Check if we need any joins for concrete inheritance cases (the
1683 # field lives in parent, but we are currently in one of its
1684 # children)
1685 if opts is not None and model is not opts.model:
1686 path_to_parent = opts.get_path_to_parent(model)
1687 if path_to_parent:
1688 path.extend(path_to_parent)
1689 cur_names_with_path[1].extend(path_to_parent)
1690 opts = path_to_parent[-1].to_opts
1691 if hasattr(field, "path_infos"):
1692 if filtered_relation:
1693 pathinfos = field.get_path_info(filtered_relation)
1694 else:
1695 pathinfos = field.path_infos
1696 if not allow_many:
1697 for inner_pos, p in enumerate(pathinfos):
1698 if p.m2m:
1699 cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1])
1700 names_with_path.append(cur_names_with_path)
1701 raise MultiJoin(pos + 1, names_with_path)
1702 last = pathinfos[-1]
1703 path.extend(pathinfos)
1704 final_field = last.join_field
1705 opts = last.to_opts
1706 targets = last.target_fields
1707 cur_names_with_path[1].extend(pathinfos)
1708 names_with_path.append(cur_names_with_path)
1709 else:
1710 # Local non-relational field.
1711 final_field = field
1712 targets = (field,)
1713 if fail_on_missing and pos + 1 != len(names):
1714 raise FieldError(
1715 "Cannot resolve keyword %r into field. Join on '%s'"
1716 " not permitted." % (names[pos + 1], name)
1717 )
1718 break
1719 return path, final_field, targets, names[pos + 1 :]
1721 def setup_joins(
1722 self,
1723 names,
1724 opts,
1725 alias,
1726 can_reuse=None,
1727 allow_many=True,
1728 reuse_with_filtered_relation=False,
1729 ):
1730 """
1731 Compute the necessary table joins for the passage through the fields
1732 given in 'names'. 'opts' is the Options class for the current model
1733 (which gives the table we are starting from), 'alias' is the alias for
1734 the table to start the joining from.
1736 The 'can_reuse' defines the reverse foreign key joins we can reuse. It
1737 can be None in which case all joins are reusable or a set of aliases
1738 that can be reused. Note that non-reverse foreign keys are always
1739 reusable when using setup_joins().
1741 The 'reuse_with_filtered_relation' can be used to force 'can_reuse'
1742 parameter and force the relation on the given connections.
1744 If 'allow_many' is False, then any reverse foreign key seen will
1745 generate a MultiJoin exception.
1747 Return the final field involved in the joins, the target field (used
1748 for any 'where' constraint), the final 'opts' value, the joins, the
1749 field path traveled to generate the joins, and a transform function
1750 that takes a field and alias and is equivalent to `field.get_col(alias)`
1751 in the simple case but wraps field transforms if they were included in
1752 names.
1754 The target field is the field containing the concrete value. Final
1755 field can be something different, for example foreign key pointing to
1756 that value. Final field is needed for example in some value
1757 conversions (convert 'obj' in fk__id=obj to pk val using the foreign
1758 key field for example).
1759 """
1760 joins = [alias]
1761 # The transform can't be applied yet, as joins must be trimmed later.
1762 # To avoid making every caller of this method look up transforms
1763 # directly, compute transforms here and create a partial that converts
1764 # fields to the appropriate wrapped version.
1766 def final_transformer(field, alias):
1767 if not self.alias_cols:
1768 alias = None
1769 return field.get_col(alias)
1771 # Try resolving all the names as fields first. If there's an error,
1772 # treat trailing names as lookups until a field can be resolved.
1773 last_field_exception = None
1774 for pivot in range(len(names), 0, -1):
1775 try:
1776 path, final_field, targets, rest = self.names_to_path(
1777 names[:pivot],
1778 opts,
1779 allow_many,
1780 fail_on_missing=True,
1781 )
1782 except FieldError as exc:
1783 if pivot == 1:
1784 # The first item cannot be a lookup, so it's safe
1785 # to raise the field error here.
1786 raise
1787 else:
1788 last_field_exception = exc
1789 else:
1790 # The transforms are the remaining items that couldn't be
1791 # resolved into fields.
1792 transforms = names[pivot:]
1793 break
1794 for name in transforms:
1796 def transform(field, alias, *, name, previous):
1797 try:
1798 wrapped = previous(field, alias)
1799 return self.try_transform(wrapped, name)
1800 except FieldError:
1801 # FieldError is raised if the transform doesn't exist.
1802 if isinstance(final_field, Field) and last_field_exception:
1803 raise last_field_exception
1804 else:
1805 raise
1807 final_transformer = functools.partial(
1808 transform, name=name, previous=final_transformer
1809 )
1810 final_transformer.has_transforms = True
1811 # Then, add the path to the query's joins. Note that we can't trim
1812 # joins at this stage - we will need the information about join type
1813 # of the trimmed joins.
1814 for join in path:
1815 if join.filtered_relation:
1816 filtered_relation = join.filtered_relation.clone()
1817 table_alias = filtered_relation.alias
1818 else:
1819 filtered_relation = None
1820 table_alias = None
1821 opts = join.to_opts
1822 if join.direct:
1823 nullable = self.is_nullable(join.join_field)
1824 else:
1825 nullable = True
1826 connection = self.join_class(
1827 opts.db_table,
1828 alias,
1829 table_alias,
1830 INNER,
1831 join.join_field,
1832 nullable,
1833 filtered_relation=filtered_relation,
1834 )
1835 reuse = can_reuse if join.m2m or reuse_with_filtered_relation else None
1836 alias = self.join(
1837 connection,
1838 reuse=reuse,
1839 reuse_with_filtered_relation=reuse_with_filtered_relation,
1840 )
1841 joins.append(alias)
1842 if filtered_relation:
1843 filtered_relation.path = joins[:]
1844 return JoinInfo(final_field, targets, opts, joins, path, final_transformer)
1846 def trim_joins(self, targets, joins, path):
1847 """
1848 The 'target' parameter is the final field being joined to, 'joins'
1849 is the full list of join aliases. The 'path' contain the PathInfos
1850 used to create the joins.
1852 Return the final target field and table alias and the new active
1853 joins.
1855 Always trim any direct join if the target column is already in the
1856 previous table. Can't trim reverse joins as it's unknown if there's
1857 anything on the other side of the join.
1858 """
1859 joins = joins[:]
1860 for pos, info in enumerate(reversed(path)):
1861 if len(joins) == 1 or not info.direct:
1862 break
1863 if info.filtered_relation:
1864 break
1865 join_targets = {t.column for t in info.join_field.foreign_related_fields}
1866 cur_targets = {t.column for t in targets}
1867 if not cur_targets.issubset(join_targets):
1868 break
1869 targets_dict = {
1870 r[1].column: r[0]
1871 for r in info.join_field.related_fields
1872 if r[1].column in cur_targets
1873 }
1874 targets = tuple(targets_dict[t.column] for t in targets)
1875 self.unref_alias(joins.pop())
1876 return targets, joins[-1], joins
1878 @classmethod
1879 def _gen_cols(cls, exprs, include_external=False, resolve_refs=True):
1880 for expr in exprs:
1881 if isinstance(expr, Col):
1882 yield expr
1883 elif include_external and callable(
1884 getattr(expr, "get_external_cols", None)
1885 ):
1886 yield from expr.get_external_cols()
1887 elif hasattr(expr, "get_source_expressions"):
1888 if not resolve_refs and isinstance(expr, Ref):
1889 continue
1890 yield from cls._gen_cols(
1891 expr.get_source_expressions(),
1892 include_external=include_external,
1893 resolve_refs=resolve_refs,
1894 )
1896 @classmethod
1897 def _gen_col_aliases(cls, exprs):
1898 yield from (expr.alias for expr in cls._gen_cols(exprs))
1900 def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False):
1901 annotation = self.annotations.get(name)
1902 if annotation is not None:
1903 if not allow_joins:
1904 for alias in self._gen_col_aliases([annotation]):
1905 if isinstance(self.alias_map[alias], Join):
1906 raise FieldError(
1907 "Joined field references are not permitted in this query"
1908 )
1909 if summarize:
1910 # Summarize currently means we are doing an aggregate() query
1911 # which is executed as a wrapped subquery if any of the
1912 # aggregate() elements reference an existing annotation. In
1913 # that case we need to return a Ref to the subquery's annotation.
1914 if name not in self.annotation_select:
1915 raise FieldError(
1916 "Cannot aggregate over the '%s' alias. Use annotate() "
1917 "to promote it." % name
1918 )
1919 return Ref(name, self.annotation_select[name])
1920 else:
1921 return annotation
1922 else:
1923 field_list = name.split(LOOKUP_SEP)
1924 annotation = self.annotations.get(field_list[0])
1925 if annotation is not None:
1926 for transform in field_list[1:]:
1927 annotation = self.try_transform(annotation, transform)
1928 return annotation
1929 join_info = self.setup_joins(
1930 field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse
1931 )
1932 targets, final_alias, join_list = self.trim_joins(
1933 join_info.targets, join_info.joins, join_info.path
1934 )
1935 if not allow_joins and len(join_list) > 1:
1936 raise FieldError(
1937 "Joined field references are not permitted in this query"
1938 )
1939 if len(targets) > 1:
1940 raise FieldError(
1941 "Referencing multicolumn fields with F() objects isn't supported"
1942 )
1943 # Verify that the last lookup in name is a field or a transform:
1944 # transform_function() raises FieldError if not.
1945 transform = join_info.transform_function(targets[0], final_alias)
1946 if reuse is not None:
1947 reuse.update(join_list)
1948 return transform
1950 def split_exclude(self, filter_expr, can_reuse, names_with_path):
1951 """
1952 When doing an exclude against any kind of N-to-many relation, we need
1953 to use a subquery. This method constructs the nested query, given the
1954 original exclude filter (filter_expr) and the portion up to the first
1955 N-to-many relation field.
1957 For example, if the origin filter is ~Q(child__name='foo'), filter_expr
1958 is ('child__name', 'foo') and can_reuse is a set of joins usable for
1959 filters in the original query.
1961 We will turn this into equivalent of:
1962 WHERE NOT EXISTS(
1963 SELECT 1
1964 FROM child
1965 WHERE name = 'foo' AND child.parent_id = parent.id
1966 LIMIT 1
1967 )
1968 """
1969 # Generate the inner query.
1970 query = self.__class__(self.model)
1971 query._filtered_relations = self._filtered_relations
1972 filter_lhs, filter_rhs = filter_expr
1973 if isinstance(filter_rhs, OuterRef):
1974 filter_rhs = OuterRef(filter_rhs)
1975 elif isinstance(filter_rhs, F):
1976 filter_rhs = OuterRef(filter_rhs.name)
1977 query.add_filter(filter_lhs, filter_rhs)
1978 query.clear_ordering(force=True)
1979 # Try to have as simple as possible subquery -> trim leading joins from
1980 # the subquery.
1981 trimmed_prefix, contains_louter = query.trim_start(names_with_path)
1983 col = query.select[0]
1984 select_field = col.target
1985 alias = col.alias
1986 if alias in can_reuse:
1987 pk = select_field.model._meta.pk
1988 # Need to add a restriction so that outer query's filters are in effect for
1989 # the subquery, too.
1990 query.bump_prefix(self)
1991 lookup_class = select_field.get_lookup("exact")
1992 # Note that the query.select[0].alias is different from alias
1993 # due to bump_prefix above.
1994 lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias))
1995 query.where.add(lookup, AND)
1996 query.external_aliases[alias] = True
1998 lookup_class = select_field.get_lookup("exact")
1999 lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix))
2000 query.where.add(lookup, AND)
2001 condition, needed_inner = self.build_filter(Exists(query))
2003 if contains_louter:
2004 or_null_condition, _ = self.build_filter(
2005 ("%s__isnull" % trimmed_prefix, True),
2006 current_negated=True,
2007 branch_negated=True,
2008 can_reuse=can_reuse,
2009 )
2010 condition.add(or_null_condition, OR)
2011 # Note that the end result will be:
2012 # (outercol NOT IN innerq AND outercol IS NOT NULL) OR outercol IS NULL.
2013 # This might look crazy but due to how IN works, this seems to be
2014 # correct. If the IS NOT NULL check is removed then outercol NOT
2015 # IN will return UNKNOWN. If the IS NULL check is removed, then if
2016 # outercol IS NULL we will not match the row.
2017 return condition, needed_inner
2019 def set_empty(self):
2020 self.where.add(NothingNode(), AND)
2021 for query in self.combined_queries:
2022 query.set_empty()
2024 def is_empty(self):
2025 return any(isinstance(c, NothingNode) for c in self.where.children)
2027 def set_limits(self, low=None, high=None):
2028 """
2029 Adjust the limits on the rows retrieved. Use low/high to set these,
2030 as it makes it more Pythonic to read and write. When the SQL query is
2031 created, convert them to the appropriate offset and limit values.
2033 Apply any limits passed in here to the existing constraints. Add low
2034 to the current low value and clamp both to any existing high value.
2035 """
2036 if high is not None:
2037 if self.high_mark is not None:
2038 self.high_mark = min(self.high_mark, self.low_mark + high)
2039 else:
2040 self.high_mark = self.low_mark + high
2041 if low is not None:
2042 if self.high_mark is not None:
2043 self.low_mark = min(self.high_mark, self.low_mark + low)
2044 else:
2045 self.low_mark = self.low_mark + low
2047 if self.low_mark == self.high_mark:
2048 self.set_empty()
2050 def clear_limits(self):
2051 """Clear any existing limits."""
2052 self.low_mark, self.high_mark = 0, None
2054 @property
2055 def is_sliced(self):
2056 return self.low_mark != 0 or self.high_mark is not None
2058 def has_limit_one(self):
2059 return self.high_mark is not None and (self.high_mark - self.low_mark) == 1
2061 def can_filter(self):
2062 """
2063 Return True if adding filters to this instance is still possible.
2065 Typically, this means no limits or offsets have been put on the results.
2066 """
2067 return not self.is_sliced
2069 def clear_select_clause(self):
2070 """Remove all fields from SELECT clause."""
2071 self.select = ()
2072 self.default_cols = False
2073 self.select_related = False
2074 self.set_extra_mask(())
2075 self.set_annotation_mask(())
2077 def clear_select_fields(self):
2078 """
2079 Clear the list of fields to select (but not extra_select columns).
2080 Some queryset types completely replace any existing list of select
2081 columns.
2082 """
2083 self.select = ()
2084 self.values_select = ()
2086 def add_select_col(self, col, name):
2087 self.select += (col,)
2088 self.values_select += (name,)
2090 def set_select(self, cols):
2091 self.default_cols = False
2092 self.select = tuple(cols)
2094 def add_distinct_fields(self, *field_names):
2095 """
2096 Add and resolve the given fields to the query's "distinct on" clause.
2097 """
2098 self.distinct_fields = field_names
2099 self.distinct = True
2101 def add_fields(self, field_names, allow_m2m=True):
2102 """
2103 Add the given (model) fields to the select set. Add the field names in
2104 the order specified.
2105 """
2106 alias = self.get_initial_alias()
2107 opts = self.get_meta()
2109 try:
2110 cols = []
2111 for name in field_names:
2112 # Join promotion note - we must not remove any rows here, so
2113 # if there is no existing joins, use outer join.
2114 join_info = self.setup_joins(
2115 name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m
2116 )
2117 targets, final_alias, joins = self.trim_joins(
2118 join_info.targets,
2119 join_info.joins,
2120 join_info.path,
2121 )
2122 for target in targets:
2123 cols.append(join_info.transform_function(target, final_alias))
2124 if cols:
2125 self.set_select(cols)
2126 except MultiJoin:
2127 raise FieldError("Invalid field name: '%s'" % name)
2128 except FieldError:
2129 if LOOKUP_SEP in name:
2130 # For lookups spanning over relationships, show the error
2131 # from the model on which the lookup failed.
2132 raise
2133 elif name in self.annotations:
2134 raise FieldError(
2135 "Cannot select the '%s' alias. Use annotate() to promote "
2136 "it." % name
2137 )
2138 else:
2139 names = sorted(
2140 [
2141 *get_field_names_from_opts(opts),
2142 *self.extra,
2143 *self.annotation_select,
2144 *self._filtered_relations,
2145 ]
2146 )
2147 raise FieldError(
2148 "Cannot resolve keyword %r into field. "
2149 "Choices are: %s" % (name, ", ".join(names))
2150 )
2152 def add_ordering(self, *ordering):
2153 """
2154 Add items from the 'ordering' sequence to the query's "order by"
2155 clause. These items are either field names (not column names) --
2156 possibly with a direction prefix ('-' or '?') -- or OrderBy
2157 expressions.
2159 If 'ordering' is empty, clear all ordering from the query.
2160 """
2161 errors = []
2162 for item in ordering:
2163 if isinstance(item, str):
2164 if item == "?":
2165 continue
2166 if item.startswith("-"):
2167 item = item[1:]
2168 if item in self.annotations:
2169 continue
2170 if self.extra and item in self.extra:
2171 continue
2172 # names_to_path() validates the lookup. A descriptive
2173 # FieldError will be raise if it's not.
2174 self.names_to_path(item.split(LOOKUP_SEP), self.model._meta)
2175 elif not hasattr(item, "resolve_expression"):
2176 errors.append(item)
2177 if getattr(item, "contains_aggregate", False):
2178 raise FieldError(
2179 "Using an aggregate in order_by() without also including "
2180 "it in annotate() is not allowed: %s" % item
2181 )
2182 if errors:
2183 raise FieldError("Invalid order_by arguments: %s" % errors)
2184 if ordering:
2185 self.order_by += ordering
2186 else:
2187 self.default_ordering = False
2189 def clear_ordering(self, force=False, clear_default=True):
2190 """
2191 Remove any ordering settings if the current query allows it without
2192 side effects, set 'force' to True to clear the ordering regardless.
2193 If 'clear_default' is True, there will be no ordering in the resulting
2194 query (not even the model's default).
2195 """
2196 if not force and (
2197 self.is_sliced or self.distinct_fields or self.select_for_update
2198 ):
2199 return
2200 self.order_by = ()
2201 self.extra_order_by = ()
2202 if clear_default:
2203 self.default_ordering = False
2205 def set_group_by(self, allow_aliases=True):
2206 """
2207 Expand the GROUP BY clause required by the query.
2209 This will usually be the set of all non-aggregate fields in the
2210 return data. If the database backend supports grouping by the
2211 primary key, and the query would be equivalent, the optimization
2212 will be made automatically.
2213 """
2214 if allow_aliases and self.values_select:
2215 # If grouping by aliases is allowed assign selected value aliases
2216 # by moving them to annotations.
2217 group_by_annotations = {}
2218 values_select = {}
2219 for alias, expr in zip(self.values_select, self.select):
2220 if isinstance(expr, Col):
2221 values_select[alias] = expr
2222 else:
2223 group_by_annotations[alias] = expr
2224 self.annotations = {**group_by_annotations, **self.annotations}
2225 self.append_annotation_mask(group_by_annotations)
2226 self.select = tuple(values_select.values())
2227 self.values_select = tuple(values_select)
2228 group_by = list(self.select)
2229 for alias, annotation in self.annotation_select.items():
2230 if not (group_by_cols := annotation.get_group_by_cols()):
2231 continue
2232 if allow_aliases and not annotation.contains_aggregate:
2233 group_by.append(Ref(alias, annotation))
2234 else:
2235 group_by.extend(group_by_cols)
2236 self.group_by = tuple(group_by)
2238 def add_select_related(self, fields):
2239 """
2240 Set up the select_related data structure so that we only select
2241 certain related models (as opposed to all models, when
2242 self.select_related=True).
2243 """
2244 if isinstance(self.select_related, bool):
2245 field_dict = {}
2246 else:
2247 field_dict = self.select_related
2248 for field in fields:
2249 d = field_dict
2250 for part in field.split(LOOKUP_SEP):
2251 d = d.setdefault(part, {})
2252 self.select_related = field_dict
2254 def add_extra(self, select, select_params, where, params, tables, order_by):
2255 """
2256 Add data to the various extra_* attributes for user-created additions
2257 to the query.
2258 """
2259 if select:
2260 # We need to pair any placeholder markers in the 'select'
2261 # dictionary with their parameters in 'select_params' so that
2262 # subsequent updates to the select dictionary also adjust the
2263 # parameters appropriately.
2264 select_pairs = {}
2265 if select_params:
2266 param_iter = iter(select_params)
2267 else:
2268 param_iter = iter([])
2269 for name, entry in select.items():
2270 self.check_alias(name)
2271 entry = str(entry)
2272 entry_params = []
2273 pos = entry.find("%s")
2274 while pos != -1:
2275 if pos == 0 or entry[pos - 1] != "%":
2276 entry_params.append(next(param_iter))
2277 pos = entry.find("%s", pos + 2)
2278 select_pairs[name] = (entry, entry_params)
2279 self.extra.update(select_pairs)
2280 if where or params:
2281 self.where.add(ExtraWhere(where, params), AND)
2282 if tables:
2283 self.extra_tables += tuple(tables)
2284 if order_by:
2285 self.extra_order_by = order_by
2287 def clear_deferred_loading(self):
2288 """Remove any fields from the deferred loading set."""
2289 self.deferred_loading = (frozenset(), True)
2291 def add_deferred_loading(self, field_names):
2292 """
2293 Add the given list of model field names to the set of fields to
2294 exclude from loading from the database when automatic column selection
2295 is done. Add the new field names to any existing field names that
2296 are deferred (or removed from any existing field names that are marked
2297 as the only ones for immediate loading).
2298 """
2299 # Fields on related models are stored in the literal double-underscore
2300 # format, so that we can use a set datastructure. We do the foo__bar
2301 # splitting and handling when computing the SQL column names (as part of
2302 # get_columns()).
2303 existing, defer = self.deferred_loading
2304 if defer:
2305 # Add to existing deferred names.
2306 self.deferred_loading = existing.union(field_names), True
2307 else:
2308 # Remove names from the set of any existing "immediate load" names.
2309 if new_existing := existing.difference(field_names):
2310 self.deferred_loading = new_existing, False
2311 else:
2312 self.clear_deferred_loading()
2313 if new_only := set(field_names).difference(existing):
2314 self.deferred_loading = new_only, True
2316 def add_immediate_loading(self, field_names):
2317 """
2318 Add the given list of model field names to the set of fields to
2319 retrieve when the SQL is executed ("immediate loading" fields). The
2320 field names replace any existing immediate loading field names. If
2321 there are field names already specified for deferred loading, remove
2322 those names from the new field_names before storing the new names
2323 for immediate loading. (That is, immediate loading overrides any
2324 existing immediate values, but respects existing deferrals.)
2325 """
2326 existing, defer = self.deferred_loading
2327 field_names = set(field_names)
2328 if "pk" in field_names:
2329 field_names.remove("pk")
2330 field_names.add(self.get_meta().pk.name)
2332 if defer:
2333 # Remove any existing deferred names from the current set before
2334 # setting the new names.
2335 self.deferred_loading = field_names.difference(existing), False
2336 else:
2337 # Replace any existing "immediate load" field names.
2338 self.deferred_loading = frozenset(field_names), False
2340 def set_annotation_mask(self, names):
2341 """Set the mask of annotations that will be returned by the SELECT."""
2342 if names is None:
2343 self.annotation_select_mask = None
2344 else:
2345 self.annotation_select_mask = set(names)
2346 self._annotation_select_cache = None
2348 def append_annotation_mask(self, names):
2349 if self.annotation_select_mask is not None:
2350 self.set_annotation_mask(self.annotation_select_mask.union(names))
2352 def set_extra_mask(self, names):
2353 """
2354 Set the mask of extra select items that will be returned by SELECT.
2355 Don't remove them from the Query since they might be used later.
2356 """
2357 if names is None:
2358 self.extra_select_mask = None
2359 else:
2360 self.extra_select_mask = set(names)
2361 self._extra_select_cache = None
2363 def set_values(self, fields):
2364 self.select_related = False
2365 self.clear_deferred_loading()
2366 self.clear_select_fields()
2367 self.has_select_fields = True
2369 if fields:
2370 field_names = []
2371 extra_names = []
2372 annotation_names = []
2373 if not self.extra and not self.annotations:
2374 # Shortcut - if there are no extra or annotations, then
2375 # the values() clause must be just field names.
2376 field_names = list(fields)
2377 else:
2378 self.default_cols = False
2379 for f in fields:
2380 if f in self.extra_select:
2381 extra_names.append(f)
2382 elif f in self.annotation_select:
2383 annotation_names.append(f)
2384 else:
2385 field_names.append(f)
2386 self.set_extra_mask(extra_names)
2387 self.set_annotation_mask(annotation_names)
2388 selected = frozenset(field_names + extra_names + annotation_names)
2389 else:
2390 field_names = [f.attname for f in self.model._meta.concrete_fields]
2391 selected = frozenset(field_names)
2392 # Selected annotations must be known before setting the GROUP BY
2393 # clause.
2394 if self.group_by is True:
2395 self.add_fields(
2396 (f.attname for f in self.model._meta.concrete_fields), False
2397 )
2398 # Disable GROUP BY aliases to avoid orphaning references to the
2399 # SELECT clause which is about to be cleared.
2400 self.set_group_by(allow_aliases=False)
2401 self.clear_select_fields()
2402 elif self.group_by:
2403 # Resolve GROUP BY annotation references if they are not part of
2404 # the selected fields anymore.
2405 group_by = []
2406 for expr in self.group_by:
2407 if isinstance(expr, Ref) and expr.refs not in selected:
2408 expr = self.annotations[expr.refs]
2409 group_by.append(expr)
2410 self.group_by = tuple(group_by)
2412 self.values_select = tuple(field_names)
2413 self.add_fields(field_names, True)
2415 @property
2416 def annotation_select(self):
2417 """
2418 Return the dictionary of aggregate columns that are not masked and
2419 should be used in the SELECT clause. Cache this result for performance.
2420 """
2421 if self._annotation_select_cache is not None:
2422 return self._annotation_select_cache
2423 elif not self.annotations:
2424 return {}
2425 elif self.annotation_select_mask is not None:
2426 self._annotation_select_cache = {
2427 k: v
2428 for k, v in self.annotations.items()
2429 if k in self.annotation_select_mask
2430 }
2431 return self._annotation_select_cache
2432 else:
2433 return self.annotations
2435 @property
2436 def extra_select(self):
2437 if self._extra_select_cache is not None:
2438 return self._extra_select_cache
2439 if not self.extra:
2440 return {}
2441 elif self.extra_select_mask is not None:
2442 self._extra_select_cache = {
2443 k: v for k, v in self.extra.items() if k in self.extra_select_mask
2444 }
2445 return self._extra_select_cache
2446 else:
2447 return self.extra
2449 def trim_start(self, names_with_path):
2450 """
2451 Trim joins from the start of the join path. The candidates for trim
2452 are the PathInfos in names_with_path structure that are m2m joins.
2454 Also set the select column so the start matches the join.
2456 This method is meant to be used for generating the subquery joins &
2457 cols in split_exclude().
2459 Return a lookup usable for doing outerq.filter(lookup=self) and a
2460 boolean indicating if the joins in the prefix contain a LEFT OUTER join.
2461 _"""
2462 all_paths = []
2463 for _, paths in names_with_path:
2464 all_paths.extend(paths)
2465 contains_louter = False
2466 # Trim and operate only on tables that were generated for
2467 # the lookup part of the query. That is, avoid trimming
2468 # joins generated for F() expressions.
2469 lookup_tables = [
2470 t for t in self.alias_map if t in self._lookup_joins or t == self.base_table
2471 ]
2472 for trimmed_paths, path in enumerate(all_paths):
2473 if path.m2m:
2474 break
2475 if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER:
2476 contains_louter = True
2477 alias = lookup_tables[trimmed_paths]
2478 self.unref_alias(alias)
2479 # The path.join_field is a Rel, lets get the other side's field
2480 join_field = path.join_field.field
2481 # Build the filter prefix.
2482 paths_in_prefix = trimmed_paths
2483 trimmed_prefix = []
2484 for name, path in names_with_path:
2485 if paths_in_prefix - len(path) < 0:
2486 break
2487 trimmed_prefix.append(name)
2488 paths_in_prefix -= len(path)
2489 trimmed_prefix.append(join_field.foreign_related_fields[0].name)
2490 trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
2491 # Lets still see if we can trim the first join from the inner query
2492 # (that is, self). We can't do this for:
2493 # - LEFT JOINs because we would miss those rows that have nothing on
2494 # the outer side,
2495 # - INNER JOINs from filtered relations because we would miss their
2496 # filters.
2497 first_join = self.alias_map[lookup_tables[trimmed_paths + 1]]
2498 if first_join.join_type != LOUTER and not first_join.filtered_relation:
2499 select_fields = [r[0] for r in join_field.related_fields]
2500 select_alias = lookup_tables[trimmed_paths + 1]
2501 self.unref_alias(lookup_tables[trimmed_paths])
2502 extra_restriction = join_field.get_extra_restriction(
2503 None, lookup_tables[trimmed_paths + 1]
2504 )
2505 if extra_restriction:
2506 self.where.add(extra_restriction, AND)
2507 else:
2508 # TODO: It might be possible to trim more joins from the start of the
2509 # inner query if it happens to have a longer join chain containing the
2510 # values in select_fields. Lets punt this one for now.
2511 select_fields = [r[1] for r in join_field.related_fields]
2512 select_alias = lookup_tables[trimmed_paths]
2513 # The found starting point is likely a join_class instead of a
2514 # base_table_class reference. But the first entry in the query's FROM
2515 # clause must not be a JOIN.
2516 for table in self.alias_map:
2517 if self.alias_refcount[table] > 0:
2518 self.alias_map[table] = self.base_table_class(
2519 self.alias_map[table].table_name,
2520 table,
2521 )
2522 break
2523 self.set_select([f.get_col(select_alias) for f in select_fields])
2524 return trimmed_prefix, contains_louter
2526 def is_nullable(self, field):
2527 """
2528 Check if the given field should be treated as nullable.
2530 Some backends treat '' as null and Django treats such fields as
2531 nullable for those backends. In such situations field.null can be
2532 False even if we should treat the field as nullable.
2533 """
2534 # We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have
2535 # (nor should it have) knowledge of which connection is going to be
2536 # used. The proper fix would be to defer all decisions where
2537 # is_nullable() is needed to the compiler stage, but that is not easy
2538 # to do currently.
2539 return field.null or (
2540 field.empty_strings_allowed
2541 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
2542 )
2545def get_order_dir(field, default="ASC"):
2546 """
2547 Return the field name and direction for an order specification. For
2548 example, '-foo' is returned as ('foo', 'DESC').
2550 The 'default' param is used to indicate which way no prefix (or a '+'
2551 prefix) should sort. The '-' prefix always sorts the opposite way.
2552 """
2553 dirn = ORDER_DIR[default]
2554 if field[0] == "-":
2555 return field[1:], dirn[1]
2556 return field, dirn[0]
2559class JoinPromoter:
2560 """
2561 A class to abstract away join promotion problems for complex filter
2562 conditions.
2563 """
2565 def __init__(self, connector, num_children, negated):
2566 self.connector = connector
2567 self.negated = negated
2568 if self.negated:
2569 if connector == AND:
2570 self.effective_connector = OR
2571 else:
2572 self.effective_connector = AND
2573 else:
2574 self.effective_connector = self.connector
2575 self.num_children = num_children
2576 # Maps of table alias to how many times it is seen as required for
2577 # inner and/or outer joins.
2578 self.votes = Counter()
2580 def __repr__(self):
2581 return (
2582 f"{self.__class__.__qualname__}(connector={self.connector!r}, "
2583 f"num_children={self.num_children!r}, negated={self.negated!r})"
2584 )
2586 def add_votes(self, votes):
2587 """
2588 Add single vote per item to self.votes. Parameter can be any
2589 iterable.
2590 """
2591 self.votes.update(votes)
2593 def update_join_types(self, query):
2594 """
2595 Change join types so that the generated query is as efficient as
2596 possible, but still correct. So, change as many joins as possible
2597 to INNER, but don't make OUTER joins INNER if that could remove
2598 results from the query.
2599 """
2600 to_promote = set()
2601 to_demote = set()
2602 # The effective_connector is used so that NOT (a AND b) is treated
2603 # similarly to (a OR b) for join promotion.
2604 for table, votes in self.votes.items():
2605 # We must use outer joins in OR case when the join isn't contained
2606 # in all of the joins. Otherwise the INNER JOIN itself could remove
2607 # valid results. Consider the case where a model with rel_a and
2608 # rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now,
2609 # if rel_a join doesn't produce any results is null (for example
2610 # reverse foreign key or null value in direct foreign key), and
2611 # there is a matching row in rel_b with col=2, then an INNER join
2612 # to rel_a would remove a valid match from the query. So, we need
2613 # to promote any existing INNER to LOUTER (it is possible this
2614 # promotion in turn will be demoted later on).
2615 if self.effective_connector == OR and votes < self.num_children:
2616 to_promote.add(table)
2617 # If connector is AND and there is a filter that can match only
2618 # when there is a joinable row, then use INNER. For example, in
2619 # rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL
2620 # as join output, then the col=1 or col=2 can't match (as
2621 # NULL=anything is always false).
2622 # For the OR case, if all children voted for a join to be inner,
2623 # then we can use INNER for the join. For example:
2624 # (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell)
2625 # then if rel_a doesn't produce any rows, the whole condition
2626 # can't match. Hence we can safely use INNER join.
2627 if self.effective_connector == AND or (
2628 self.effective_connector == OR and votes == self.num_children
2629 ):
2630 to_demote.add(table)
2631 # Finally, what happens in cases where we have:
2632 # (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0
2633 # Now, we first generate the OR clause, and promote joins for it
2634 # in the first if branch above. Both rel_a and rel_b are promoted
2635 # to LOUTER joins. After that we do the AND case. The OR case
2636 # voted no inner joins but the rel_a__col__gte=0 votes inner join
2637 # for rel_a. We demote it back to INNER join (in AND case a single
2638 # vote is enough). The demotion is OK, if rel_a doesn't produce
2639 # rows, then the rel_a__col__gte=0 clause can't be true, and thus
2640 # the whole clause must be false. So, it is safe to use INNER
2641 # join.
2642 # Note that in this example we could just as well have the __gte
2643 # clause and the OR clause swapped. Or we could replace the __gte
2644 # clause with an OR clause containing rel_a__col=1|rel_a__col=2,
2645 # and again we could safely demote to INNER.
2646 query.promote_joins(to_promote)
2647 query.demote_joins(to_demote)
2648 return to_demote