Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/django/db/models/query.py: 16%
1301 statements
« prev ^ index » next coverage.py v7.0.5, created at 2023-01-17 06:13 +0000
1"""
2The main QuerySet implementation. This provides the public API for the ORM.
3"""
5import copy
6import operator
7import warnings
8from itertools import chain, islice
10from asgiref.sync import sync_to_async
12import django
13from django.conf import settings
14from django.core import exceptions
15from django.db import (
16 DJANGO_VERSION_PICKLE_KEY,
17 IntegrityError,
18 NotSupportedError,
19 connections,
20 router,
21 transaction,
22)
23from django.db.models import AutoField, DateField, DateTimeField, Field, sql
24from django.db.models.constants import LOOKUP_SEP, OnConflict
25from django.db.models.deletion import Collector
26from django.db.models.expressions import Case, F, Value, When
27from django.db.models.functions import Cast, Trunc
28from django.db.models.query_utils import FilteredRelation, Q
29from django.db.models.sql.constants import CURSOR, GET_ITERATOR_CHUNK_SIZE
30from django.db.models.utils import (
31 AltersData,
32 create_namedtuple_class,
33 resolve_callables,
34)
35from django.utils import timezone
36from django.utils.deprecation import RemovedInDjango50Warning
37from django.utils.functional import cached_property, partition
39# The maximum number of results to fetch in a get() query.
40MAX_GET_RESULTS = 21
42# The maximum number of items to display in a QuerySet.__repr__
43REPR_OUTPUT_SIZE = 20
46class BaseIterable:
47 def __init__(
48 self, queryset, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE
49 ):
50 self.queryset = queryset
51 self.chunked_fetch = chunked_fetch
52 self.chunk_size = chunk_size
54 async def _async_generator(self):
55 # Generators don't actually start running until the first time you call
56 # next() on them, so make the generator object in the async thread and
57 # then repeatedly dispatch to it in a sync thread.
58 sync_generator = self.__iter__()
60 def next_slice(gen):
61 return list(islice(gen, self.chunk_size))
63 while True:
64 chunk = await sync_to_async(next_slice)(sync_generator)
65 for item in chunk:
66 yield item
67 if len(chunk) < self.chunk_size:
68 break
70 # __aiter__() is a *synchronous* method that has to then return an
71 # *asynchronous* iterator/generator. Thus, nest an async generator inside
72 # it.
73 # This is a generic iterable converter for now, and is going to suffer a
74 # performance penalty on large sets of items due to the cost of crossing
75 # over the sync barrier for each chunk. Custom __aiter__() methods should
76 # be added to each Iterable subclass, but that needs some work in the
77 # Compiler first.
78 def __aiter__(self):
79 return self._async_generator()
82class ModelIterable(BaseIterable):
83 """Iterable that yields a model instance for each row."""
85 def __iter__(self):
86 queryset = self.queryset
87 db = queryset.db
88 compiler = queryset.query.get_compiler(using=db)
89 # Execute the query. This will also fill compiler.select, klass_info,
90 # and annotations.
91 results = compiler.execute_sql(
92 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
93 )
94 select, klass_info, annotation_col_map = (
95 compiler.select,
96 compiler.klass_info,
97 compiler.annotation_col_map,
98 )
99 model_cls = klass_info["model"]
100 select_fields = klass_info["select_fields"]
101 model_fields_start, model_fields_end = select_fields[0], select_fields[-1] + 1
102 init_list = [
103 f[0].target.attname for f in select[model_fields_start:model_fields_end]
104 ]
105 related_populators = get_related_populators(klass_info, select, db)
106 known_related_objects = [
107 (
108 field,
109 related_objs,
110 operator.attrgetter(
111 *[
112 field.attname
113 if from_field == "self"
114 else queryset.model._meta.get_field(from_field).attname
115 for from_field in field.from_fields
116 ]
117 ),
118 )
119 for field, related_objs in queryset._known_related_objects.items()
120 ]
121 for row in compiler.results_iter(results):
122 obj = model_cls.from_db(
123 db, init_list, row[model_fields_start:model_fields_end]
124 )
125 for rel_populator in related_populators:
126 rel_populator.populate(row, obj)
127 if annotation_col_map:
128 for attr_name, col_pos in annotation_col_map.items():
129 setattr(obj, attr_name, row[col_pos])
131 # Add the known related objects to the model.
132 for field, rel_objs, rel_getter in known_related_objects:
133 # Avoid overwriting objects loaded by, e.g., select_related().
134 if field.is_cached(obj):
135 continue
136 rel_obj_id = rel_getter(obj)
137 try:
138 rel_obj = rel_objs[rel_obj_id]
139 except KeyError:
140 pass # May happen in qs1 | qs2 scenarios.
141 else:
142 setattr(obj, field.name, rel_obj)
144 yield obj
147class RawModelIterable(BaseIterable):
148 """
149 Iterable that yields a model instance for each row from a raw queryset.
150 """
152 def __iter__(self):
153 # Cache some things for performance reasons outside the loop.
154 db = self.queryset.db
155 query = self.queryset.query
156 connection = connections[db]
157 compiler = connection.ops.compiler("SQLCompiler")(query, connection, db)
158 query_iterator = iter(query)
160 try:
161 (
162 model_init_names,
163 model_init_pos,
164 annotation_fields,
165 ) = self.queryset.resolve_model_init_order()
166 model_cls = self.queryset.model
167 if model_cls._meta.pk.attname not in model_init_names:
168 raise exceptions.FieldDoesNotExist(
169 "Raw query must include the primary key"
170 )
171 fields = [self.queryset.model_fields.get(c) for c in self.queryset.columns]
172 converters = compiler.get_converters(
173 [f.get_col(f.model._meta.db_table) if f else None for f in fields]
174 )
175 if converters:
176 query_iterator = compiler.apply_converters(query_iterator, converters)
177 for values in query_iterator:
178 # Associate fields to values
179 model_init_values = [values[pos] for pos in model_init_pos]
180 instance = model_cls.from_db(db, model_init_names, model_init_values)
181 if annotation_fields:
182 for column, pos in annotation_fields:
183 setattr(instance, column, values[pos])
184 yield instance
185 finally:
186 # Done iterating the Query. If it has its own cursor, close it.
187 if hasattr(query, "cursor") and query.cursor:
188 query.cursor.close()
191class ValuesIterable(BaseIterable):
192 """
193 Iterable returned by QuerySet.values() that yields a dict for each row.
194 """
196 def __iter__(self):
197 queryset = self.queryset
198 query = queryset.query
199 compiler = query.get_compiler(queryset.db)
201 # extra(select=...) cols are always at the start of the row.
202 names = [
203 *query.extra_select,
204 *query.values_select,
205 *query.annotation_select,
206 ]
207 indexes = range(len(names))
208 for row in compiler.results_iter(
209 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
210 ):
211 yield {names[i]: row[i] for i in indexes}
214class ValuesListIterable(BaseIterable):
215 """
216 Iterable returned by QuerySet.values_list(flat=False) that yields a tuple
217 for each row.
218 """
220 def __iter__(self):
221 queryset = self.queryset
222 query = queryset.query
223 compiler = query.get_compiler(queryset.db)
225 if queryset._fields:
226 # extra(select=...) cols are always at the start of the row.
227 names = [
228 *query.extra_select,
229 *query.values_select,
230 *query.annotation_select,
231 ]
232 fields = [
233 *queryset._fields,
234 *(f for f in query.annotation_select if f not in queryset._fields),
235 ]
236 if fields != names:
237 # Reorder according to fields.
238 index_map = {name: idx for idx, name in enumerate(names)}
239 rowfactory = operator.itemgetter(*[index_map[f] for f in fields])
240 return map(
241 rowfactory,
242 compiler.results_iter(
243 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
244 ),
245 )
246 return compiler.results_iter(
247 tuple_expected=True,
248 chunked_fetch=self.chunked_fetch,
249 chunk_size=self.chunk_size,
250 )
253class NamedValuesListIterable(ValuesListIterable):
254 """
255 Iterable returned by QuerySet.values_list(named=True) that yields a
256 namedtuple for each row.
257 """
259 def __iter__(self):
260 queryset = self.queryset
261 if queryset._fields:
262 names = queryset._fields
263 else:
264 query = queryset.query
265 names = [
266 *query.extra_select,
267 *query.values_select,
268 *query.annotation_select,
269 ]
270 tuple_class = create_namedtuple_class(*names)
271 new = tuple.__new__
272 for row in super().__iter__():
273 yield new(tuple_class, row)
276class FlatValuesListIterable(BaseIterable):
277 """
278 Iterable returned by QuerySet.values_list(flat=True) that yields single
279 values.
280 """
282 def __iter__(self):
283 queryset = self.queryset
284 compiler = queryset.query.get_compiler(queryset.db)
285 for row in compiler.results_iter(
286 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
287 ):
288 yield row[0]
291class QuerySet(AltersData):
292 """Represent a lazy database lookup for a set of objects."""
294 def __init__(self, model=None, query=None, using=None, hints=None):
295 self.model = model
296 self._db = using
297 self._hints = hints or {}
298 self._query = query or sql.Query(self.model)
299 self._result_cache = None
300 self._sticky_filter = False
301 self._for_write = False
302 self._prefetch_related_lookups = ()
303 self._prefetch_done = False
304 self._known_related_objects = {} # {rel_field: {pk: rel_obj}}
305 self._iterable_class = ModelIterable
306 self._fields = None
307 self._defer_next_filter = False
308 self._deferred_filter = None
310 @property
311 def query(self):
312 if self._deferred_filter:
313 negate, args, kwargs = self._deferred_filter
314 self._filter_or_exclude_inplace(negate, args, kwargs)
315 self._deferred_filter = None
316 return self._query
318 @query.setter
319 def query(self, value):
320 if value.values_select:
321 self._iterable_class = ValuesIterable
322 self._query = value
324 def as_manager(cls):
325 # Address the circular dependency between `Queryset` and `Manager`.
326 from django.db.models.manager import Manager
328 manager = Manager.from_queryset(cls)()
329 manager._built_with_as_manager = True
330 return manager
332 as_manager.queryset_only = True
333 as_manager = classmethod(as_manager)
335 ########################
336 # PYTHON MAGIC METHODS #
337 ########################
339 def __deepcopy__(self, memo):
340 """Don't populate the QuerySet's cache."""
341 obj = self.__class__()
342 for k, v in self.__dict__.items():
343 if k == "_result_cache":
344 obj.__dict__[k] = None
345 else:
346 obj.__dict__[k] = copy.deepcopy(v, memo)
347 return obj
349 def __getstate__(self):
350 # Force the cache to be fully populated.
351 self._fetch_all()
352 return {**self.__dict__, DJANGO_VERSION_PICKLE_KEY: django.__version__}
354 def __setstate__(self, state):
355 pickled_version = state.get(DJANGO_VERSION_PICKLE_KEY)
356 if pickled_version:
357 if pickled_version != django.__version__:
358 warnings.warn(
359 "Pickled queryset instance's Django version %s does not "
360 "match the current version %s."
361 % (pickled_version, django.__version__),
362 RuntimeWarning,
363 stacklevel=2,
364 )
365 else:
366 warnings.warn(
367 "Pickled queryset instance's Django version is not specified.",
368 RuntimeWarning,
369 stacklevel=2,
370 )
371 self.__dict__.update(state)
373 def __repr__(self):
374 data = list(self[: REPR_OUTPUT_SIZE + 1])
375 if len(data) > REPR_OUTPUT_SIZE:
376 data[-1] = "...(remaining elements truncated)..."
377 return "<%s %r>" % (self.__class__.__name__, data)
379 def __len__(self):
380 self._fetch_all()
381 return len(self._result_cache)
383 def __iter__(self):
384 """
385 The queryset iterator protocol uses three nested iterators in the
386 default case:
387 1. sql.compiler.execute_sql()
388 - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE)
389 using cursor.fetchmany(). This part is responsible for
390 doing some column masking, and returning the rows in chunks.
391 2. sql.compiler.results_iter()
392 - Returns one row at time. At this point the rows are still just
393 tuples. In some cases the return values are converted to
394 Python values at this location.
395 3. self.iterator()
396 - Responsible for turning the rows into model objects.
397 """
398 self._fetch_all()
399 return iter(self._result_cache)
401 def __aiter__(self):
402 # Remember, __aiter__ itself is synchronous, it's the thing it returns
403 # that is async!
404 async def generator():
405 await sync_to_async(self._fetch_all)()
406 for item in self._result_cache:
407 yield item
409 return generator()
411 def __bool__(self):
412 self._fetch_all()
413 return bool(self._result_cache)
415 def __getitem__(self, k):
416 """Retrieve an item or slice from the set of results."""
417 if not isinstance(k, (int, slice)):
418 raise TypeError(
419 "QuerySet indices must be integers or slices, not %s."
420 % type(k).__name__
421 )
422 if (isinstance(k, int) and k < 0) or (
423 isinstance(k, slice)
424 and (
425 (k.start is not None and k.start < 0)
426 or (k.stop is not None and k.stop < 0)
427 )
428 ):
429 raise ValueError("Negative indexing is not supported.")
431 if self._result_cache is not None:
432 return self._result_cache[k]
434 if isinstance(k, slice):
435 qs = self._chain()
436 if k.start is not None:
437 start = int(k.start)
438 else:
439 start = None
440 if k.stop is not None:
441 stop = int(k.stop)
442 else:
443 stop = None
444 qs.query.set_limits(start, stop)
445 return list(qs)[:: k.step] if k.step else qs
447 qs = self._chain()
448 qs.query.set_limits(k, k + 1)
449 qs._fetch_all()
450 return qs._result_cache[0]
452 def __class_getitem__(cls, *args, **kwargs):
453 return cls
455 def __and__(self, other):
456 self._check_operator_queryset(other, "&")
457 self._merge_sanity_check(other)
458 if isinstance(other, EmptyQuerySet):
459 return other
460 if isinstance(self, EmptyQuerySet):
461 return self
462 combined = self._chain()
463 combined._merge_known_related_objects(other)
464 combined.query.combine(other.query, sql.AND)
465 return combined
467 def __or__(self, other):
468 self._check_operator_queryset(other, "|")
469 self._merge_sanity_check(other)
470 if isinstance(self, EmptyQuerySet):
471 return other
472 if isinstance(other, EmptyQuerySet):
473 return self
474 query = (
475 self
476 if self.query.can_filter()
477 else self.model._base_manager.filter(pk__in=self.values("pk"))
478 )
479 combined = query._chain()
480 combined._merge_known_related_objects(other)
481 if not other.query.can_filter():
482 other = other.model._base_manager.filter(pk__in=other.values("pk"))
483 combined.query.combine(other.query, sql.OR)
484 return combined
486 def __xor__(self, other):
487 self._check_operator_queryset(other, "^")
488 self._merge_sanity_check(other)
489 if isinstance(self, EmptyQuerySet):
490 return other
491 if isinstance(other, EmptyQuerySet):
492 return self
493 query = (
494 self
495 if self.query.can_filter()
496 else self.model._base_manager.filter(pk__in=self.values("pk"))
497 )
498 combined = query._chain()
499 combined._merge_known_related_objects(other)
500 if not other.query.can_filter():
501 other = other.model._base_manager.filter(pk__in=other.values("pk"))
502 combined.query.combine(other.query, sql.XOR)
503 return combined
505 ####################################
506 # METHODS THAT DO DATABASE QUERIES #
507 ####################################
509 def _iterator(self, use_chunked_fetch, chunk_size):
510 iterable = self._iterable_class(
511 self,
512 chunked_fetch=use_chunked_fetch,
513 chunk_size=chunk_size or 2000,
514 )
515 if not self._prefetch_related_lookups or chunk_size is None:
516 yield from iterable
517 return
519 iterator = iter(iterable)
520 while results := list(islice(iterator, chunk_size)):
521 prefetch_related_objects(results, *self._prefetch_related_lookups)
522 yield from results
524 def iterator(self, chunk_size=None):
525 """
526 An iterator over the results from applying this QuerySet to the
527 database. chunk_size must be provided for QuerySets that prefetch
528 related objects. Otherwise, a default chunk_size of 2000 is supplied.
529 """
530 if chunk_size is None:
531 if self._prefetch_related_lookups:
532 # When the deprecation ends, replace with:
533 # raise ValueError(
534 # 'chunk_size must be provided when using '
535 # 'QuerySet.iterator() after prefetch_related().'
536 # )
537 warnings.warn(
538 "Using QuerySet.iterator() after prefetch_related() "
539 "without specifying chunk_size is deprecated.",
540 category=RemovedInDjango50Warning,
541 stacklevel=2,
542 )
543 elif chunk_size <= 0:
544 raise ValueError("Chunk size must be strictly positive.")
545 use_chunked_fetch = not connections[self.db].settings_dict.get(
546 "DISABLE_SERVER_SIDE_CURSORS"
547 )
548 return self._iterator(use_chunked_fetch, chunk_size)
550 async def aiterator(self, chunk_size=2000):
551 """
552 An asynchronous iterator over the results from applying this QuerySet
553 to the database.
554 """
555 if self._prefetch_related_lookups:
556 raise NotSupportedError(
557 "Using QuerySet.aiterator() after prefetch_related() is not supported."
558 )
559 if chunk_size <= 0:
560 raise ValueError("Chunk size must be strictly positive.")
561 use_chunked_fetch = not connections[self.db].settings_dict.get(
562 "DISABLE_SERVER_SIDE_CURSORS"
563 )
564 async for item in self._iterable_class(
565 self, chunked_fetch=use_chunked_fetch, chunk_size=chunk_size
566 ):
567 yield item
569 def aggregate(self, *args, **kwargs):
570 """
571 Return a dictionary containing the calculations (aggregation)
572 over the current queryset.
574 If args is present the expression is passed as a kwarg using
575 the Aggregate object's default alias.
576 """
577 if self.query.distinct_fields:
578 raise NotImplementedError("aggregate() + distinct(fields) not implemented.")
579 self._validate_values_are_expressions(
580 (*args, *kwargs.values()), method_name="aggregate"
581 )
582 for arg in args:
583 # The default_alias property raises TypeError if default_alias
584 # can't be set automatically or AttributeError if it isn't an
585 # attribute.
586 try:
587 arg.default_alias
588 except (AttributeError, TypeError):
589 raise TypeError("Complex aggregates require an alias")
590 kwargs[arg.default_alias] = arg
592 return self.query.chain().get_aggregation(self.db, kwargs)
594 async def aaggregate(self, *args, **kwargs):
595 return await sync_to_async(self.aggregate)(*args, **kwargs)
597 def count(self):
598 """
599 Perform a SELECT COUNT() and return the number of records as an
600 integer.
602 If the QuerySet is already fully cached, return the length of the
603 cached results set to avoid multiple SELECT COUNT(*) calls.
604 """
605 if self._result_cache is not None:
606 return len(self._result_cache)
608 return self.query.get_count(using=self.db)
610 async def acount(self):
611 return await sync_to_async(self.count)()
613 def get(self, *args, **kwargs):
614 """
615 Perform the query and return a single object matching the given
616 keyword arguments.
617 """
618 if self.query.combinator and (args or kwargs):
619 raise NotSupportedError(
620 "Calling QuerySet.get(...) with filters after %s() is not "
621 "supported." % self.query.combinator
622 )
623 clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)
624 if self.query.can_filter() and not self.query.distinct_fields:
625 clone = clone.order_by()
626 limit = None
627 if (
628 not clone.query.select_for_update
629 or connections[clone.db].features.supports_select_for_update_with_limit
630 ):
631 limit = MAX_GET_RESULTS
632 clone.query.set_limits(high=limit)
633 num = len(clone)
634 if num == 1:
635 return clone._result_cache[0]
636 if not num:
637 raise self.model.DoesNotExist(
638 "%s matching query does not exist." % self.model._meta.object_name
639 )
640 raise self.model.MultipleObjectsReturned(
641 "get() returned more than one %s -- it returned %s!"
642 % (
643 self.model._meta.object_name,
644 num if not limit or num < limit else "more than %s" % (limit - 1),
645 )
646 )
648 async def aget(self, *args, **kwargs):
649 return await sync_to_async(self.get)(*args, **kwargs)
651 def create(self, **kwargs):
652 """
653 Create a new object with the given kwargs, saving it to the database
654 and returning the created object.
655 """
656 obj = self.model(**kwargs)
657 self._for_write = True
658 obj.save(force_insert=True, using=self.db)
659 return obj
661 async def acreate(self, **kwargs):
662 return await sync_to_async(self.create)(**kwargs)
664 def _prepare_for_bulk_create(self, objs):
665 for obj in objs:
666 if obj.pk is None:
667 # Populate new PK values.
668 obj.pk = obj._meta.pk.get_pk_value_on_save(obj)
669 obj._prepare_related_fields_for_save(operation_name="bulk_create")
671 def _check_bulk_create_options(
672 self, ignore_conflicts, update_conflicts, update_fields, unique_fields
673 ):
674 if ignore_conflicts and update_conflicts:
675 raise ValueError(
676 "ignore_conflicts and update_conflicts are mutually exclusive."
677 )
678 db_features = connections[self.db].features
679 if ignore_conflicts:
680 if not db_features.supports_ignore_conflicts:
681 raise NotSupportedError(
682 "This database backend does not support ignoring conflicts."
683 )
684 return OnConflict.IGNORE
685 elif update_conflicts:
686 if not db_features.supports_update_conflicts:
687 raise NotSupportedError(
688 "This database backend does not support updating conflicts."
689 )
690 if not update_fields:
691 raise ValueError(
692 "Fields that will be updated when a row insertion fails "
693 "on conflicts must be provided."
694 )
695 if unique_fields and not db_features.supports_update_conflicts_with_target:
696 raise NotSupportedError(
697 "This database backend does not support updating "
698 "conflicts with specifying unique fields that can trigger "
699 "the upsert."
700 )
701 if not unique_fields and db_features.supports_update_conflicts_with_target:
702 raise ValueError(
703 "Unique fields that can trigger the upsert must be provided."
704 )
705 # Updating primary keys and non-concrete fields is forbidden.
706 if any(not f.concrete or f.many_to_many for f in update_fields):
707 raise ValueError(
708 "bulk_create() can only be used with concrete fields in "
709 "update_fields."
710 )
711 if any(f.primary_key for f in update_fields):
712 raise ValueError(
713 "bulk_create() cannot be used with primary keys in "
714 "update_fields."
715 )
716 if unique_fields:
717 if any(not f.concrete or f.many_to_many for f in unique_fields):
718 raise ValueError(
719 "bulk_create() can only be used with concrete fields "
720 "in unique_fields."
721 )
722 return OnConflict.UPDATE
723 return None
725 def bulk_create(
726 self,
727 objs,
728 batch_size=None,
729 ignore_conflicts=False,
730 update_conflicts=False,
731 update_fields=None,
732 unique_fields=None,
733 ):
734 """
735 Insert each of the instances into the database. Do *not* call
736 save() on each of the instances, do not send any pre/post_save
737 signals, and do not set the primary key attribute if it is an
738 autoincrement field (except if features.can_return_rows_from_bulk_insert=True).
739 Multi-table models are not supported.
740 """
741 # When you bulk insert you don't get the primary keys back (if it's an
742 # autoincrement, except if can_return_rows_from_bulk_insert=True), so
743 # you can't insert into the child tables which references this. There
744 # are two workarounds:
745 # 1) This could be implemented if you didn't have an autoincrement pk
746 # 2) You could do it by doing O(n) normal inserts into the parent
747 # tables to get the primary keys back and then doing a single bulk
748 # insert into the childmost table.
749 # We currently set the primary keys on the objects when using
750 # PostgreSQL via the RETURNING ID clause. It should be possible for
751 # Oracle as well, but the semantics for extracting the primary keys is
752 # trickier so it's not done yet.
753 if batch_size is not None and batch_size <= 0:
754 raise ValueError("Batch size must be a positive integer.")
755 # Check that the parents share the same concrete model with the our
756 # model to detect the inheritance pattern ConcreteGrandParent ->
757 # MultiTableParent -> ProxyChild. Simply checking self.model._meta.proxy
758 # would not identify that case as involving multiple tables.
759 for parent in self.model._meta.get_parent_list():
760 if parent._meta.concrete_model is not self.model._meta.concrete_model:
761 raise ValueError("Can't bulk create a multi-table inherited model")
762 if not objs:
763 return objs
764 opts = self.model._meta
765 if unique_fields:
766 # Primary key is allowed in unique_fields.
767 unique_fields = [
768 self.model._meta.get_field(opts.pk.name if name == "pk" else name)
769 for name in unique_fields
770 ]
771 if update_fields:
772 update_fields = [self.model._meta.get_field(name) for name in update_fields]
773 on_conflict = self._check_bulk_create_options(
774 ignore_conflicts,
775 update_conflicts,
776 update_fields,
777 unique_fields,
778 )
779 self._for_write = True
780 fields = opts.concrete_fields
781 objs = list(objs)
782 self._prepare_for_bulk_create(objs)
783 with transaction.atomic(using=self.db, savepoint=False):
784 objs_with_pk, objs_without_pk = partition(lambda o: o.pk is None, objs)
785 if objs_with_pk:
786 returned_columns = self._batched_insert(
787 objs_with_pk,
788 fields,
789 batch_size,
790 on_conflict=on_conflict,
791 update_fields=update_fields,
792 unique_fields=unique_fields,
793 )
794 for obj_with_pk, results in zip(objs_with_pk, returned_columns):
795 for result, field in zip(results, opts.db_returning_fields):
796 if field != opts.pk:
797 setattr(obj_with_pk, field.attname, result)
798 for obj_with_pk in objs_with_pk:
799 obj_with_pk._state.adding = False
800 obj_with_pk._state.db = self.db
801 if objs_without_pk:
802 fields = [f for f in fields if not isinstance(f, AutoField)]
803 returned_columns = self._batched_insert(
804 objs_without_pk,
805 fields,
806 batch_size,
807 on_conflict=on_conflict,
808 update_fields=update_fields,
809 unique_fields=unique_fields,
810 )
811 connection = connections[self.db]
812 if (
813 connection.features.can_return_rows_from_bulk_insert
814 and on_conflict is None
815 ):
816 assert len(returned_columns) == len(objs_without_pk)
817 for obj_without_pk, results in zip(objs_without_pk, returned_columns):
818 for result, field in zip(results, opts.db_returning_fields):
819 setattr(obj_without_pk, field.attname, result)
820 obj_without_pk._state.adding = False
821 obj_without_pk._state.db = self.db
823 return objs
825 async def abulk_create(
826 self,
827 objs,
828 batch_size=None,
829 ignore_conflicts=False,
830 update_conflicts=False,
831 update_fields=None,
832 unique_fields=None,
833 ):
834 return await sync_to_async(self.bulk_create)(
835 objs=objs,
836 batch_size=batch_size,
837 ignore_conflicts=ignore_conflicts,
838 update_conflicts=update_conflicts,
839 update_fields=update_fields,
840 unique_fields=unique_fields,
841 )
843 def bulk_update(self, objs, fields, batch_size=None):
844 """
845 Update the given fields in each of the given objects in the database.
846 """
847 if batch_size is not None and batch_size <= 0:
848 raise ValueError("Batch size must be a positive integer.")
849 if not fields:
850 raise ValueError("Field names must be given to bulk_update().")
851 objs = tuple(objs)
852 if any(obj.pk is None for obj in objs):
853 raise ValueError("All bulk_update() objects must have a primary key set.")
854 fields = [self.model._meta.get_field(name) for name in fields]
855 if any(not f.concrete or f.many_to_many for f in fields):
856 raise ValueError("bulk_update() can only be used with concrete fields.")
857 if any(f.primary_key for f in fields):
858 raise ValueError("bulk_update() cannot be used with primary key fields.")
859 if not objs:
860 return 0
861 for obj in objs:
862 obj._prepare_related_fields_for_save(
863 operation_name="bulk_update", fields=fields
864 )
865 # PK is used twice in the resulting update query, once in the filter
866 # and once in the WHEN. Each field will also have one CAST.
867 self._for_write = True
868 connection = connections[self.db]
869 max_batch_size = connection.ops.bulk_batch_size(["pk", "pk"] + fields, objs)
870 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
871 requires_casting = connection.features.requires_casted_case_in_updates
872 batches = (objs[i : i + batch_size] for i in range(0, len(objs), batch_size))
873 updates = []
874 for batch_objs in batches:
875 update_kwargs = {}
876 for field in fields:
877 when_statements = []
878 for obj in batch_objs:
879 attr = getattr(obj, field.attname)
880 if not hasattr(attr, "resolve_expression"):
881 attr = Value(attr, output_field=field)
882 when_statements.append(When(pk=obj.pk, then=attr))
883 case_statement = Case(*when_statements, output_field=field)
884 if requires_casting:
885 case_statement = Cast(case_statement, output_field=field)
886 update_kwargs[field.attname] = case_statement
887 updates.append(([obj.pk for obj in batch_objs], update_kwargs))
888 rows_updated = 0
889 queryset = self.using(self.db)
890 with transaction.atomic(using=self.db, savepoint=False):
891 for pks, update_kwargs in updates:
892 rows_updated += queryset.filter(pk__in=pks).update(**update_kwargs)
893 return rows_updated
895 bulk_update.alters_data = True
897 async def abulk_update(self, objs, fields, batch_size=None):
898 return await sync_to_async(self.bulk_update)(
899 objs=objs,
900 fields=fields,
901 batch_size=batch_size,
902 )
904 abulk_update.alters_data = True
906 def get_or_create(self, defaults=None, **kwargs):
907 """
908 Look up an object with the given kwargs, creating one if necessary.
909 Return a tuple of (object, created), where created is a boolean
910 specifying whether an object was created.
911 """
912 # The get() needs to be targeted at the write database in order
913 # to avoid potential transaction consistency problems.
914 self._for_write = True
915 try:
916 return self.get(**kwargs), False
917 except self.model.DoesNotExist:
918 params = self._extract_model_params(defaults, **kwargs)
919 # Try to create an object using passed params.
920 try:
921 with transaction.atomic(using=self.db):
922 params = dict(resolve_callables(params))
923 return self.create(**params), True
924 except IntegrityError:
925 try:
926 return self.get(**kwargs), False
927 except self.model.DoesNotExist:
928 pass
929 raise
931 async def aget_or_create(self, defaults=None, **kwargs):
932 return await sync_to_async(self.get_or_create)(
933 defaults=defaults,
934 **kwargs,
935 )
937 def update_or_create(self, defaults=None, **kwargs):
938 """
939 Look up an object with the given kwargs, updating one with defaults
940 if it exists, otherwise create a new one.
941 Return a tuple (object, created), where created is a boolean
942 specifying whether an object was created.
943 """
944 defaults = defaults or {}
945 self._for_write = True
946 with transaction.atomic(using=self.db):
947 # Lock the row so that a concurrent update is blocked until
948 # update_or_create() has performed its save.
949 obj, created = self.select_for_update().get_or_create(defaults, **kwargs)
950 if created:
951 return obj, created
952 for k, v in resolve_callables(defaults):
953 setattr(obj, k, v)
955 update_fields = set(defaults)
956 concrete_field_names = self.model._meta._non_pk_concrete_field_names
957 # update_fields does not support non-concrete fields.
958 if concrete_field_names.issuperset(update_fields):
959 # Add fields which are set on pre_save(), e.g. auto_now fields.
960 # This is to maintain backward compatibility as these fields
961 # are not updated unless explicitly specified in the
962 # update_fields list.
963 for field in self.model._meta.local_concrete_fields:
964 if not (
965 field.primary_key or field.__class__.pre_save is Field.pre_save
966 ):
967 update_fields.add(field.name)
968 if field.name != field.attname:
969 update_fields.add(field.attname)
970 obj.save(using=self.db, update_fields=update_fields)
971 else:
972 obj.save(using=self.db)
973 return obj, False
975 async def aupdate_or_create(self, defaults=None, **kwargs):
976 return await sync_to_async(self.update_or_create)(
977 defaults=defaults,
978 **kwargs,
979 )
981 def _extract_model_params(self, defaults, **kwargs):
982 """
983 Prepare `params` for creating a model instance based on the given
984 kwargs; for use by get_or_create().
985 """
986 defaults = defaults or {}
987 params = {k: v for k, v in kwargs.items() if LOOKUP_SEP not in k}
988 params.update(defaults)
989 property_names = self.model._meta._property_names
990 invalid_params = []
991 for param in params:
992 try:
993 self.model._meta.get_field(param)
994 except exceptions.FieldDoesNotExist:
995 # It's okay to use a model's property if it has a setter.
996 if not (param in property_names and getattr(self.model, param).fset):
997 invalid_params.append(param)
998 if invalid_params:
999 raise exceptions.FieldError(
1000 "Invalid field name(s) for model %s: '%s'."
1001 % (
1002 self.model._meta.object_name,
1003 "', '".join(sorted(invalid_params)),
1004 )
1005 )
1006 return params
1008 def _earliest(self, *fields):
1009 """
1010 Return the earliest object according to fields (if given) or by the
1011 model's Meta.get_latest_by.
1012 """
1013 if fields:
1014 order_by = fields
1015 else:
1016 order_by = getattr(self.model._meta, "get_latest_by")
1017 if order_by and not isinstance(order_by, (tuple, list)):
1018 order_by = (order_by,)
1019 if order_by is None:
1020 raise ValueError(
1021 "earliest() and latest() require either fields as positional "
1022 "arguments or 'get_latest_by' in the model's Meta."
1023 )
1024 obj = self._chain()
1025 obj.query.set_limits(high=1)
1026 obj.query.clear_ordering(force=True)
1027 obj.query.add_ordering(*order_by)
1028 return obj.get()
1030 def earliest(self, *fields):
1031 if self.query.is_sliced:
1032 raise TypeError("Cannot change a query once a slice has been taken.")
1033 return self._earliest(*fields)
1035 async def aearliest(self, *fields):
1036 return await sync_to_async(self.earliest)(*fields)
1038 def latest(self, *fields):
1039 """
1040 Return the latest object according to fields (if given) or by the
1041 model's Meta.get_latest_by.
1042 """
1043 if self.query.is_sliced:
1044 raise TypeError("Cannot change a query once a slice has been taken.")
1045 return self.reverse()._earliest(*fields)
1047 async def alatest(self, *fields):
1048 return await sync_to_async(self.latest)(*fields)
1050 def first(self):
1051 """Return the first object of a query or None if no match is found."""
1052 if self.ordered:
1053 queryset = self
1054 else:
1055 self._check_ordering_first_last_queryset_aggregation(method="first")
1056 queryset = self.order_by("pk")
1057 for obj in queryset[:1]:
1058 return obj
1060 async def afirst(self):
1061 return await sync_to_async(self.first)()
1063 def last(self):
1064 """Return the last object of a query or None if no match is found."""
1065 if self.ordered:
1066 queryset = self.reverse()
1067 else:
1068 self._check_ordering_first_last_queryset_aggregation(method="last")
1069 queryset = self.order_by("-pk")
1070 for obj in queryset[:1]:
1071 return obj
1073 async def alast(self):
1074 return await sync_to_async(self.last)()
1076 def in_bulk(self, id_list=None, *, field_name="pk"):
1077 """
1078 Return a dictionary mapping each of the given IDs to the object with
1079 that ID. If `id_list` isn't provided, evaluate the entire QuerySet.
1080 """
1081 if self.query.is_sliced:
1082 raise TypeError("Cannot use 'limit' or 'offset' with in_bulk().")
1083 opts = self.model._meta
1084 unique_fields = [
1085 constraint.fields[0]
1086 for constraint in opts.total_unique_constraints
1087 if len(constraint.fields) == 1
1088 ]
1089 if (
1090 field_name != "pk"
1091 and not opts.get_field(field_name).unique
1092 and field_name not in unique_fields
1093 and self.query.distinct_fields != (field_name,)
1094 ):
1095 raise ValueError(
1096 "in_bulk()'s field_name must be a unique field but %r isn't."
1097 % field_name
1098 )
1099 if id_list is not None:
1100 if not id_list:
1101 return {}
1102 filter_key = "{}__in".format(field_name)
1103 batch_size = connections[self.db].features.max_query_params
1104 id_list = tuple(id_list)
1105 # If the database has a limit on the number of query parameters
1106 # (e.g. SQLite), retrieve objects in batches if necessary.
1107 if batch_size and batch_size < len(id_list):
1108 qs = ()
1109 for offset in range(0, len(id_list), batch_size):
1110 batch = id_list[offset : offset + batch_size]
1111 qs += tuple(self.filter(**{filter_key: batch}).order_by())
1112 else:
1113 qs = self.filter(**{filter_key: id_list}).order_by()
1114 else:
1115 qs = self._chain()
1116 return {getattr(obj, field_name): obj for obj in qs}
1118 async def ain_bulk(self, id_list=None, *, field_name="pk"):
1119 return await sync_to_async(self.in_bulk)(
1120 id_list=id_list,
1121 field_name=field_name,
1122 )
1124 def delete(self):
1125 """Delete the records in the current QuerySet."""
1126 self._not_support_combined_queries("delete")
1127 if self.query.is_sliced:
1128 raise TypeError("Cannot use 'limit' or 'offset' with delete().")
1129 if self.query.distinct or self.query.distinct_fields:
1130 raise TypeError("Cannot call delete() after .distinct().")
1131 if self._fields is not None:
1132 raise TypeError("Cannot call delete() after .values() or .values_list()")
1134 del_query = self._chain()
1136 # The delete is actually 2 queries - one to find related objects,
1137 # and one to delete. Make sure that the discovery of related
1138 # objects is performed on the same database as the deletion.
1139 del_query._for_write = True
1141 # Disable non-supported fields.
1142 del_query.query.select_for_update = False
1143 del_query.query.select_related = False
1144 del_query.query.clear_ordering(force=True)
1146 collector = Collector(using=del_query.db, origin=self)
1147 collector.collect(del_query)
1148 deleted, _rows_count = collector.delete()
1150 # Clear the result cache, in case this QuerySet gets reused.
1151 self._result_cache = None
1152 return deleted, _rows_count
1154 delete.alters_data = True
1155 delete.queryset_only = True
1157 async def adelete(self):
1158 return await sync_to_async(self.delete)()
1160 adelete.alters_data = True
1161 adelete.queryset_only = True
1163 def _raw_delete(self, using):
1164 """
1165 Delete objects found from the given queryset in single direct SQL
1166 query. No signals are sent and there is no protection for cascades.
1167 """
1168 query = self.query.clone()
1169 query.__class__ = sql.DeleteQuery
1170 cursor = query.get_compiler(using).execute_sql(CURSOR)
1171 if cursor:
1172 with cursor:
1173 return cursor.rowcount
1174 return 0
1176 _raw_delete.alters_data = True
1178 def update(self, **kwargs):
1179 """
1180 Update all elements in the current QuerySet, setting all the given
1181 fields to the appropriate values.
1182 """
1183 self._not_support_combined_queries("update")
1184 if self.query.is_sliced:
1185 raise TypeError("Cannot update a query once a slice has been taken.")
1186 self._for_write = True
1187 query = self.query.chain(sql.UpdateQuery)
1188 query.add_update_values(kwargs)
1190 # Inline annotations in order_by(), if possible.
1191 new_order_by = []
1192 for col in query.order_by:
1193 if annotation := query.annotations.get(col):
1194 if getattr(annotation, "contains_aggregate", False):
1195 raise exceptions.FieldError(
1196 f"Cannot update when ordering by an aggregate: {annotation}"
1197 )
1198 new_order_by.append(annotation)
1199 else:
1200 new_order_by.append(col)
1201 query.order_by = tuple(new_order_by)
1203 # Clear any annotations so that they won't be present in subqueries.
1204 query.annotations = {}
1205 with transaction.mark_for_rollback_on_error(using=self.db):
1206 rows = query.get_compiler(self.db).execute_sql(CURSOR)
1207 self._result_cache = None
1208 return rows
1210 update.alters_data = True
1212 async def aupdate(self, **kwargs):
1213 return await sync_to_async(self.update)(**kwargs)
1215 aupdate.alters_data = True
1217 def _update(self, values):
1218 """
1219 A version of update() that accepts field objects instead of field names.
1220 Used primarily for model saving and not intended for use by general
1221 code (it requires too much poking around at model internals to be
1222 useful at that level).
1223 """
1224 if self.query.is_sliced:
1225 raise TypeError("Cannot update a query once a slice has been taken.")
1226 query = self.query.chain(sql.UpdateQuery)
1227 query.add_update_fields(values)
1228 # Clear any annotations so that they won't be present in subqueries.
1229 query.annotations = {}
1230 self._result_cache = None
1231 return query.get_compiler(self.db).execute_sql(CURSOR)
1233 _update.alters_data = True
1234 _update.queryset_only = False
1236 def exists(self):
1237 """
1238 Return True if the QuerySet would have any results, False otherwise.
1239 """
1240 if self._result_cache is None:
1241 return self.query.has_results(using=self.db)
1242 return bool(self._result_cache)
1244 async def aexists(self):
1245 return await sync_to_async(self.exists)()
1247 def contains(self, obj):
1248 """
1249 Return True if the QuerySet contains the provided obj,
1250 False otherwise.
1251 """
1252 self._not_support_combined_queries("contains")
1253 if self._fields is not None:
1254 raise TypeError(
1255 "Cannot call QuerySet.contains() after .values() or .values_list()."
1256 )
1257 try:
1258 if obj._meta.concrete_model != self.model._meta.concrete_model:
1259 return False
1260 except AttributeError:
1261 raise TypeError("'obj' must be a model instance.")
1262 if obj.pk is None:
1263 raise ValueError("QuerySet.contains() cannot be used on unsaved objects.")
1264 if self._result_cache is not None:
1265 return obj in self._result_cache
1266 return self.filter(pk=obj.pk).exists()
1268 async def acontains(self, obj):
1269 return await sync_to_async(self.contains)(obj=obj)
1271 def _prefetch_related_objects(self):
1272 # This method can only be called once the result cache has been filled.
1273 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
1274 self._prefetch_done = True
1276 def explain(self, *, format=None, **options):
1277 """
1278 Runs an EXPLAIN on the SQL query this QuerySet would perform, and
1279 returns the results.
1280 """
1281 return self.query.explain(using=self.db, format=format, **options)
1283 async def aexplain(self, *, format=None, **options):
1284 return await sync_to_async(self.explain)(format=format, **options)
1286 ##################################################
1287 # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
1288 ##################################################
1290 def raw(self, raw_query, params=(), translations=None, using=None):
1291 if using is None:
1292 using = self.db
1293 qs = RawQuerySet(
1294 raw_query,
1295 model=self.model,
1296 params=params,
1297 translations=translations,
1298 using=using,
1299 )
1300 qs._prefetch_related_lookups = self._prefetch_related_lookups[:]
1301 return qs
1303 def _values(self, *fields, **expressions):
1304 clone = self._chain()
1305 if expressions:
1306 clone = clone.annotate(**expressions)
1307 clone._fields = fields
1308 clone.query.set_values(fields)
1309 return clone
1311 def values(self, *fields, **expressions):
1312 fields += tuple(expressions)
1313 clone = self._values(*fields, **expressions)
1314 clone._iterable_class = ValuesIterable
1315 return clone
1317 def values_list(self, *fields, flat=False, named=False):
1318 if flat and named:
1319 raise TypeError("'flat' and 'named' can't be used together.")
1320 if flat and len(fields) > 1:
1321 raise TypeError(
1322 "'flat' is not valid when values_list is called with more than one "
1323 "field."
1324 )
1326 field_names = {f for f in fields if not hasattr(f, "resolve_expression")}
1327 _fields = []
1328 expressions = {}
1329 counter = 1
1330 for field in fields:
1331 if hasattr(field, "resolve_expression"):
1332 field_id_prefix = getattr(
1333 field, "default_alias", field.__class__.__name__.lower()
1334 )
1335 while True:
1336 field_id = field_id_prefix + str(counter)
1337 counter += 1
1338 if field_id not in field_names:
1339 break
1340 expressions[field_id] = field
1341 _fields.append(field_id)
1342 else:
1343 _fields.append(field)
1345 clone = self._values(*_fields, **expressions)
1346 clone._iterable_class = (
1347 NamedValuesListIterable
1348 if named
1349 else FlatValuesListIterable
1350 if flat
1351 else ValuesListIterable
1352 )
1353 return clone
1355 def dates(self, field_name, kind, order="ASC"):
1356 """
1357 Return a list of date objects representing all available dates for
1358 the given field_name, scoped to 'kind'.
1359 """
1360 if kind not in ("year", "month", "week", "day"):
1361 raise ValueError("'kind' must be one of 'year', 'month', 'week', or 'day'.")
1362 if order not in ("ASC", "DESC"):
1363 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1364 return (
1365 self.annotate(
1366 datefield=Trunc(field_name, kind, output_field=DateField()),
1367 plain_field=F(field_name),
1368 )
1369 .values_list("datefield", flat=True)
1370 .distinct()
1371 .filter(plain_field__isnull=False)
1372 .order_by(("-" if order == "DESC" else "") + "datefield")
1373 )
1375 # RemovedInDjango50Warning: when the deprecation ends, remove is_dst
1376 # argument.
1377 def datetimes(
1378 self, field_name, kind, order="ASC", tzinfo=None, is_dst=timezone.NOT_PASSED
1379 ):
1380 """
1381 Return a list of datetime objects representing all available
1382 datetimes for the given field_name, scoped to 'kind'.
1383 """
1384 if kind not in ("year", "month", "week", "day", "hour", "minute", "second"):
1385 raise ValueError(
1386 "'kind' must be one of 'year', 'month', 'week', 'day', "
1387 "'hour', 'minute', or 'second'."
1388 )
1389 if order not in ("ASC", "DESC"):
1390 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1391 if settings.USE_TZ:
1392 if tzinfo is None:
1393 tzinfo = timezone.get_current_timezone()
1394 else:
1395 tzinfo = None
1396 return (
1397 self.annotate(
1398 datetimefield=Trunc(
1399 field_name,
1400 kind,
1401 output_field=DateTimeField(),
1402 tzinfo=tzinfo,
1403 is_dst=is_dst,
1404 ),
1405 plain_field=F(field_name),
1406 )
1407 .values_list("datetimefield", flat=True)
1408 .distinct()
1409 .filter(plain_field__isnull=False)
1410 .order_by(("-" if order == "DESC" else "") + "datetimefield")
1411 )
1413 def none(self):
1414 """Return an empty QuerySet."""
1415 clone = self._chain()
1416 clone.query.set_empty()
1417 return clone
1419 ##################################################################
1420 # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET #
1421 ##################################################################
1423 def all(self):
1424 """
1425 Return a new QuerySet that is a copy of the current one. This allows a
1426 QuerySet to proxy for a model manager in some cases.
1427 """
1428 return self._chain()
1430 def filter(self, *args, **kwargs):
1431 """
1432 Return a new QuerySet instance with the args ANDed to the existing
1433 set.
1434 """
1435 self._not_support_combined_queries("filter")
1436 return self._filter_or_exclude(False, args, kwargs)
1438 def exclude(self, *args, **kwargs):
1439 """
1440 Return a new QuerySet instance with NOT (args) ANDed to the existing
1441 set.
1442 """
1443 self._not_support_combined_queries("exclude")
1444 return self._filter_or_exclude(True, args, kwargs)
1446 def _filter_or_exclude(self, negate, args, kwargs):
1447 if (args or kwargs) and self.query.is_sliced:
1448 raise TypeError("Cannot filter a query once a slice has been taken.")
1449 clone = self._chain()
1450 if self._defer_next_filter:
1451 self._defer_next_filter = False
1452 clone._deferred_filter = negate, args, kwargs
1453 else:
1454 clone._filter_or_exclude_inplace(negate, args, kwargs)
1455 return clone
1457 def _filter_or_exclude_inplace(self, negate, args, kwargs):
1458 if negate:
1459 self._query.add_q(~Q(*args, **kwargs))
1460 else:
1461 self._query.add_q(Q(*args, **kwargs))
1463 def complex_filter(self, filter_obj):
1464 """
1465 Return a new QuerySet instance with filter_obj added to the filters.
1467 filter_obj can be a Q object or a dictionary of keyword lookup
1468 arguments.
1470 This exists to support framework features such as 'limit_choices_to',
1471 and usually it will be more natural to use other methods.
1472 """
1473 if isinstance(filter_obj, Q):
1474 clone = self._chain()
1475 clone.query.add_q(filter_obj)
1476 return clone
1477 else:
1478 return self._filter_or_exclude(False, args=(), kwargs=filter_obj)
1480 def _combinator_query(self, combinator, *other_qs, all=False):
1481 # Clone the query to inherit the select list and everything
1482 clone = self._chain()
1483 # Clear limits and ordering so they can be reapplied
1484 clone.query.clear_ordering(force=True)
1485 clone.query.clear_limits()
1486 clone.query.combined_queries = (self.query,) + tuple(
1487 qs.query for qs in other_qs
1488 )
1489 clone.query.combinator = combinator
1490 clone.query.combinator_all = all
1491 return clone
1493 def union(self, *other_qs, all=False):
1494 # If the query is an EmptyQuerySet, combine all nonempty querysets.
1495 if isinstance(self, EmptyQuerySet):
1496 qs = [q for q in other_qs if not isinstance(q, EmptyQuerySet)]
1497 if not qs:
1498 return self
1499 if len(qs) == 1:
1500 return qs[0]
1501 return qs[0]._combinator_query("union", *qs[1:], all=all)
1502 return self._combinator_query("union", *other_qs, all=all)
1504 def intersection(self, *other_qs):
1505 # If any query is an EmptyQuerySet, return it.
1506 if isinstance(self, EmptyQuerySet):
1507 return self
1508 for other in other_qs:
1509 if isinstance(other, EmptyQuerySet):
1510 return other
1511 return self._combinator_query("intersection", *other_qs)
1513 def difference(self, *other_qs):
1514 # If the query is an EmptyQuerySet, return it.
1515 if isinstance(self, EmptyQuerySet):
1516 return self
1517 return self._combinator_query("difference", *other_qs)
1519 def select_for_update(self, nowait=False, skip_locked=False, of=(), no_key=False):
1520 """
1521 Return a new QuerySet instance that will select objects with a
1522 FOR UPDATE lock.
1523 """
1524 if nowait and skip_locked:
1525 raise ValueError("The nowait option cannot be used with skip_locked.")
1526 obj = self._chain()
1527 obj._for_write = True
1528 obj.query.select_for_update = True
1529 obj.query.select_for_update_nowait = nowait
1530 obj.query.select_for_update_skip_locked = skip_locked
1531 obj.query.select_for_update_of = of
1532 obj.query.select_for_no_key_update = no_key
1533 return obj
1535 def select_related(self, *fields):
1536 """
1537 Return a new QuerySet instance that will select related objects.
1539 If fields are specified, they must be ForeignKey fields and only those
1540 related objects are included in the selection.
1542 If select_related(None) is called, clear the list.
1543 """
1544 self._not_support_combined_queries("select_related")
1545 if self._fields is not None:
1546 raise TypeError(
1547 "Cannot call select_related() after .values() or .values_list()"
1548 )
1550 obj = self._chain()
1551 if fields == (None,):
1552 obj.query.select_related = False
1553 elif fields:
1554 obj.query.add_select_related(fields)
1555 else:
1556 obj.query.select_related = True
1557 return obj
1559 def prefetch_related(self, *lookups):
1560 """
1561 Return a new QuerySet instance that will prefetch the specified
1562 Many-To-One and Many-To-Many related objects when the QuerySet is
1563 evaluated.
1565 When prefetch_related() is called more than once, append to the list of
1566 prefetch lookups. If prefetch_related(None) is called, clear the list.
1567 """
1568 self._not_support_combined_queries("prefetch_related")
1569 clone = self._chain()
1570 if lookups == (None,):
1571 clone._prefetch_related_lookups = ()
1572 else:
1573 for lookup in lookups:
1574 if isinstance(lookup, Prefetch):
1575 lookup = lookup.prefetch_to
1576 lookup = lookup.split(LOOKUP_SEP, 1)[0]
1577 if lookup in self.query._filtered_relations:
1578 raise ValueError(
1579 "prefetch_related() is not supported with FilteredRelation."
1580 )
1581 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
1582 return clone
1584 def annotate(self, *args, **kwargs):
1585 """
1586 Return a query set in which the returned objects have been annotated
1587 with extra data or aggregations.
1588 """
1589 self._not_support_combined_queries("annotate")
1590 return self._annotate(args, kwargs, select=True)
1592 def alias(self, *args, **kwargs):
1593 """
1594 Return a query set with added aliases for extra data or aggregations.
1595 """
1596 self._not_support_combined_queries("alias")
1597 return self._annotate(args, kwargs, select=False)
1599 def _annotate(self, args, kwargs, select=True):
1600 self._validate_values_are_expressions(
1601 args + tuple(kwargs.values()), method_name="annotate"
1602 )
1603 annotations = {}
1604 for arg in args:
1605 # The default_alias property may raise a TypeError.
1606 try:
1607 if arg.default_alias in kwargs:
1608 raise ValueError(
1609 "The named annotation '%s' conflicts with the "
1610 "default name for another annotation." % arg.default_alias
1611 )
1612 except TypeError:
1613 raise TypeError("Complex annotations require an alias")
1614 annotations[arg.default_alias] = arg
1615 annotations.update(kwargs)
1617 clone = self._chain()
1618 names = self._fields
1619 if names is None:
1620 names = set(
1621 chain.from_iterable(
1622 (field.name, field.attname)
1623 if hasattr(field, "attname")
1624 else (field.name,)
1625 for field in self.model._meta.get_fields()
1626 )
1627 )
1629 for alias, annotation in annotations.items():
1630 if alias in names:
1631 raise ValueError(
1632 "The annotation '%s' conflicts with a field on "
1633 "the model." % alias
1634 )
1635 if isinstance(annotation, FilteredRelation):
1636 clone.query.add_filtered_relation(annotation, alias)
1637 else:
1638 clone.query.add_annotation(
1639 annotation,
1640 alias,
1641 select=select,
1642 )
1643 for alias, annotation in clone.query.annotations.items():
1644 if alias in annotations and annotation.contains_aggregate:
1645 if clone._fields is None:
1646 clone.query.group_by = True
1647 else:
1648 clone.query.set_group_by()
1649 break
1651 return clone
1653 def order_by(self, *field_names):
1654 """Return a new QuerySet instance with the ordering changed."""
1655 if self.query.is_sliced:
1656 raise TypeError("Cannot reorder a query once a slice has been taken.")
1657 obj = self._chain()
1658 obj.query.clear_ordering(force=True, clear_default=False)
1659 obj.query.add_ordering(*field_names)
1660 return obj
1662 def distinct(self, *field_names):
1663 """
1664 Return a new QuerySet instance that will select only distinct results.
1665 """
1666 self._not_support_combined_queries("distinct")
1667 if self.query.is_sliced:
1668 raise TypeError(
1669 "Cannot create distinct fields once a slice has been taken."
1670 )
1671 obj = self._chain()
1672 obj.query.add_distinct_fields(*field_names)
1673 return obj
1675 def extra(
1676 self,
1677 select=None,
1678 where=None,
1679 params=None,
1680 tables=None,
1681 order_by=None,
1682 select_params=None,
1683 ):
1684 """Add extra SQL fragments to the query."""
1685 self._not_support_combined_queries("extra")
1686 if self.query.is_sliced:
1687 raise TypeError("Cannot change a query once a slice has been taken.")
1688 clone = self._chain()
1689 clone.query.add_extra(select, select_params, where, params, tables, order_by)
1690 return clone
1692 def reverse(self):
1693 """Reverse the ordering of the QuerySet."""
1694 if self.query.is_sliced:
1695 raise TypeError("Cannot reverse a query once a slice has been taken.")
1696 clone = self._chain()
1697 clone.query.standard_ordering = not clone.query.standard_ordering
1698 return clone
1700 def defer(self, *fields):
1701 """
1702 Defer the loading of data for certain fields until they are accessed.
1703 Add the set of deferred fields to any existing set of deferred fields.
1704 The only exception to this is if None is passed in as the only
1705 parameter, in which case removal all deferrals.
1706 """
1707 self._not_support_combined_queries("defer")
1708 if self._fields is not None:
1709 raise TypeError("Cannot call defer() after .values() or .values_list()")
1710 clone = self._chain()
1711 if fields == (None,):
1712 clone.query.clear_deferred_loading()
1713 else:
1714 clone.query.add_deferred_loading(fields)
1715 return clone
1717 def only(self, *fields):
1718 """
1719 Essentially, the opposite of defer(). Only the fields passed into this
1720 method and that are not already specified as deferred are loaded
1721 immediately when the queryset is evaluated.
1722 """
1723 self._not_support_combined_queries("only")
1724 if self._fields is not None:
1725 raise TypeError("Cannot call only() after .values() or .values_list()")
1726 if fields == (None,):
1727 # Can only pass None to defer(), not only(), as the rest option.
1728 # That won't stop people trying to do this, so let's be explicit.
1729 raise TypeError("Cannot pass None as an argument to only().")
1730 for field in fields:
1731 field = field.split(LOOKUP_SEP, 1)[0]
1732 if field in self.query._filtered_relations:
1733 raise ValueError("only() is not supported with FilteredRelation.")
1734 clone = self._chain()
1735 clone.query.add_immediate_loading(fields)
1736 return clone
1738 def using(self, alias):
1739 """Select which database this QuerySet should execute against."""
1740 clone = self._chain()
1741 clone._db = alias
1742 return clone
1744 ###################################
1745 # PUBLIC INTROSPECTION ATTRIBUTES #
1746 ###################################
1748 @property
1749 def ordered(self):
1750 """
1751 Return True if the QuerySet is ordered -- i.e. has an order_by()
1752 clause or a default ordering on the model (or is empty).
1753 """
1754 if isinstance(self, EmptyQuerySet):
1755 return True
1756 if self.query.extra_order_by or self.query.order_by:
1757 return True
1758 elif (
1759 self.query.default_ordering
1760 and self.query.get_meta().ordering
1761 and
1762 # A default ordering doesn't affect GROUP BY queries.
1763 not self.query.group_by
1764 ):
1765 return True
1766 else:
1767 return False
1769 @property
1770 def db(self):
1771 """Return the database used if this query is executed now."""
1772 if self._for_write:
1773 return self._db or router.db_for_write(self.model, **self._hints)
1774 return self._db or router.db_for_read(self.model, **self._hints)
1776 ###################
1777 # PRIVATE METHODS #
1778 ###################
1780 def _insert(
1781 self,
1782 objs,
1783 fields,
1784 returning_fields=None,
1785 raw=False,
1786 using=None,
1787 on_conflict=None,
1788 update_fields=None,
1789 unique_fields=None,
1790 ):
1791 """
1792 Insert a new record for the given model. This provides an interface to
1793 the InsertQuery class and is how Model.save() is implemented.
1794 """
1795 self._for_write = True
1796 if using is None:
1797 using = self.db
1798 query = sql.InsertQuery(
1799 self.model,
1800 on_conflict=on_conflict,
1801 update_fields=update_fields,
1802 unique_fields=unique_fields,
1803 )
1804 query.insert_values(fields, objs, raw=raw)
1805 return query.get_compiler(using=using).execute_sql(returning_fields)
1807 _insert.alters_data = True
1808 _insert.queryset_only = False
1810 def _batched_insert(
1811 self,
1812 objs,
1813 fields,
1814 batch_size,
1815 on_conflict=None,
1816 update_fields=None,
1817 unique_fields=None,
1818 ):
1819 """
1820 Helper method for bulk_create() to insert objs one batch at a time.
1821 """
1822 connection = connections[self.db]
1823 ops = connection.ops
1824 max_batch_size = max(ops.bulk_batch_size(fields, objs), 1)
1825 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
1826 inserted_rows = []
1827 bulk_return = connection.features.can_return_rows_from_bulk_insert
1828 for item in [objs[i : i + batch_size] for i in range(0, len(objs), batch_size)]:
1829 if bulk_return and on_conflict is None:
1830 inserted_rows.extend(
1831 self._insert(
1832 item,
1833 fields=fields,
1834 using=self.db,
1835 returning_fields=self.model._meta.db_returning_fields,
1836 )
1837 )
1838 else:
1839 self._insert(
1840 item,
1841 fields=fields,
1842 using=self.db,
1843 on_conflict=on_conflict,
1844 update_fields=update_fields,
1845 unique_fields=unique_fields,
1846 )
1847 return inserted_rows
1849 def _chain(self):
1850 """
1851 Return a copy of the current QuerySet that's ready for another
1852 operation.
1853 """
1854 obj = self._clone()
1855 if obj._sticky_filter:
1856 obj.query.filter_is_sticky = True
1857 obj._sticky_filter = False
1858 return obj
1860 def _clone(self):
1861 """
1862 Return a copy of the current QuerySet. A lightweight alternative
1863 to deepcopy().
1864 """
1865 c = self.__class__(
1866 model=self.model,
1867 query=self.query.chain(),
1868 using=self._db,
1869 hints=self._hints,
1870 )
1871 c._sticky_filter = self._sticky_filter
1872 c._for_write = self._for_write
1873 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
1874 c._known_related_objects = self._known_related_objects
1875 c._iterable_class = self._iterable_class
1876 c._fields = self._fields
1877 return c
1879 def _fetch_all(self):
1880 if self._result_cache is None:
1881 self._result_cache = list(self._iterable_class(self))
1882 if self._prefetch_related_lookups and not self._prefetch_done:
1883 self._prefetch_related_objects()
1885 def _next_is_sticky(self):
1886 """
1887 Indicate that the next filter call and the one following that should
1888 be treated as a single filter. This is only important when it comes to
1889 determining when to reuse tables for many-to-many filters. Required so
1890 that we can filter naturally on the results of related managers.
1892 This doesn't return a clone of the current QuerySet (it returns
1893 "self"). The method is only used internally and should be immediately
1894 followed by a filter() that does create a clone.
1895 """
1896 self._sticky_filter = True
1897 return self
1899 def _merge_sanity_check(self, other):
1900 """Check that two QuerySet classes may be merged."""
1901 if self._fields is not None and (
1902 set(self.query.values_select) != set(other.query.values_select)
1903 or set(self.query.extra_select) != set(other.query.extra_select)
1904 or set(self.query.annotation_select) != set(other.query.annotation_select)
1905 ):
1906 raise TypeError(
1907 "Merging '%s' classes must involve the same values in each case."
1908 % self.__class__.__name__
1909 )
1911 def _merge_known_related_objects(self, other):
1912 """
1913 Keep track of all known related objects from either QuerySet instance.
1914 """
1915 for field, objects in other._known_related_objects.items():
1916 self._known_related_objects.setdefault(field, {}).update(objects)
1918 def resolve_expression(self, *args, **kwargs):
1919 if self._fields and len(self._fields) > 1:
1920 # values() queryset can only be used as nested queries
1921 # if they are set up to select only a single field.
1922 raise TypeError("Cannot use multi-field values as a filter value.")
1923 query = self.query.resolve_expression(*args, **kwargs)
1924 query._db = self._db
1925 return query
1927 resolve_expression.queryset_only = True
1929 def _add_hints(self, **hints):
1930 """
1931 Update hinting information for use by routers. Add new key/values or
1932 overwrite existing key/values.
1933 """
1934 self._hints.update(hints)
1936 def _has_filters(self):
1937 """
1938 Check if this QuerySet has any filtering going on. This isn't
1939 equivalent with checking if all objects are present in results, for
1940 example, qs[1:]._has_filters() -> False.
1941 """
1942 return self.query.has_filters()
1944 @staticmethod
1945 def _validate_values_are_expressions(values, method_name):
1946 invalid_args = sorted(
1947 str(arg) for arg in values if not hasattr(arg, "resolve_expression")
1948 )
1949 if invalid_args:
1950 raise TypeError(
1951 "QuerySet.%s() received non-expression(s): %s."
1952 % (
1953 method_name,
1954 ", ".join(invalid_args),
1955 )
1956 )
1958 def _not_support_combined_queries(self, operation_name):
1959 if self.query.combinator:
1960 raise NotSupportedError(
1961 "Calling QuerySet.%s() after %s() is not supported."
1962 % (operation_name, self.query.combinator)
1963 )
1965 def _check_operator_queryset(self, other, operator_):
1966 if self.query.combinator or other.query.combinator:
1967 raise TypeError(f"Cannot use {operator_} operator with combined queryset.")
1969 def _check_ordering_first_last_queryset_aggregation(self, method):
1970 if isinstance(self.query.group_by, tuple) and not any(
1971 col.output_field is self.model._meta.pk for col in self.query.group_by
1972 ):
1973 raise TypeError(
1974 f"Cannot use QuerySet.{method}() on an unordered queryset performing "
1975 f"aggregation. Add an ordering with order_by()."
1976 )
1979class InstanceCheckMeta(type):
1980 def __instancecheck__(self, instance):
1981 return isinstance(instance, QuerySet) and instance.query.is_empty()
1984class EmptyQuerySet(metaclass=InstanceCheckMeta):
1985 """
1986 Marker class to checking if a queryset is empty by .none():
1987 isinstance(qs.none(), EmptyQuerySet) -> True
1988 """
1990 def __init__(self, *args, **kwargs):
1991 raise TypeError("EmptyQuerySet can't be instantiated")
1994class RawQuerySet:
1995 """
1996 Provide an iterator which converts the results of raw SQL queries into
1997 annotated model instances.
1998 """
2000 def __init__(
2001 self,
2002 raw_query,
2003 model=None,
2004 query=None,
2005 params=(),
2006 translations=None,
2007 using=None,
2008 hints=None,
2009 ):
2010 self.raw_query = raw_query
2011 self.model = model
2012 self._db = using
2013 self._hints = hints or {}
2014 self.query = query or sql.RawQuery(sql=raw_query, using=self.db, params=params)
2015 self.params = params
2016 self.translations = translations or {}
2017 self._result_cache = None
2018 self._prefetch_related_lookups = ()
2019 self._prefetch_done = False
2021 def resolve_model_init_order(self):
2022 """Resolve the init field names and value positions."""
2023 converter = connections[self.db].introspection.identifier_converter
2024 model_init_fields = [
2025 f for f in self.model._meta.fields if converter(f.column) in self.columns
2026 ]
2027 annotation_fields = [
2028 (column, pos)
2029 for pos, column in enumerate(self.columns)
2030 if column not in self.model_fields
2031 ]
2032 model_init_order = [
2033 self.columns.index(converter(f.column)) for f in model_init_fields
2034 ]
2035 model_init_names = [f.attname for f in model_init_fields]
2036 return model_init_names, model_init_order, annotation_fields
2038 def prefetch_related(self, *lookups):
2039 """Same as QuerySet.prefetch_related()"""
2040 clone = self._clone()
2041 if lookups == (None,):
2042 clone._prefetch_related_lookups = ()
2043 else:
2044 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
2045 return clone
2047 def _prefetch_related_objects(self):
2048 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
2049 self._prefetch_done = True
2051 def _clone(self):
2052 """Same as QuerySet._clone()"""
2053 c = self.__class__(
2054 self.raw_query,
2055 model=self.model,
2056 query=self.query,
2057 params=self.params,
2058 translations=self.translations,
2059 using=self._db,
2060 hints=self._hints,
2061 )
2062 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
2063 return c
2065 def _fetch_all(self):
2066 if self._result_cache is None:
2067 self._result_cache = list(self.iterator())
2068 if self._prefetch_related_lookups and not self._prefetch_done:
2069 self._prefetch_related_objects()
2071 def __len__(self):
2072 self._fetch_all()
2073 return len(self._result_cache)
2075 def __bool__(self):
2076 self._fetch_all()
2077 return bool(self._result_cache)
2079 def __iter__(self):
2080 self._fetch_all()
2081 return iter(self._result_cache)
2083 def __aiter__(self):
2084 # Remember, __aiter__ itself is synchronous, it's the thing it returns
2085 # that is async!
2086 async def generator():
2087 await sync_to_async(self._fetch_all)()
2088 for item in self._result_cache:
2089 yield item
2091 return generator()
2093 def iterator(self):
2094 yield from RawModelIterable(self)
2096 def __repr__(self):
2097 return "<%s: %s>" % (self.__class__.__name__, self.query)
2099 def __getitem__(self, k):
2100 return list(self)[k]
2102 @property
2103 def db(self):
2104 """Return the database used if this query is executed now."""
2105 return self._db or router.db_for_read(self.model, **self._hints)
2107 def using(self, alias):
2108 """Select the database this RawQuerySet should execute against."""
2109 return RawQuerySet(
2110 self.raw_query,
2111 model=self.model,
2112 query=self.query.chain(using=alias),
2113 params=self.params,
2114 translations=self.translations,
2115 using=alias,
2116 )
2118 @cached_property
2119 def columns(self):
2120 """
2121 A list of model field names in the order they'll appear in the
2122 query results.
2123 """
2124 columns = self.query.get_columns()
2125 # Adjust any column names which don't match field names
2126 for (query_name, model_name) in self.translations.items():
2127 # Ignore translations for nonexistent column names
2128 try:
2129 index = columns.index(query_name)
2130 except ValueError:
2131 pass
2132 else:
2133 columns[index] = model_name
2134 return columns
2136 @cached_property
2137 def model_fields(self):
2138 """A dict mapping column names to model field names."""
2139 converter = connections[self.db].introspection.identifier_converter
2140 model_fields = {}
2141 for field in self.model._meta.fields:
2142 name, column = field.get_attname_column()
2143 model_fields[converter(column)] = field
2144 return model_fields
2147class Prefetch:
2148 def __init__(self, lookup, queryset=None, to_attr=None):
2149 # `prefetch_through` is the path we traverse to perform the prefetch.
2150 self.prefetch_through = lookup
2151 # `prefetch_to` is the path to the attribute that stores the result.
2152 self.prefetch_to = lookup
2153 if queryset is not None and (
2154 isinstance(queryset, RawQuerySet)
2155 or (
2156 hasattr(queryset, "_iterable_class")
2157 and not issubclass(queryset._iterable_class, ModelIterable)
2158 )
2159 ):
2160 raise ValueError(
2161 "Prefetch querysets cannot use raw(), values(), and values_list()."
2162 )
2163 if to_attr:
2164 self.prefetch_to = LOOKUP_SEP.join(
2165 lookup.split(LOOKUP_SEP)[:-1] + [to_attr]
2166 )
2168 self.queryset = queryset
2169 self.to_attr = to_attr
2171 def __getstate__(self):
2172 obj_dict = self.__dict__.copy()
2173 if self.queryset is not None:
2174 queryset = self.queryset._chain()
2175 # Prevent the QuerySet from being evaluated
2176 queryset._result_cache = []
2177 queryset._prefetch_done = True
2178 obj_dict["queryset"] = queryset
2179 return obj_dict
2181 def add_prefix(self, prefix):
2182 self.prefetch_through = prefix + LOOKUP_SEP + self.prefetch_through
2183 self.prefetch_to = prefix + LOOKUP_SEP + self.prefetch_to
2185 def get_current_prefetch_to(self, level):
2186 return LOOKUP_SEP.join(self.prefetch_to.split(LOOKUP_SEP)[: level + 1])
2188 def get_current_to_attr(self, level):
2189 parts = self.prefetch_to.split(LOOKUP_SEP)
2190 to_attr = parts[level]
2191 as_attr = self.to_attr and level == len(parts) - 1
2192 return to_attr, as_attr
2194 def get_current_queryset(self, level):
2195 if self.get_current_prefetch_to(level) == self.prefetch_to:
2196 return self.queryset
2197 return None
2199 def __eq__(self, other):
2200 if not isinstance(other, Prefetch):
2201 return NotImplemented
2202 return self.prefetch_to == other.prefetch_to
2204 def __hash__(self):
2205 return hash((self.__class__, self.prefetch_to))
2208def normalize_prefetch_lookups(lookups, prefix=None):
2209 """Normalize lookups into Prefetch objects."""
2210 ret = []
2211 for lookup in lookups:
2212 if not isinstance(lookup, Prefetch):
2213 lookup = Prefetch(lookup)
2214 if prefix:
2215 lookup.add_prefix(prefix)
2216 ret.append(lookup)
2217 return ret
2220def prefetch_related_objects(model_instances, *related_lookups):
2221 """
2222 Populate prefetched object caches for a list of model instances based on
2223 the lookups/Prefetch instances given.
2224 """
2225 if not model_instances:
2226 return # nothing to do
2228 # We need to be able to dynamically add to the list of prefetch_related
2229 # lookups that we look up (see below). So we need some book keeping to
2230 # ensure we don't do duplicate work.
2231 done_queries = {} # dictionary of things like 'foo__bar': [results]
2233 auto_lookups = set() # we add to this as we go through.
2234 followed_descriptors = set() # recursion protection
2236 all_lookups = normalize_prefetch_lookups(reversed(related_lookups))
2237 while all_lookups:
2238 lookup = all_lookups.pop()
2239 if lookup.prefetch_to in done_queries:
2240 if lookup.queryset is not None:
2241 raise ValueError(
2242 "'%s' lookup was already seen with a different queryset. "
2243 "You may need to adjust the ordering of your lookups."
2244 % lookup.prefetch_to
2245 )
2247 continue
2249 # Top level, the list of objects to decorate is the result cache
2250 # from the primary QuerySet. It won't be for deeper levels.
2251 obj_list = model_instances
2253 through_attrs = lookup.prefetch_through.split(LOOKUP_SEP)
2254 for level, through_attr in enumerate(through_attrs):
2255 # Prepare main instances
2256 if not obj_list:
2257 break
2259 prefetch_to = lookup.get_current_prefetch_to(level)
2260 if prefetch_to in done_queries:
2261 # Skip any prefetching, and any object preparation
2262 obj_list = done_queries[prefetch_to]
2263 continue
2265 # Prepare objects:
2266 good_objects = True
2267 for obj in obj_list:
2268 # Since prefetching can re-use instances, it is possible to have
2269 # the same instance multiple times in obj_list, so obj might
2270 # already be prepared.
2271 if not hasattr(obj, "_prefetched_objects_cache"):
2272 try:
2273 obj._prefetched_objects_cache = {}
2274 except (AttributeError, TypeError):
2275 # Must be an immutable object from
2276 # values_list(flat=True), for example (TypeError) or
2277 # a QuerySet subclass that isn't returning Model
2278 # instances (AttributeError), either in Django or a 3rd
2279 # party. prefetch_related() doesn't make sense, so quit.
2280 good_objects = False
2281 break
2282 if not good_objects:
2283 break
2285 # Descend down tree
2287 # We assume that objects retrieved are homogeneous (which is the premise
2288 # of prefetch_related), so what applies to first object applies to all.
2289 first_obj = obj_list[0]
2290 to_attr = lookup.get_current_to_attr(level)[0]
2291 prefetcher, descriptor, attr_found, is_fetched = get_prefetcher(
2292 first_obj, through_attr, to_attr
2293 )
2295 if not attr_found:
2296 raise AttributeError(
2297 "Cannot find '%s' on %s object, '%s' is an invalid "
2298 "parameter to prefetch_related()"
2299 % (
2300 through_attr,
2301 first_obj.__class__.__name__,
2302 lookup.prefetch_through,
2303 )
2304 )
2306 if level == len(through_attrs) - 1 and prefetcher is None:
2307 # Last one, this *must* resolve to something that supports
2308 # prefetching, otherwise there is no point adding it and the
2309 # developer asking for it has made a mistake.
2310 raise ValueError(
2311 "'%s' does not resolve to an item that supports "
2312 "prefetching - this is an invalid parameter to "
2313 "prefetch_related()." % lookup.prefetch_through
2314 )
2316 obj_to_fetch = None
2317 if prefetcher is not None:
2318 obj_to_fetch = [obj for obj in obj_list if not is_fetched(obj)]
2320 if obj_to_fetch:
2321 obj_list, additional_lookups = prefetch_one_level(
2322 obj_to_fetch,
2323 prefetcher,
2324 lookup,
2325 level,
2326 )
2327 # We need to ensure we don't keep adding lookups from the
2328 # same relationships to stop infinite recursion. So, if we
2329 # are already on an automatically added lookup, don't add
2330 # the new lookups from relationships we've seen already.
2331 if not (
2332 prefetch_to in done_queries
2333 and lookup in auto_lookups
2334 and descriptor in followed_descriptors
2335 ):
2336 done_queries[prefetch_to] = obj_list
2337 new_lookups = normalize_prefetch_lookups(
2338 reversed(additional_lookups), prefetch_to
2339 )
2340 auto_lookups.update(new_lookups)
2341 all_lookups.extend(new_lookups)
2342 followed_descriptors.add(descriptor)
2343 else:
2344 # Either a singly related object that has already been fetched
2345 # (e.g. via select_related), or hopefully some other property
2346 # that doesn't support prefetching but needs to be traversed.
2348 # We replace the current list of parent objects with the list
2349 # of related objects, filtering out empty or missing values so
2350 # that we can continue with nullable or reverse relations.
2351 new_obj_list = []
2352 for obj in obj_list:
2353 if through_attr in getattr(obj, "_prefetched_objects_cache", ()):
2354 # If related objects have been prefetched, use the
2355 # cache rather than the object's through_attr.
2356 new_obj = list(obj._prefetched_objects_cache.get(through_attr))
2357 else:
2358 try:
2359 new_obj = getattr(obj, through_attr)
2360 except exceptions.ObjectDoesNotExist:
2361 continue
2362 if new_obj is None:
2363 continue
2364 # We special-case `list` rather than something more generic
2365 # like `Iterable` because we don't want to accidentally match
2366 # user models that define __iter__.
2367 if isinstance(new_obj, list):
2368 new_obj_list.extend(new_obj)
2369 else:
2370 new_obj_list.append(new_obj)
2371 obj_list = new_obj_list
2374def get_prefetcher(instance, through_attr, to_attr):
2375 """
2376 For the attribute 'through_attr' on the given instance, find
2377 an object that has a get_prefetch_queryset().
2378 Return a 4 tuple containing:
2379 (the object with get_prefetch_queryset (or None),
2380 the descriptor object representing this relationship (or None),
2381 a boolean that is False if the attribute was not found at all,
2382 a function that takes an instance and returns a boolean that is True if
2383 the attribute has already been fetched for that instance)
2384 """
2386 def has_to_attr_attribute(instance):
2387 return hasattr(instance, to_attr)
2389 prefetcher = None
2390 is_fetched = has_to_attr_attribute
2392 # For singly related objects, we have to avoid getting the attribute
2393 # from the object, as this will trigger the query. So we first try
2394 # on the class, in order to get the descriptor object.
2395 rel_obj_descriptor = getattr(instance.__class__, through_attr, None)
2396 if rel_obj_descriptor is None:
2397 attr_found = hasattr(instance, through_attr)
2398 else:
2399 attr_found = True
2400 if rel_obj_descriptor:
2401 # singly related object, descriptor object has the
2402 # get_prefetch_queryset() method.
2403 if hasattr(rel_obj_descriptor, "get_prefetch_queryset"):
2404 prefetcher = rel_obj_descriptor
2405 is_fetched = rel_obj_descriptor.is_cached
2406 else:
2407 # descriptor doesn't support prefetching, so we go ahead and get
2408 # the attribute on the instance rather than the class to
2409 # support many related managers
2410 rel_obj = getattr(instance, through_attr)
2411 if hasattr(rel_obj, "get_prefetch_queryset"):
2412 prefetcher = rel_obj
2413 if through_attr != to_attr:
2414 # Special case cached_property instances because hasattr
2415 # triggers attribute computation and assignment.
2416 if isinstance(
2417 getattr(instance.__class__, to_attr, None), cached_property
2418 ):
2420 def has_cached_property(instance):
2421 return to_attr in instance.__dict__
2423 is_fetched = has_cached_property
2424 else:
2426 def in_prefetched_cache(instance):
2427 return through_attr in instance._prefetched_objects_cache
2429 is_fetched = in_prefetched_cache
2430 return prefetcher, rel_obj_descriptor, attr_found, is_fetched
2433def prefetch_one_level(instances, prefetcher, lookup, level):
2434 """
2435 Helper function for prefetch_related_objects().
2437 Run prefetches on all instances using the prefetcher object,
2438 assigning results to relevant caches in instance.
2440 Return the prefetched objects along with any additional prefetches that
2441 must be done due to prefetch_related lookups found from default managers.
2442 """
2443 # prefetcher must have a method get_prefetch_queryset() which takes a list
2444 # of instances, and returns a tuple:
2446 # (queryset of instances of self.model that are related to passed in instances,
2447 # callable that gets value to be matched for returned instances,
2448 # callable that gets value to be matched for passed in instances,
2449 # boolean that is True for singly related objects,
2450 # cache or field name to assign to,
2451 # boolean that is True when the previous argument is a cache name vs a field name).
2453 # The 'values to be matched' must be hashable as they will be used
2454 # in a dictionary.
2456 (
2457 rel_qs,
2458 rel_obj_attr,
2459 instance_attr,
2460 single,
2461 cache_name,
2462 is_descriptor,
2463 ) = prefetcher.get_prefetch_queryset(instances, lookup.get_current_queryset(level))
2464 # We have to handle the possibility that the QuerySet we just got back
2465 # contains some prefetch_related lookups. We don't want to trigger the
2466 # prefetch_related functionality by evaluating the query. Rather, we need
2467 # to merge in the prefetch_related lookups.
2468 # Copy the lookups in case it is a Prefetch object which could be reused
2469 # later (happens in nested prefetch_related).
2470 additional_lookups = [
2471 copy.copy(additional_lookup)
2472 for additional_lookup in getattr(rel_qs, "_prefetch_related_lookups", ())
2473 ]
2474 if additional_lookups:
2475 # Don't need to clone because the manager should have given us a fresh
2476 # instance, so we access an internal instead of using public interface
2477 # for performance reasons.
2478 rel_qs._prefetch_related_lookups = ()
2480 all_related_objects = list(rel_qs)
2482 rel_obj_cache = {}
2483 for rel_obj in all_related_objects:
2484 rel_attr_val = rel_obj_attr(rel_obj)
2485 rel_obj_cache.setdefault(rel_attr_val, []).append(rel_obj)
2487 to_attr, as_attr = lookup.get_current_to_attr(level)
2488 # Make sure `to_attr` does not conflict with a field.
2489 if as_attr and instances:
2490 # We assume that objects retrieved are homogeneous (which is the premise
2491 # of prefetch_related), so what applies to first object applies to all.
2492 model = instances[0].__class__
2493 try:
2494 model._meta.get_field(to_attr)
2495 except exceptions.FieldDoesNotExist:
2496 pass
2497 else:
2498 msg = "to_attr={} conflicts with a field on the {} model."
2499 raise ValueError(msg.format(to_attr, model.__name__))
2501 # Whether or not we're prefetching the last part of the lookup.
2502 leaf = len(lookup.prefetch_through.split(LOOKUP_SEP)) - 1 == level
2504 for obj in instances:
2505 instance_attr_val = instance_attr(obj)
2506 vals = rel_obj_cache.get(instance_attr_val, [])
2508 if single:
2509 val = vals[0] if vals else None
2510 if as_attr:
2511 # A to_attr has been given for the prefetch.
2512 setattr(obj, to_attr, val)
2513 elif is_descriptor:
2514 # cache_name points to a field name in obj.
2515 # This field is a descriptor for a related object.
2516 setattr(obj, cache_name, val)
2517 else:
2518 # No to_attr has been given for this prefetch operation and the
2519 # cache_name does not point to a descriptor. Store the value of
2520 # the field in the object's field cache.
2521 obj._state.fields_cache[cache_name] = val
2522 else:
2523 if as_attr:
2524 setattr(obj, to_attr, vals)
2525 else:
2526 manager = getattr(obj, to_attr)
2527 if leaf and lookup.queryset is not None:
2528 qs = manager._apply_rel_filters(lookup.queryset)
2529 else:
2530 qs = manager.get_queryset()
2531 qs._result_cache = vals
2532 # We don't want the individual qs doing prefetch_related now,
2533 # since we have merged this into the current work.
2534 qs._prefetch_done = True
2535 obj._prefetched_objects_cache[cache_name] = qs
2536 return all_related_objects, additional_lookups
2539class RelatedPopulator:
2540 """
2541 RelatedPopulator is used for select_related() object instantiation.
2543 The idea is that each select_related() model will be populated by a
2544 different RelatedPopulator instance. The RelatedPopulator instances get
2545 klass_info and select (computed in SQLCompiler) plus the used db as
2546 input for initialization. That data is used to compute which columns
2547 to use, how to instantiate the model, and how to populate the links
2548 between the objects.
2550 The actual creation of the objects is done in populate() method. This
2551 method gets row and from_obj as input and populates the select_related()
2552 model instance.
2553 """
2555 def __init__(self, klass_info, select, db):
2556 self.db = db
2557 # Pre-compute needed attributes. The attributes are:
2558 # - model_cls: the possibly deferred model class to instantiate
2559 # - either:
2560 # - cols_start, cols_end: usually the columns in the row are
2561 # in the same order model_cls.__init__ expects them, so we
2562 # can instantiate by model_cls(*row[cols_start:cols_end])
2563 # - reorder_for_init: When select_related descends to a child
2564 # class, then we want to reuse the already selected parent
2565 # data. However, in this case the parent data isn't necessarily
2566 # in the same order that Model.__init__ expects it to be, so
2567 # we have to reorder the parent data. The reorder_for_init
2568 # attribute contains a function used to reorder the field data
2569 # in the order __init__ expects it.
2570 # - pk_idx: the index of the primary key field in the reordered
2571 # model data. Used to check if a related object exists at all.
2572 # - init_list: the field attnames fetched from the database. For
2573 # deferred models this isn't the same as all attnames of the
2574 # model's fields.
2575 # - related_populators: a list of RelatedPopulator instances if
2576 # select_related() descends to related models from this model.
2577 # - local_setter, remote_setter: Methods to set cached values on
2578 # the object being populated and on the remote object. Usually
2579 # these are Field.set_cached_value() methods.
2580 select_fields = klass_info["select_fields"]
2581 from_parent = klass_info["from_parent"]
2582 if not from_parent:
2583 self.cols_start = select_fields[0]
2584 self.cols_end = select_fields[-1] + 1
2585 self.init_list = [
2586 f[0].target.attname for f in select[self.cols_start : self.cols_end]
2587 ]
2588 self.reorder_for_init = None
2589 else:
2590 attname_indexes = {
2591 select[idx][0].target.attname: idx for idx in select_fields
2592 }
2593 model_init_attnames = (
2594 f.attname for f in klass_info["model"]._meta.concrete_fields
2595 )
2596 self.init_list = [
2597 attname for attname in model_init_attnames if attname in attname_indexes
2598 ]
2599 self.reorder_for_init = operator.itemgetter(
2600 *[attname_indexes[attname] for attname in self.init_list]
2601 )
2603 self.model_cls = klass_info["model"]
2604 self.pk_idx = self.init_list.index(self.model_cls._meta.pk.attname)
2605 self.related_populators = get_related_populators(klass_info, select, self.db)
2606 self.local_setter = klass_info["local_setter"]
2607 self.remote_setter = klass_info["remote_setter"]
2609 def populate(self, row, from_obj):
2610 if self.reorder_for_init:
2611 obj_data = self.reorder_for_init(row)
2612 else:
2613 obj_data = row[self.cols_start : self.cols_end]
2614 if obj_data[self.pk_idx] is None:
2615 obj = None
2616 else:
2617 obj = self.model_cls.from_db(self.db, self.init_list, obj_data)
2618 for rel_iter in self.related_populators:
2619 rel_iter.populate(row, obj)
2620 self.local_setter(from_obj, obj)
2621 if obj is not None:
2622 self.remote_setter(obj, from_obj)
2625def get_related_populators(klass_info, select, db):
2626 iterators = []
2627 related_klass_infos = klass_info.get("related_klass_infos", [])
2628 for rel_klass_info in related_klass_infos:
2629 rel_cls = RelatedPopulator(rel_klass_info, select, db)
2630 iterators.append(rel_cls)
2631 return iterators