Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2The main QuerySet implementation. This provides the public API for the ORM.
3"""
5import copy
6import operator
7import warnings
8from itertools import chain, islice
10from asgiref.sync import sync_to_async
12import django
13from django.conf import settings
14from django.core import exceptions
15from django.db import (
16 DJANGO_VERSION_PICKLE_KEY,
17 IntegrityError,
18 NotSupportedError,
19 connections,
20 router,
21 transaction,
22)
23from django.db.models import AutoField, DateField, DateTimeField, Field, sql
24from django.db.models.constants import LOOKUP_SEP, OnConflict
25from django.db.models.deletion import Collector
26from django.db.models.expressions import Case, F, Value, When
27from django.db.models.functions import Cast, Trunc
28from django.db.models.query_utils import FilteredRelation, Q
29from django.db.models.sql.constants import GET_ITERATOR_CHUNK_SIZE, ROW_COUNT
30from django.db.models.utils import (
31 AltersData,
32 create_namedtuple_class,
33 resolve_callables,
34)
35from django.utils import timezone
36from django.utils.functional import cached_property, partition
38# The maximum number of results to fetch in a get() query.
39MAX_GET_RESULTS = 21
41# The maximum number of items to display in a QuerySet.__repr__
42REPR_OUTPUT_SIZE = 20
45class BaseIterable:
46 def __init__(
47 self, queryset, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE
48 ):
49 self.queryset = queryset
50 self.chunked_fetch = chunked_fetch
51 self.chunk_size = chunk_size
53 async def _async_generator(self):
54 # Generators don't actually start running until the first time you call
55 # next() on them, so make the generator object in the async thread and
56 # then repeatedly dispatch to it in a sync thread.
57 sync_generator = self.__iter__()
59 def next_slice(gen):
60 return list(islice(gen, self.chunk_size))
62 while True:
63 chunk = await sync_to_async(next_slice)(sync_generator)
64 for item in chunk:
65 yield item
66 if len(chunk) < self.chunk_size:
67 break
69 # __aiter__() is a *synchronous* method that has to then return an
70 # *asynchronous* iterator/generator. Thus, nest an async generator inside
71 # it.
72 # This is a generic iterable converter for now, and is going to suffer a
73 # performance penalty on large sets of items due to the cost of crossing
74 # over the sync barrier for each chunk. Custom __aiter__() methods should
75 # be added to each Iterable subclass, but that needs some work in the
76 # Compiler first.
77 def __aiter__(self):
78 return self._async_generator()
81class ModelIterable(BaseIterable):
82 """Iterable that yields a model instance for each row."""
84 def __iter__(self):
85 queryset = self.queryset
86 db = queryset.db
87 compiler = queryset.query.get_compiler(using=db)
88 # Execute the query. This will also fill compiler.select, klass_info,
89 # and annotations.
90 results = compiler.execute_sql(
91 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
92 )
93 select, klass_info, annotation_col_map = (
94 compiler.select,
95 compiler.klass_info,
96 compiler.annotation_col_map,
97 )
98 model_cls = klass_info["model"]
99 select_fields = klass_info["select_fields"]
100 model_fields_start, model_fields_end = select_fields[0], select_fields[-1] + 1
101 init_list = [
102 f[0].target.attname for f in select[model_fields_start:model_fields_end]
103 ]
104 related_populators = get_related_populators(klass_info, select, db)
105 known_related_objects = [
106 (
107 field,
108 related_objs,
109 operator.attrgetter(
110 *[
111 (
112 field.attname
113 if from_field == "self"
114 else queryset.model._meta.get_field(from_field).attname
115 )
116 for from_field in field.from_fields
117 ]
118 ),
119 )
120 for field, related_objs in queryset._known_related_objects.items()
121 ]
122 for row in compiler.results_iter(results):
123 obj = model_cls.from_db(
124 db, init_list, row[model_fields_start:model_fields_end]
125 )
126 for rel_populator in related_populators:
127 rel_populator.populate(row, obj)
128 if annotation_col_map:
129 for attr_name, col_pos in annotation_col_map.items():
130 setattr(obj, attr_name, row[col_pos])
132 # Add the known related objects to the model.
133 for field, rel_objs, rel_getter in known_related_objects:
134 # Avoid overwriting objects loaded by, e.g., select_related().
135 if field.is_cached(obj):
136 continue
137 rel_obj_id = rel_getter(obj)
138 try:
139 rel_obj = rel_objs[rel_obj_id]
140 except KeyError:
141 pass # May happen in qs1 | qs2 scenarios.
142 else:
143 setattr(obj, field.name, rel_obj)
145 yield obj
148class RawModelIterable(BaseIterable):
149 """
150 Iterable that yields a model instance for each row from a raw queryset.
151 """
153 def __iter__(self):
154 # Cache some things for performance reasons outside the loop.
155 db = self.queryset.db
156 query = self.queryset.query
157 connection = connections[db]
158 compiler = connection.ops.compiler("SQLCompiler")(query, connection, db)
159 query_iterator = iter(query)
161 try:
162 (
163 model_init_names,
164 model_init_pos,
165 annotation_fields,
166 ) = self.queryset.resolve_model_init_order()
167 model_cls = self.queryset.model
168 if model_cls._meta.pk.attname not in model_init_names:
169 raise exceptions.FieldDoesNotExist(
170 "Raw query must include the primary key"
171 )
172 fields = [self.queryset.model_fields.get(c) for c in self.queryset.columns]
173 cols = [f.get_col(f.model._meta.db_table) if f else None for f in fields]
174 converters = compiler.get_converters(cols)
175 if converters:
176 query_iterator = compiler.apply_converters(query_iterator, converters)
177 if compiler.has_composite_fields(cols):
178 query_iterator = compiler.composite_fields_to_tuples(
179 query_iterator, cols
180 )
181 for values in query_iterator:
182 # Associate fields to values
183 model_init_values = [values[pos] for pos in model_init_pos]
184 instance = model_cls.from_db(db, model_init_names, model_init_values)
185 if annotation_fields:
186 for column, pos in annotation_fields:
187 setattr(instance, column, values[pos])
188 yield instance
189 finally:
190 # Done iterating the Query. If it has its own cursor, close it.
191 if hasattr(query, "cursor") and query.cursor:
192 query.cursor.close()
195class ValuesIterable(BaseIterable):
196 """
197 Iterable returned by QuerySet.values() that yields a dict for each row.
198 """
200 def __iter__(self):
201 queryset = self.queryset
202 query = queryset.query
203 compiler = query.get_compiler(queryset.db)
205 if query.selected:
206 names = list(query.selected)
207 else:
208 # extra(select=...) cols are always at the start of the row.
209 names = [
210 *query.extra_select,
211 *query.values_select,
212 *query.annotation_select,
213 ]
214 indexes = range(len(names))
215 for row in compiler.results_iter(
216 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
217 ):
218 yield {names[i]: row[i] for i in indexes}
221class ValuesListIterable(BaseIterable):
222 """
223 Iterable returned by QuerySet.values_list(flat=False) that yields a tuple
224 for each row.
225 """
227 def __iter__(self):
228 queryset = self.queryset
229 query = queryset.query
230 compiler = query.get_compiler(queryset.db)
231 return compiler.results_iter(
232 tuple_expected=True,
233 chunked_fetch=self.chunked_fetch,
234 chunk_size=self.chunk_size,
235 )
238class NamedValuesListIterable(ValuesListIterable):
239 """
240 Iterable returned by QuerySet.values_list(named=True) that yields a
241 namedtuple for each row.
242 """
244 def __iter__(self):
245 queryset = self.queryset
246 if queryset._fields:
247 names = queryset._fields
248 else:
249 query = queryset.query
250 names = [
251 *query.extra_select,
252 *query.values_select,
253 *query.annotation_select,
254 ]
255 tuple_class = create_namedtuple_class(*names)
256 new = tuple.__new__
257 for row in super().__iter__():
258 yield new(tuple_class, row)
261class FlatValuesListIterable(BaseIterable):
262 """
263 Iterable returned by QuerySet.values_list(flat=True) that yields single
264 values.
265 """
267 def __iter__(self):
268 queryset = self.queryset
269 compiler = queryset.query.get_compiler(queryset.db)
270 for row in compiler.results_iter(
271 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
272 ):
273 yield row[0]
276class QuerySet(AltersData):
277 """Represent a lazy database lookup for a set of objects."""
279 def __init__(self, model=None, query=None, using=None, hints=None):
280 self.model = model
281 self._db = using
282 self._hints = hints or {}
283 self._query = query or sql.Query(self.model)
284 self._result_cache = None
285 self._sticky_filter = False
286 self._for_write = False
287 self._prefetch_related_lookups = ()
288 self._prefetch_done = False
289 self._known_related_objects = {} # {rel_field: {pk: rel_obj}}
290 self._iterable_class = ModelIterable
291 self._fields = None
292 self._defer_next_filter = False
293 self._deferred_filter = None
295 @property
296 def query(self):
297 if self._deferred_filter:
298 negate, args, kwargs = self._deferred_filter
299 self._filter_or_exclude_inplace(negate, args, kwargs)
300 self._deferred_filter = None
301 return self._query
303 @query.setter
304 def query(self, value):
305 if value.values_select:
306 self._iterable_class = ValuesIterable
307 self._query = value
309 def as_manager(cls):
310 # Address the circular dependency between `Queryset` and `Manager`.
311 from django.db.models.manager import Manager
313 manager = Manager.from_queryset(cls)()
314 manager._built_with_as_manager = True
315 return manager
317 as_manager.queryset_only = True
318 as_manager = classmethod(as_manager)
320 ########################
321 # PYTHON MAGIC METHODS #
322 ########################
324 def __deepcopy__(self, memo):
325 """Don't populate the QuerySet's cache."""
326 obj = self.__class__()
327 for k, v in self.__dict__.items():
328 if k == "_result_cache":
329 obj.__dict__[k] = None
330 else:
331 obj.__dict__[k] = copy.deepcopy(v, memo)
332 return obj
334 def __getstate__(self):
335 # Force the cache to be fully populated.
336 self._fetch_all()
337 return {**self.__dict__, DJANGO_VERSION_PICKLE_KEY: django.__version__}
339 def __setstate__(self, state):
340 pickled_version = state.get(DJANGO_VERSION_PICKLE_KEY)
341 if pickled_version:
342 if pickled_version != django.__version__:
343 warnings.warn(
344 "Pickled queryset instance's Django version %s does not "
345 "match the current version %s."
346 % (pickled_version, django.__version__),
347 RuntimeWarning,
348 stacklevel=2,
349 )
350 else:
351 warnings.warn(
352 "Pickled queryset instance's Django version is not specified.",
353 RuntimeWarning,
354 stacklevel=2,
355 )
356 self.__dict__.update(state)
358 def __repr__(self):
359 data = list(self[: REPR_OUTPUT_SIZE + 1])
360 if len(data) > REPR_OUTPUT_SIZE:
361 data[-1] = "...(remaining elements truncated)..."
362 return "<%s %r>" % (self.__class__.__name__, data)
364 def __len__(self):
365 self._fetch_all()
366 return len(self._result_cache)
368 def __iter__(self):
369 """
370 The queryset iterator protocol uses three nested iterators in the
371 default case:
372 1. sql.compiler.execute_sql()
373 - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE)
374 using cursor.fetchmany(). This part is responsible for
375 doing some column masking, and returning the rows in chunks.
376 2. sql.compiler.results_iter()
377 - Returns one row at time. At this point the rows are still just
378 tuples. In some cases the return values are converted to
379 Python values at this location.
380 3. self.iterator()
381 - Responsible for turning the rows into model objects.
382 """
383 self._fetch_all()
384 return iter(self._result_cache)
386 def __aiter__(self):
387 # Remember, __aiter__ itself is synchronous, it's the thing it returns
388 # that is async!
389 async def generator():
390 await sync_to_async(self._fetch_all)()
391 for item in self._result_cache:
392 yield item
394 return generator()
396 def __bool__(self):
397 self._fetch_all()
398 return bool(self._result_cache)
400 def __getitem__(self, k):
401 """Retrieve an item or slice from the set of results."""
402 if not isinstance(k, (int, slice)):
403 raise TypeError(
404 "QuerySet indices must be integers or slices, not %s."
405 % type(k).__name__
406 )
407 if (isinstance(k, int) and k < 0) or (
408 isinstance(k, slice)
409 and (
410 (k.start is not None and k.start < 0)
411 or (k.stop is not None and k.stop < 0)
412 )
413 ):
414 raise ValueError("Negative indexing is not supported.")
416 if self._result_cache is not None:
417 return self._result_cache[k]
419 if isinstance(k, slice):
420 qs = self._chain()
421 if k.start is not None:
422 start = int(k.start)
423 else:
424 start = None
425 if k.stop is not None:
426 stop = int(k.stop)
427 else:
428 stop = None
429 qs.query.set_limits(start, stop)
430 return list(qs)[:: k.step] if k.step else qs
432 qs = self._chain()
433 qs.query.set_limits(k, k + 1)
434 qs._fetch_all()
435 return qs._result_cache[0]
437 def __class_getitem__(cls, *args, **kwargs):
438 return cls
440 def __and__(self, other):
441 self._check_operator_queryset(other, "&")
442 self._merge_sanity_check(other)
443 if isinstance(other, EmptyQuerySet):
444 return other
445 if isinstance(self, EmptyQuerySet):
446 return self
447 combined = self._chain()
448 combined._merge_known_related_objects(other)
449 combined.query.combine(other.query, sql.AND)
450 return combined
452 def __or__(self, other):
453 self._check_operator_queryset(other, "|")
454 self._merge_sanity_check(other)
455 if isinstance(self, EmptyQuerySet):
456 return other
457 if isinstance(other, EmptyQuerySet):
458 return self
459 query = (
460 self
461 if self.query.can_filter()
462 else self.model._base_manager.filter(pk__in=self.values("pk"))
463 )
464 combined = query._chain()
465 combined._merge_known_related_objects(other)
466 if not other.query.can_filter():
467 other = other.model._base_manager.filter(pk__in=other.values("pk"))
468 combined.query.combine(other.query, sql.OR)
469 return combined
471 def __xor__(self, other):
472 self._check_operator_queryset(other, "^")
473 self._merge_sanity_check(other)
474 if isinstance(self, EmptyQuerySet):
475 return other
476 if isinstance(other, EmptyQuerySet):
477 return self
478 query = (
479 self
480 if self.query.can_filter()
481 else self.model._base_manager.filter(pk__in=self.values("pk"))
482 )
483 combined = query._chain()
484 combined._merge_known_related_objects(other)
485 if not other.query.can_filter():
486 other = other.model._base_manager.filter(pk__in=other.values("pk"))
487 combined.query.combine(other.query, sql.XOR)
488 return combined
490 ####################################
491 # METHODS THAT DO DATABASE QUERIES #
492 ####################################
494 def _iterator(self, use_chunked_fetch, chunk_size):
495 iterable = self._iterable_class(
496 self,
497 chunked_fetch=use_chunked_fetch,
498 chunk_size=chunk_size or 2000,
499 )
500 if not self._prefetch_related_lookups or chunk_size is None:
501 yield from iterable
502 return
504 iterator = iter(iterable)
505 while results := list(islice(iterator, chunk_size)):
506 prefetch_related_objects(results, *self._prefetch_related_lookups)
507 yield from results
509 def iterator(self, chunk_size=None):
510 """
511 An iterator over the results from applying this QuerySet to the
512 database. chunk_size must be provided for QuerySets that prefetch
513 related objects. Otherwise, a default chunk_size of 2000 is supplied.
514 """
515 if chunk_size is None:
516 if self._prefetch_related_lookups:
517 raise ValueError(
518 "chunk_size must be provided when using QuerySet.iterator() after "
519 "prefetch_related()."
520 )
521 elif chunk_size <= 0:
522 raise ValueError("Chunk size must be strictly positive.")
523 use_chunked_fetch = not connections[self.db].settings_dict.get(
524 "DISABLE_SERVER_SIDE_CURSORS"
525 )
526 return self._iterator(use_chunked_fetch, chunk_size)
528 async def aiterator(self, chunk_size=2000):
529 """
530 An asynchronous iterator over the results from applying this QuerySet
531 to the database.
532 """
533 if chunk_size <= 0:
534 raise ValueError("Chunk size must be strictly positive.")
535 use_chunked_fetch = not connections[self.db].settings_dict.get(
536 "DISABLE_SERVER_SIDE_CURSORS"
537 )
538 iterable = self._iterable_class(
539 self, chunked_fetch=use_chunked_fetch, chunk_size=chunk_size
540 )
541 if self._prefetch_related_lookups:
542 results = []
544 async for item in iterable:
545 results.append(item)
546 if len(results) >= chunk_size:
547 await aprefetch_related_objects(
548 results, *self._prefetch_related_lookups
549 )
550 for result in results:
551 yield result
552 results.clear()
554 if results:
555 await aprefetch_related_objects(
556 results, *self._prefetch_related_lookups
557 )
558 for result in results:
559 yield result
560 else:
561 async for item in iterable:
562 yield item
564 def aggregate(self, *args, **kwargs):
565 """
566 Return a dictionary containing the calculations (aggregation)
567 over the current queryset.
569 If args is present the expression is passed as a kwarg using
570 the Aggregate object's default alias.
571 """
572 if self.query.distinct_fields:
573 raise NotImplementedError("aggregate() + distinct(fields) not implemented.")
574 self._validate_values_are_expressions(
575 (*args, *kwargs.values()), method_name="aggregate"
576 )
577 for arg in args:
578 # The default_alias property raises TypeError if default_alias
579 # can't be set automatically or AttributeError if it isn't an
580 # attribute.
581 try:
582 arg.default_alias
583 except (AttributeError, TypeError):
584 raise TypeError("Complex aggregates require an alias")
585 kwargs[arg.default_alias] = arg
587 return self.query.chain().get_aggregation(self.db, kwargs)
589 async def aaggregate(self, *args, **kwargs):
590 return await sync_to_async(self.aggregate)(*args, **kwargs)
592 def count(self):
593 """
594 Perform a SELECT COUNT() and return the number of records as an
595 integer.
597 If the QuerySet is already fully cached, return the length of the
598 cached results set to avoid multiple SELECT COUNT(*) calls.
599 """
600 if self._result_cache is not None:
601 return len(self._result_cache)
603 return self.query.get_count(using=self.db)
605 async def acount(self):
606 return await sync_to_async(self.count)()
608 def get(self, *args, **kwargs):
609 """
610 Perform the query and return a single object matching the given
611 keyword arguments.
612 """
613 if self.query.combinator and (args or kwargs):
614 raise NotSupportedError(
615 "Calling QuerySet.get(...) with filters after %s() is not "
616 "supported." % self.query.combinator
617 )
618 clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)
619 if self.query.can_filter() and not self.query.distinct_fields:
620 clone = clone.order_by()
621 limit = None
622 if (
623 not clone.query.select_for_update
624 or connections[clone.db].features.supports_select_for_update_with_limit
625 ):
626 limit = MAX_GET_RESULTS
627 clone.query.set_limits(high=limit)
628 num = len(clone)
629 if num == 1:
630 return clone._result_cache[0]
631 if not num:
632 raise self.model.DoesNotExist(
633 "%s matching query does not exist." % self.model._meta.object_name
634 )
635 raise self.model.MultipleObjectsReturned(
636 "get() returned more than one %s -- it returned %s!"
637 % (
638 self.model._meta.object_name,
639 num if not limit or num < limit else "more than %s" % (limit - 1),
640 )
641 )
643 async def aget(self, *args, **kwargs):
644 return await sync_to_async(self.get)(*args, **kwargs)
646 def create(self, **kwargs):
647 """
648 Create a new object with the given kwargs, saving it to the database
649 and returning the created object.
650 """
651 reverse_one_to_one_fields = frozenset(kwargs).intersection(
652 self.model._meta._reverse_one_to_one_field_names
653 )
654 if reverse_one_to_one_fields:
655 raise ValueError(
656 "The following fields do not exist in this model: %s"
657 % ", ".join(reverse_one_to_one_fields)
658 )
660 obj = self.model(**kwargs)
661 self._for_write = True
662 obj.save(force_insert=True, using=self.db)
663 return obj
665 create.alters_data = True
667 async def acreate(self, **kwargs):
668 return await sync_to_async(self.create)(**kwargs)
670 acreate.alters_data = True
672 def _prepare_for_bulk_create(self, objs):
673 from django.db.models.expressions import DatabaseDefault
675 connection = connections[self.db]
676 for obj in objs:
677 if not obj._is_pk_set():
678 # Populate new PK values.
679 obj.pk = obj._meta.pk.get_pk_value_on_save(obj)
680 if not connection.features.supports_default_keyword_in_bulk_insert:
681 for field in obj._meta.fields:
682 if field.generated:
683 continue
684 value = getattr(obj, field.attname)
685 if isinstance(value, DatabaseDefault):
686 setattr(obj, field.attname, field.db_default)
688 obj._prepare_related_fields_for_save(operation_name="bulk_create")
690 def _check_bulk_create_options(
691 self, ignore_conflicts, update_conflicts, update_fields, unique_fields
692 ):
693 if ignore_conflicts and update_conflicts:
694 raise ValueError(
695 "ignore_conflicts and update_conflicts are mutually exclusive."
696 )
697 db_features = connections[self.db].features
698 if ignore_conflicts:
699 if not db_features.supports_ignore_conflicts:
700 raise NotSupportedError(
701 "This database backend does not support ignoring conflicts."
702 )
703 return OnConflict.IGNORE
704 elif update_conflicts:
705 if not db_features.supports_update_conflicts:
706 raise NotSupportedError(
707 "This database backend does not support updating conflicts."
708 )
709 if not update_fields:
710 raise ValueError(
711 "Fields that will be updated when a row insertion fails "
712 "on conflicts must be provided."
713 )
714 if unique_fields and not db_features.supports_update_conflicts_with_target:
715 raise NotSupportedError(
716 "This database backend does not support updating "
717 "conflicts with specifying unique fields that can trigger "
718 "the upsert."
719 )
720 if not unique_fields and db_features.supports_update_conflicts_with_target:
721 raise ValueError(
722 "Unique fields that can trigger the upsert must be provided."
723 )
724 # Updating primary keys and non-concrete fields is forbidden.
725 if any(not f.concrete or f.many_to_many for f in update_fields):
726 raise ValueError(
727 "bulk_create() can only be used with concrete fields in "
728 "update_fields."
729 )
730 if any(f in self.model._meta.pk_fields for f in update_fields):
731 raise ValueError(
732 "bulk_create() cannot be used with primary keys in "
733 "update_fields."
734 )
735 if unique_fields:
736 if any(not f.concrete or f.many_to_many for f in unique_fields):
737 raise ValueError(
738 "bulk_create() can only be used with concrete fields "
739 "in unique_fields."
740 )
741 return OnConflict.UPDATE
742 return None
744 def bulk_create(
745 self,
746 objs,
747 batch_size=None,
748 ignore_conflicts=False,
749 update_conflicts=False,
750 update_fields=None,
751 unique_fields=None,
752 ):
753 """
754 Insert each of the instances into the database. Do *not* call
755 save() on each of the instances, do not send any pre/post_save
756 signals, and do not set the primary key attribute if it is an
757 autoincrement field (except if features.can_return_rows_from_bulk_insert=True).
758 Multi-table models are not supported.
759 """
760 # When you bulk insert you don't get the primary keys back (if it's an
761 # autoincrement, except if can_return_rows_from_bulk_insert=True), so
762 # you can't insert into the child tables which references this. There
763 # are two workarounds:
764 # 1) This could be implemented if you didn't have an autoincrement pk
765 # 2) You could do it by doing O(n) normal inserts into the parent
766 # tables to get the primary keys back and then doing a single bulk
767 # insert into the childmost table.
768 # We currently set the primary keys on the objects when using
769 # PostgreSQL via the RETURNING ID clause. It should be possible for
770 # Oracle as well, but the semantics for extracting the primary keys is
771 # trickier so it's not done yet.
772 if batch_size is not None and batch_size <= 0:
773 raise ValueError("Batch size must be a positive integer.")
774 # Check that the parents share the same concrete model with the our
775 # model to detect the inheritance pattern ConcreteGrandParent ->
776 # MultiTableParent -> ProxyChild. Simply checking self.model._meta.proxy
777 # would not identify that case as involving multiple tables.
778 for parent in self.model._meta.all_parents:
779 if parent._meta.concrete_model is not self.model._meta.concrete_model:
780 raise ValueError("Can't bulk create a multi-table inherited model")
781 if not objs:
782 return objs
783 opts = self.model._meta
784 if unique_fields:
785 # Primary key is allowed in unique_fields.
786 unique_fields = [
787 self.model._meta.get_field(opts.pk.name if name == "pk" else name)
788 for name in unique_fields
789 ]
790 if update_fields:
791 update_fields = [self.model._meta.get_field(name) for name in update_fields]
792 on_conflict = self._check_bulk_create_options(
793 ignore_conflicts,
794 update_conflicts,
795 update_fields,
796 unique_fields,
797 )
798 self._for_write = True
799 fields = [f for f in opts.concrete_fields if not f.generated]
800 objs = list(objs)
801 self._prepare_for_bulk_create(objs)
802 with transaction.atomic(using=self.db, savepoint=False):
803 objs_without_pk, objs_with_pk = partition(lambda o: o._is_pk_set(), objs)
804 if objs_with_pk:
805 returned_columns = self._batched_insert(
806 objs_with_pk,
807 fields,
808 batch_size,
809 on_conflict=on_conflict,
810 update_fields=update_fields,
811 unique_fields=unique_fields,
812 )
813 for obj_with_pk, results in zip(objs_with_pk, returned_columns):
814 for result, field in zip(results, opts.db_returning_fields):
815 if field != opts.pk:
816 setattr(obj_with_pk, field.attname, result)
817 for obj_with_pk in objs_with_pk:
818 obj_with_pk._state.adding = False
819 obj_with_pk._state.db = self.db
820 if objs_without_pk:
821 fields = [f for f in fields if not isinstance(f, AutoField)]
822 returned_columns = self._batched_insert(
823 objs_without_pk,
824 fields,
825 batch_size,
826 on_conflict=on_conflict,
827 update_fields=update_fields,
828 unique_fields=unique_fields,
829 )
830 connection = connections[self.db]
831 if (
832 connection.features.can_return_rows_from_bulk_insert
833 and on_conflict is None
834 ):
835 assert len(returned_columns) == len(objs_without_pk)
836 for obj_without_pk, results in zip(objs_without_pk, returned_columns):
837 for result, field in zip(results, opts.db_returning_fields):
838 setattr(obj_without_pk, field.attname, result)
839 obj_without_pk._state.adding = False
840 obj_without_pk._state.db = self.db
842 return objs
844 bulk_create.alters_data = True
846 async def abulk_create(
847 self,
848 objs,
849 batch_size=None,
850 ignore_conflicts=False,
851 update_conflicts=False,
852 update_fields=None,
853 unique_fields=None,
854 ):
855 return await sync_to_async(self.bulk_create)(
856 objs=objs,
857 batch_size=batch_size,
858 ignore_conflicts=ignore_conflicts,
859 update_conflicts=update_conflicts,
860 update_fields=update_fields,
861 unique_fields=unique_fields,
862 )
864 abulk_create.alters_data = True
866 def bulk_update(self, objs, fields, batch_size=None):
867 """
868 Update the given fields in each of the given objects in the database.
869 """
870 if batch_size is not None and batch_size <= 0:
871 raise ValueError("Batch size must be a positive integer.")
872 if not fields:
873 raise ValueError("Field names must be given to bulk_update().")
874 objs = tuple(objs)
875 if not all(obj._is_pk_set() for obj in objs):
876 raise ValueError("All bulk_update() objects must have a primary key set.")
877 fields = [self.model._meta.get_field(name) for name in fields]
878 if any(not f.concrete or f.many_to_many for f in fields):
879 raise ValueError("bulk_update() can only be used with concrete fields.")
880 all_pk_fields = set(self.model._meta.pk_fields)
881 for parent in self.model._meta.all_parents:
882 all_pk_fields.update(parent._meta.pk_fields)
883 if any(f in all_pk_fields for f in fields):
884 raise ValueError("bulk_update() cannot be used with primary key fields.")
885 if not objs:
886 return 0
887 for obj in objs:
888 obj._prepare_related_fields_for_save(
889 operation_name="bulk_update", fields=fields
890 )
891 # PK is used twice in the resulting update query, once in the filter
892 # and once in the WHEN. Each field will also have one CAST.
893 self._for_write = True
894 connection = connections[self.db]
895 max_batch_size = connection.ops.bulk_batch_size(["pk", "pk"] + fields, objs)
896 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
897 requires_casting = connection.features.requires_casted_case_in_updates
898 batches = (objs[i : i + batch_size] for i in range(0, len(objs), batch_size))
899 updates = []
900 for batch_objs in batches:
901 update_kwargs = {}
902 for field in fields:
903 when_statements = []
904 for obj in batch_objs:
905 attr = getattr(obj, field.attname)
906 if not hasattr(attr, "resolve_expression"):
907 attr = Value(attr, output_field=field)
908 when_statements.append(When(pk=obj.pk, then=attr))
909 case_statement = Case(*when_statements, output_field=field)
910 if requires_casting:
911 case_statement = Cast(case_statement, output_field=field)
912 update_kwargs[field.attname] = case_statement
913 updates.append(([obj.pk for obj in batch_objs], update_kwargs))
914 rows_updated = 0
915 queryset = self.using(self.db)
916 with transaction.atomic(using=self.db, savepoint=False):
917 for pks, update_kwargs in updates:
918 rows_updated += queryset.filter(pk__in=pks).update(**update_kwargs)
919 return rows_updated
921 bulk_update.alters_data = True
923 async def abulk_update(self, objs, fields, batch_size=None):
924 return await sync_to_async(self.bulk_update)(
925 objs=objs,
926 fields=fields,
927 batch_size=batch_size,
928 )
930 abulk_update.alters_data = True
932 def get_or_create(self, defaults=None, **kwargs):
933 """
934 Look up an object with the given kwargs, creating one if necessary.
935 Return a tuple of (object, created), where created is a boolean
936 specifying whether an object was created.
937 """
938 # The get() needs to be targeted at the write database in order
939 # to avoid potential transaction consistency problems.
940 self._for_write = True
941 try:
942 return self.get(**kwargs), False
943 except self.model.DoesNotExist:
944 params = self._extract_model_params(defaults, **kwargs)
945 # Try to create an object using passed params.
946 try:
947 with transaction.atomic(using=self.db):
948 params = dict(resolve_callables(params))
949 return self.create(**params), True
950 except IntegrityError:
951 try:
952 return self.get(**kwargs), False
953 except self.model.DoesNotExist:
954 pass
955 raise
957 get_or_create.alters_data = True
959 async def aget_or_create(self, defaults=None, **kwargs):
960 return await sync_to_async(self.get_or_create)(
961 defaults=defaults,
962 **kwargs,
963 )
965 aget_or_create.alters_data = True
967 def update_or_create(self, defaults=None, create_defaults=None, **kwargs):
968 """
969 Look up an object with the given kwargs, updating one with defaults
970 if it exists, otherwise create a new one. Optionally, an object can
971 be created with different values than defaults by using
972 create_defaults.
973 Return a tuple (object, created), where created is a boolean
974 specifying whether an object was created.
975 """
976 update_defaults = defaults or {}
977 if create_defaults is None:
978 create_defaults = update_defaults
980 self._for_write = True
981 with transaction.atomic(using=self.db):
982 # Lock the row so that a concurrent update is blocked until
983 # update_or_create() has performed its save.
984 obj, created = self.select_for_update().get_or_create(
985 create_defaults, **kwargs
986 )
987 if created:
988 return obj, created
989 for k, v in resolve_callables(update_defaults):
990 setattr(obj, k, v)
992 update_fields = set(update_defaults)
993 concrete_field_names = self.model._meta._non_pk_concrete_field_names
994 # update_fields does not support non-concrete fields.
995 if concrete_field_names.issuperset(update_fields):
996 # Add fields which are set on pre_save(), e.g. auto_now fields.
997 # This is to maintain backward compatibility as these fields
998 # are not updated unless explicitly specified in the
999 # update_fields list.
1000 pk_fields = self.model._meta.pk_fields
1001 for field in self.model._meta.local_concrete_fields:
1002 if not (
1003 field in pk_fields or field.__class__.pre_save is Field.pre_save
1004 ):
1005 update_fields.add(field.name)
1006 if field.name != field.attname:
1007 update_fields.add(field.attname)
1008 obj.save(using=self.db, update_fields=update_fields)
1009 else:
1010 obj.save(using=self.db)
1011 return obj, False
1013 update_or_create.alters_data = True
1015 async def aupdate_or_create(self, defaults=None, create_defaults=None, **kwargs):
1016 return await sync_to_async(self.update_or_create)(
1017 defaults=defaults,
1018 create_defaults=create_defaults,
1019 **kwargs,
1020 )
1022 aupdate_or_create.alters_data = True
1024 def _extract_model_params(self, defaults, **kwargs):
1025 """
1026 Prepare `params` for creating a model instance based on the given
1027 kwargs; for use by get_or_create().
1028 """
1029 defaults = defaults or {}
1030 params = {k: v for k, v in kwargs.items() if LOOKUP_SEP not in k}
1031 params.update(defaults)
1032 property_names = self.model._meta._property_names
1033 invalid_params = []
1034 for param in params:
1035 try:
1036 self.model._meta.get_field(param)
1037 except exceptions.FieldDoesNotExist:
1038 # It's okay to use a model's property if it has a setter.
1039 if not (param in property_names and getattr(self.model, param).fset):
1040 invalid_params.append(param)
1041 if invalid_params:
1042 raise exceptions.FieldError(
1043 "Invalid field name(s) for model %s: '%s'."
1044 % (
1045 self.model._meta.object_name,
1046 "', '".join(sorted(invalid_params)),
1047 )
1048 )
1049 return params
1051 def _earliest(self, *fields):
1052 """
1053 Return the earliest object according to fields (if given) or by the
1054 model's Meta.get_latest_by.
1055 """
1056 if fields:
1057 order_by = fields
1058 else:
1059 order_by = getattr(self.model._meta, "get_latest_by")
1060 if order_by and not isinstance(order_by, (tuple, list)):
1061 order_by = (order_by,)
1062 if order_by is None:
1063 raise ValueError(
1064 "earliest() and latest() require either fields as positional "
1065 "arguments or 'get_latest_by' in the model's Meta."
1066 )
1067 obj = self._chain()
1068 obj.query.set_limits(high=1)
1069 obj.query.clear_ordering(force=True)
1070 obj.query.add_ordering(*order_by)
1071 return obj.get()
1073 def earliest(self, *fields):
1074 if self.query.is_sliced:
1075 raise TypeError("Cannot change a query once a slice has been taken.")
1076 return self._earliest(*fields)
1078 async def aearliest(self, *fields):
1079 return await sync_to_async(self.earliest)(*fields)
1081 def latest(self, *fields):
1082 """
1083 Return the latest object according to fields (if given) or by the
1084 model's Meta.get_latest_by.
1085 """
1086 if self.query.is_sliced:
1087 raise TypeError("Cannot change a query once a slice has been taken.")
1088 return self.reverse()._earliest(*fields)
1090 async def alatest(self, *fields):
1091 return await sync_to_async(self.latest)(*fields)
1093 def first(self):
1094 """Return the first object of a query or None if no match is found."""
1095 if self.ordered:
1096 queryset = self
1097 else:
1098 self._check_ordering_first_last_queryset_aggregation(method="first")
1099 queryset = self.order_by("pk")
1100 for obj in queryset[:1]:
1101 return obj
1103 async def afirst(self):
1104 return await sync_to_async(self.first)()
1106 def last(self):
1107 """Return the last object of a query or None if no match is found."""
1108 if self.ordered:
1109 queryset = self.reverse()
1110 else:
1111 self._check_ordering_first_last_queryset_aggregation(method="last")
1112 queryset = self.order_by("-pk")
1113 for obj in queryset[:1]:
1114 return obj
1116 async def alast(self):
1117 return await sync_to_async(self.last)()
1119 def in_bulk(self, id_list=None, *, field_name="pk"):
1120 """
1121 Return a dictionary mapping each of the given IDs to the object with
1122 that ID. If `id_list` isn't provided, evaluate the entire QuerySet.
1123 """
1124 if self.query.is_sliced:
1125 raise TypeError("Cannot use 'limit' or 'offset' with in_bulk().")
1126 if not issubclass(self._iterable_class, ModelIterable):
1127 raise TypeError("in_bulk() cannot be used with values() or values_list().")
1128 opts = self.model._meta
1129 unique_fields = [
1130 constraint.fields[0]
1131 for constraint in opts.total_unique_constraints
1132 if len(constraint.fields) == 1
1133 ]
1134 if (
1135 field_name != "pk"
1136 and not opts.get_field(field_name).unique
1137 and field_name not in unique_fields
1138 and self.query.distinct_fields != (field_name,)
1139 ):
1140 raise ValueError(
1141 "in_bulk()'s field_name must be a unique field but %r isn't."
1142 % field_name
1143 )
1144 if id_list is not None:
1145 if not id_list:
1146 return {}
1147 filter_key = "{}__in".format(field_name)
1148 batch_size = connections[self.db].features.max_query_params
1149 id_list = tuple(id_list)
1150 # If the database has a limit on the number of query parameters
1151 # (e.g. SQLite), retrieve objects in batches if necessary.
1152 if batch_size and batch_size < len(id_list):
1153 qs = ()
1154 for offset in range(0, len(id_list), batch_size):
1155 batch = id_list[offset : offset + batch_size]
1156 qs += tuple(self.filter(**{filter_key: batch}))
1157 else:
1158 qs = self.filter(**{filter_key: id_list})
1159 else:
1160 qs = self._chain()
1161 return {getattr(obj, field_name): obj for obj in qs}
1163 async def ain_bulk(self, id_list=None, *, field_name="pk"):
1164 return await sync_to_async(self.in_bulk)(
1165 id_list=id_list,
1166 field_name=field_name,
1167 )
1169 def delete(self):
1170 """Delete the records in the current QuerySet."""
1171 self._not_support_combined_queries("delete")
1172 if self.query.is_sliced:
1173 raise TypeError("Cannot use 'limit' or 'offset' with delete().")
1174 if self.query.distinct_fields:
1175 raise TypeError("Cannot call delete() after .distinct(*fields).")
1176 if self._fields is not None:
1177 raise TypeError("Cannot call delete() after .values() or .values_list()")
1179 del_query = self._chain()
1181 # The delete is actually 2 queries - one to find related objects,
1182 # and one to delete. Make sure that the discovery of related
1183 # objects is performed on the same database as the deletion.
1184 del_query._for_write = True
1186 # Disable non-supported fields.
1187 del_query.query.select_for_update = False
1188 del_query.query.select_related = False
1189 del_query.query.clear_ordering(force=True)
1191 collector = Collector(using=del_query.db, origin=self)
1192 collector.collect(del_query)
1193 num_deleted, num_deleted_per_model = collector.delete()
1195 # Clear the result cache, in case this QuerySet gets reused.
1196 self._result_cache = None
1197 return num_deleted, num_deleted_per_model
1199 delete.alters_data = True
1200 delete.queryset_only = True
1202 async def adelete(self):
1203 return await sync_to_async(self.delete)()
1205 adelete.alters_data = True
1206 adelete.queryset_only = True
1208 def _raw_delete(self, using):
1209 """
1210 Delete objects found from the given queryset in single direct SQL
1211 query. No signals are sent and there is no protection for cascades.
1212 """
1213 query = self.query.clone()
1214 query.__class__ = sql.DeleteQuery
1215 return query.get_compiler(using).execute_sql(ROW_COUNT)
1217 _raw_delete.alters_data = True
1219 def update(self, **kwargs):
1220 """
1221 Update all elements in the current QuerySet, setting all the given
1222 fields to the appropriate values.
1223 """
1224 self._not_support_combined_queries("update")
1225 if self.query.is_sliced:
1226 raise TypeError("Cannot update a query once a slice has been taken.")
1227 self._for_write = True
1228 query = self.query.chain(sql.UpdateQuery)
1229 query.add_update_values(kwargs)
1231 # Inline annotations in order_by(), if possible.
1232 new_order_by = []
1233 for col in query.order_by:
1234 alias = col
1235 descending = False
1236 if isinstance(alias, str) and alias.startswith("-"):
1237 alias = alias.removeprefix("-")
1238 descending = True
1239 if annotation := query.annotations.get(alias):
1240 if getattr(annotation, "contains_aggregate", False):
1241 raise exceptions.FieldError(
1242 f"Cannot update when ordering by an aggregate: {annotation}"
1243 )
1244 if descending:
1245 annotation = annotation.desc()
1246 new_order_by.append(annotation)
1247 else:
1248 new_order_by.append(col)
1249 query.order_by = tuple(new_order_by)
1251 # Clear any annotations so that they won't be present in subqueries.
1252 query.annotations = {}
1253 with transaction.mark_for_rollback_on_error(using=self.db):
1254 rows = query.get_compiler(self.db).execute_sql(ROW_COUNT)
1255 self._result_cache = None
1256 return rows
1258 update.alters_data = True
1260 async def aupdate(self, **kwargs):
1261 return await sync_to_async(self.update)(**kwargs)
1263 aupdate.alters_data = True
1265 def _update(self, values):
1266 """
1267 A version of update() that accepts field objects instead of field names.
1268 Used primarily for model saving and not intended for use by general
1269 code (it requires too much poking around at model internals to be
1270 useful at that level).
1271 """
1272 if self.query.is_sliced:
1273 raise TypeError("Cannot update a query once a slice has been taken.")
1274 query = self.query.chain(sql.UpdateQuery)
1275 query.add_update_fields(values)
1276 # Clear any annotations so that they won't be present in subqueries.
1277 query.annotations = {}
1278 self._result_cache = None
1279 return query.get_compiler(self.db).execute_sql(ROW_COUNT)
1281 _update.alters_data = True
1282 _update.queryset_only = False
1284 def exists(self):
1285 """
1286 Return True if the QuerySet would have any results, False otherwise.
1287 """
1288 if self._result_cache is None:
1289 return self.query.has_results(using=self.db)
1290 return bool(self._result_cache)
1292 async def aexists(self):
1293 return await sync_to_async(self.exists)()
1295 def contains(self, obj):
1296 """
1297 Return True if the QuerySet contains the provided obj,
1298 False otherwise.
1299 """
1300 self._not_support_combined_queries("contains")
1301 if self._fields is not None:
1302 raise TypeError(
1303 "Cannot call QuerySet.contains() after .values() or .values_list()."
1304 )
1305 try:
1306 if obj._meta.concrete_model != self.model._meta.concrete_model:
1307 return False
1308 except AttributeError:
1309 raise TypeError("'obj' must be a model instance.")
1310 if not obj._is_pk_set():
1311 raise ValueError("QuerySet.contains() cannot be used on unsaved objects.")
1312 if self._result_cache is not None:
1313 return obj in self._result_cache
1314 return self.filter(pk=obj.pk).exists()
1316 async def acontains(self, obj):
1317 return await sync_to_async(self.contains)(obj=obj)
1319 def _prefetch_related_objects(self):
1320 # This method can only be called once the result cache has been filled.
1321 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
1322 self._prefetch_done = True
1324 def explain(self, *, format=None, **options):
1325 """
1326 Runs an EXPLAIN on the SQL query this QuerySet would perform, and
1327 returns the results.
1328 """
1329 return self.query.explain(using=self.db, format=format, **options)
1331 async def aexplain(self, *, format=None, **options):
1332 return await sync_to_async(self.explain)(format=format, **options)
1334 ##################################################
1335 # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
1336 ##################################################
1338 def raw(self, raw_query, params=(), translations=None, using=None):
1339 if using is None:
1340 using = self.db
1341 qs = RawQuerySet(
1342 raw_query,
1343 model=self.model,
1344 params=params,
1345 translations=translations,
1346 using=using,
1347 )
1348 qs._prefetch_related_lookups = self._prefetch_related_lookups[:]
1349 return qs
1351 def _values(self, *fields, **expressions):
1352 clone = self._chain()
1353 if expressions:
1354 clone = clone.annotate(**expressions)
1355 clone._fields = fields
1356 clone.query.set_values(fields)
1357 return clone
1359 def values(self, *fields, **expressions):
1360 fields += tuple(expressions)
1361 clone = self._values(*fields, **expressions)
1362 clone._iterable_class = ValuesIterable
1363 return clone
1365 def values_list(self, *fields, flat=False, named=False):
1366 if flat and named:
1367 raise TypeError("'flat' and 'named' can't be used together.")
1368 if flat and len(fields) > 1:
1369 raise TypeError(
1370 "'flat' is not valid when values_list is called with more than one "
1371 "field."
1372 )
1374 field_names = {f for f in fields if not hasattr(f, "resolve_expression")}
1375 _fields = []
1376 expressions = {}
1377 counter = 1
1378 for field in fields:
1379 if hasattr(field, "resolve_expression"):
1380 field_id_prefix = getattr(
1381 field, "default_alias", field.__class__.__name__.lower()
1382 )
1383 while True:
1384 field_id = field_id_prefix + str(counter)
1385 counter += 1
1386 if field_id not in field_names:
1387 break
1388 expressions[field_id] = field
1389 _fields.append(field_id)
1390 else:
1391 _fields.append(field)
1393 clone = self._values(*_fields, **expressions)
1394 clone._iterable_class = (
1395 NamedValuesListIterable
1396 if named
1397 else FlatValuesListIterable if flat else ValuesListIterable
1398 )
1399 return clone
1401 def dates(self, field_name, kind, order="ASC"):
1402 """
1403 Return a list of date objects representing all available dates for
1404 the given field_name, scoped to 'kind'.
1405 """
1406 if kind not in ("year", "month", "week", "day"):
1407 raise ValueError("'kind' must be one of 'year', 'month', 'week', or 'day'.")
1408 if order not in ("ASC", "DESC"):
1409 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1410 return (
1411 self.annotate(
1412 datefield=Trunc(field_name, kind, output_field=DateField()),
1413 plain_field=F(field_name),
1414 )
1415 .values_list("datefield", flat=True)
1416 .distinct()
1417 .filter(plain_field__isnull=False)
1418 .order_by(("-" if order == "DESC" else "") + "datefield")
1419 )
1421 def datetimes(self, field_name, kind, order="ASC", tzinfo=None):
1422 """
1423 Return a list of datetime objects representing all available
1424 datetimes for the given field_name, scoped to 'kind'.
1425 """
1426 if kind not in ("year", "month", "week", "day", "hour", "minute", "second"):
1427 raise ValueError(
1428 "'kind' must be one of 'year', 'month', 'week', 'day', "
1429 "'hour', 'minute', or 'second'."
1430 )
1431 if order not in ("ASC", "DESC"):
1432 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1433 if settings.USE_TZ:
1434 if tzinfo is None:
1435 tzinfo = timezone.get_current_timezone()
1436 else:
1437 tzinfo = None
1438 return (
1439 self.annotate(
1440 datetimefield=Trunc(
1441 field_name,
1442 kind,
1443 output_field=DateTimeField(),
1444 tzinfo=tzinfo,
1445 ),
1446 plain_field=F(field_name),
1447 )
1448 .values_list("datetimefield", flat=True)
1449 .distinct()
1450 .filter(plain_field__isnull=False)
1451 .order_by(("-" if order == "DESC" else "") + "datetimefield")
1452 )
1454 def none(self):
1455 """Return an empty QuerySet."""
1456 clone = self._chain()
1457 clone.query.set_empty()
1458 return clone
1460 ##################################################################
1461 # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET #
1462 ##################################################################
1464 def all(self):
1465 """
1466 Return a new QuerySet that is a copy of the current one. This allows a
1467 QuerySet to proxy for a model manager in some cases.
1468 """
1469 return self._chain()
1471 def filter(self, *args, **kwargs):
1472 """
1473 Return a new QuerySet instance with the args ANDed to the existing
1474 set.
1475 """
1476 self._not_support_combined_queries("filter")
1477 return self._filter_or_exclude(False, args, kwargs)
1479 def exclude(self, *args, **kwargs):
1480 """
1481 Return a new QuerySet instance with NOT (args) ANDed to the existing
1482 set.
1483 """
1484 self._not_support_combined_queries("exclude")
1485 return self._filter_or_exclude(True, args, kwargs)
1487 def _filter_or_exclude(self, negate, args, kwargs):
1488 if (args or kwargs) and self.query.is_sliced:
1489 raise TypeError("Cannot filter a query once a slice has been taken.")
1490 clone = self._chain()
1491 if self._defer_next_filter:
1492 self._defer_next_filter = False
1493 clone._deferred_filter = negate, args, kwargs
1494 else:
1495 clone._filter_or_exclude_inplace(negate, args, kwargs)
1496 return clone
1498 def _filter_or_exclude_inplace(self, negate, args, kwargs):
1499 if negate:
1500 self._query.add_q(~Q(*args, **kwargs))
1501 else:
1502 self._query.add_q(Q(*args, **kwargs))
1504 def complex_filter(self, filter_obj):
1505 """
1506 Return a new QuerySet instance with filter_obj added to the filters.
1508 filter_obj can be a Q object or a dictionary of keyword lookup
1509 arguments.
1511 This exists to support framework features such as 'limit_choices_to',
1512 and usually it will be more natural to use other methods.
1513 """
1514 if isinstance(filter_obj, Q):
1515 clone = self._chain()
1516 clone.query.add_q(filter_obj)
1517 return clone
1518 else:
1519 return self._filter_or_exclude(False, args=(), kwargs=filter_obj)
1521 def _combinator_query(self, combinator, *other_qs, all=False):
1522 # Clone the query to inherit the select list and everything
1523 clone = self._chain()
1524 # Clear limits and ordering so they can be reapplied
1525 clone.query.clear_ordering(force=True)
1526 clone.query.clear_limits()
1527 clone.query.combined_queries = (self.query,) + tuple(
1528 qs.query for qs in other_qs
1529 )
1530 clone.query.combinator = combinator
1531 clone.query.combinator_all = all
1532 return clone
1534 def union(self, *other_qs, all=False):
1535 # If the query is an EmptyQuerySet, combine all nonempty querysets.
1536 if isinstance(self, EmptyQuerySet):
1537 qs = [q for q in other_qs if not isinstance(q, EmptyQuerySet)]
1538 if not qs:
1539 return self
1540 if len(qs) == 1:
1541 return qs[0]
1542 return qs[0]._combinator_query("union", *qs[1:], all=all)
1543 return self._combinator_query("union", *other_qs, all=all)
1545 def intersection(self, *other_qs):
1546 # If any query is an EmptyQuerySet, return it.
1547 if isinstance(self, EmptyQuerySet):
1548 return self
1549 for other in other_qs:
1550 if isinstance(other, EmptyQuerySet):
1551 return other
1552 return self._combinator_query("intersection", *other_qs)
1554 def difference(self, *other_qs):
1555 # If the query is an EmptyQuerySet, return it.
1556 if isinstance(self, EmptyQuerySet):
1557 return self
1558 return self._combinator_query("difference", *other_qs)
1560 def select_for_update(self, nowait=False, skip_locked=False, of=(), no_key=False):
1561 """
1562 Return a new QuerySet instance that will select objects with a
1563 FOR UPDATE lock.
1564 """
1565 if nowait and skip_locked:
1566 raise ValueError("The nowait option cannot be used with skip_locked.")
1567 obj = self._chain()
1568 obj._for_write = True
1569 obj.query.select_for_update = True
1570 obj.query.select_for_update_nowait = nowait
1571 obj.query.select_for_update_skip_locked = skip_locked
1572 obj.query.select_for_update_of = of
1573 obj.query.select_for_no_key_update = no_key
1574 return obj
1576 def select_related(self, *fields):
1577 """
1578 Return a new QuerySet instance that will select related objects.
1580 If fields are specified, they must be ForeignKey fields and only those
1581 related objects are included in the selection.
1583 If select_related(None) is called, clear the list.
1584 """
1585 self._not_support_combined_queries("select_related")
1586 if self._fields is not None:
1587 raise TypeError(
1588 "Cannot call select_related() after .values() or .values_list()"
1589 )
1591 obj = self._chain()
1592 if fields == (None,):
1593 obj.query.select_related = False
1594 elif fields:
1595 obj.query.add_select_related(fields)
1596 else:
1597 obj.query.select_related = True
1598 return obj
1600 def prefetch_related(self, *lookups):
1601 """
1602 Return a new QuerySet instance that will prefetch the specified
1603 Many-To-One and Many-To-Many related objects when the QuerySet is
1604 evaluated.
1606 When prefetch_related() is called more than once, append to the list of
1607 prefetch lookups. If prefetch_related(None) is called, clear the list.
1608 """
1609 self._not_support_combined_queries("prefetch_related")
1610 clone = self._chain()
1611 if lookups == (None,):
1612 clone._prefetch_related_lookups = ()
1613 else:
1614 for lookup in lookups:
1615 if isinstance(lookup, Prefetch):
1616 lookup = lookup.prefetch_to
1617 lookup = lookup.split(LOOKUP_SEP, 1)[0]
1618 if lookup in self.query._filtered_relations:
1619 raise ValueError(
1620 "prefetch_related() is not supported with FilteredRelation."
1621 )
1622 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
1623 return clone
1625 def annotate(self, *args, **kwargs):
1626 """
1627 Return a query set in which the returned objects have been annotated
1628 with extra data or aggregations.
1629 """
1630 self._not_support_combined_queries("annotate")
1631 return self._annotate(args, kwargs, select=True)
1633 def alias(self, *args, **kwargs):
1634 """
1635 Return a query set with added aliases for extra data or aggregations.
1636 """
1637 self._not_support_combined_queries("alias")
1638 return self._annotate(args, kwargs, select=False)
1640 def _annotate(self, args, kwargs, select=True):
1641 self._validate_values_are_expressions(
1642 args + tuple(kwargs.values()), method_name="annotate"
1643 )
1644 annotations = {}
1645 for arg in args:
1646 # The default_alias property may raise a TypeError.
1647 try:
1648 if arg.default_alias in kwargs:
1649 raise ValueError(
1650 "The named annotation '%s' conflicts with the "
1651 "default name for another annotation." % arg.default_alias
1652 )
1653 except TypeError:
1654 raise TypeError("Complex annotations require an alias")
1655 annotations[arg.default_alias] = arg
1656 annotations.update(kwargs)
1658 clone = self._chain()
1659 names = self._fields
1660 if names is None:
1661 names = set(
1662 chain.from_iterable(
1663 (
1664 (field.name, field.attname)
1665 if hasattr(field, "attname")
1666 else (field.name,)
1667 )
1668 for field in self.model._meta.get_fields()
1669 )
1670 )
1672 for alias, annotation in annotations.items():
1673 if alias in names:
1674 raise ValueError(
1675 "The annotation '%s' conflicts with a field on "
1676 "the model." % alias
1677 )
1678 if isinstance(annotation, FilteredRelation):
1679 clone.query.add_filtered_relation(annotation, alias)
1680 else:
1681 clone.query.add_annotation(
1682 annotation,
1683 alias,
1684 select=select,
1685 )
1686 for alias, annotation in clone.query.annotations.items():
1687 if alias in annotations and annotation.contains_aggregate:
1688 if clone._fields is None:
1689 clone.query.group_by = True
1690 else:
1691 clone.query.set_group_by()
1692 break
1694 return clone
1696 def order_by(self, *field_names):
1697 """Return a new QuerySet instance with the ordering changed."""
1698 if self.query.is_sliced:
1699 raise TypeError("Cannot reorder a query once a slice has been taken.")
1700 obj = self._chain()
1701 obj.query.clear_ordering(force=True, clear_default=False)
1702 obj.query.add_ordering(*field_names)
1703 return obj
1705 def distinct(self, *field_names):
1706 """
1707 Return a new QuerySet instance that will select only distinct results.
1708 """
1709 self._not_support_combined_queries("distinct")
1710 if self.query.is_sliced:
1711 raise TypeError(
1712 "Cannot create distinct fields once a slice has been taken."
1713 )
1714 obj = self._chain()
1715 obj.query.add_distinct_fields(*field_names)
1716 return obj
1718 def extra(
1719 self,
1720 select=None,
1721 where=None,
1722 params=None,
1723 tables=None,
1724 order_by=None,
1725 select_params=None,
1726 ):
1727 """Add extra SQL fragments to the query."""
1728 self._not_support_combined_queries("extra")
1729 if self.query.is_sliced:
1730 raise TypeError("Cannot change a query once a slice has been taken.")
1731 clone = self._chain()
1732 clone.query.add_extra(select, select_params, where, params, tables, order_by)
1733 return clone
1735 def reverse(self):
1736 """Reverse the ordering of the QuerySet."""
1737 if self.query.is_sliced:
1738 raise TypeError("Cannot reverse a query once a slice has been taken.")
1739 clone = self._chain()
1740 clone.query.standard_ordering = not clone.query.standard_ordering
1741 return clone
1743 def defer(self, *fields):
1744 """
1745 Defer the loading of data for certain fields until they are accessed.
1746 Add the set of deferred fields to any existing set of deferred fields.
1747 The only exception to this is if None is passed in as the only
1748 parameter, in which case remove all deferrals.
1749 """
1750 self._not_support_combined_queries("defer")
1751 if self._fields is not None:
1752 raise TypeError("Cannot call defer() after .values() or .values_list()")
1753 clone = self._chain()
1754 if fields == (None,):
1755 clone.query.clear_deferred_loading()
1756 else:
1757 clone.query.add_deferred_loading(fields)
1758 return clone
1760 def only(self, *fields):
1761 """
1762 Essentially, the opposite of defer(). Only the fields passed into this
1763 method and that are not already specified as deferred are loaded
1764 immediately when the queryset is evaluated.
1765 """
1766 self._not_support_combined_queries("only")
1767 if self._fields is not None:
1768 raise TypeError("Cannot call only() after .values() or .values_list()")
1769 if fields == (None,):
1770 # Can only pass None to defer(), not only(), as the rest option.
1771 # That won't stop people trying to do this, so let's be explicit.
1772 raise TypeError("Cannot pass None as an argument to only().")
1773 for field in fields:
1774 field = field.split(LOOKUP_SEP, 1)[0]
1775 if field in self.query._filtered_relations:
1776 raise ValueError("only() is not supported with FilteredRelation.")
1777 clone = self._chain()
1778 clone.query.add_immediate_loading(fields)
1779 return clone
1781 def using(self, alias):
1782 """Select which database this QuerySet should execute against."""
1783 clone = self._chain()
1784 clone._db = alias
1785 return clone
1787 ###################################
1788 # PUBLIC INTROSPECTION ATTRIBUTES #
1789 ###################################
1791 @property
1792 def ordered(self):
1793 """
1794 Return True if the QuerySet is ordered -- i.e. has an order_by()
1795 clause or a default ordering on the model (or is empty).
1796 """
1797 if isinstance(self, EmptyQuerySet):
1798 return True
1799 if self.query.extra_order_by or self.query.order_by:
1800 return True
1801 elif (
1802 self.query.default_ordering
1803 and self.query.get_meta().ordering
1804 and
1805 # A default ordering doesn't affect GROUP BY queries.
1806 not self.query.group_by
1807 ):
1808 return True
1809 else:
1810 return False
1812 @property
1813 def db(self):
1814 """Return the database used if this query is executed now."""
1815 if self._for_write:
1816 return self._db or router.db_for_write(self.model, **self._hints)
1817 return self._db or router.db_for_read(self.model, **self._hints)
1819 ###################
1820 # PRIVATE METHODS #
1821 ###################
1823 def _insert(
1824 self,
1825 objs,
1826 fields,
1827 returning_fields=None,
1828 raw=False,
1829 using=None,
1830 on_conflict=None,
1831 update_fields=None,
1832 unique_fields=None,
1833 ):
1834 """
1835 Insert a new record for the given model. This provides an interface to
1836 the InsertQuery class and is how Model.save() is implemented.
1837 """
1838 self._for_write = True
1839 if using is None:
1840 using = self.db
1841 query = sql.InsertQuery(
1842 self.model,
1843 on_conflict=on_conflict,
1844 update_fields=update_fields,
1845 unique_fields=unique_fields,
1846 )
1847 query.insert_values(fields, objs, raw=raw)
1848 return query.get_compiler(using=using).execute_sql(returning_fields)
1850 _insert.alters_data = True
1851 _insert.queryset_only = False
1853 def _batched_insert(
1854 self,
1855 objs,
1856 fields,
1857 batch_size,
1858 on_conflict=None,
1859 update_fields=None,
1860 unique_fields=None,
1861 ):
1862 """
1863 Helper method for bulk_create() to insert objs one batch at a time.
1864 """
1865 connection = connections[self.db]
1866 ops = connection.ops
1867 max_batch_size = max(ops.bulk_batch_size(fields, objs), 1)
1868 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
1869 inserted_rows = []
1870 bulk_return = connection.features.can_return_rows_from_bulk_insert
1871 for item in [objs[i : i + batch_size] for i in range(0, len(objs), batch_size)]:
1872 if bulk_return and (
1873 on_conflict is None or on_conflict == OnConflict.UPDATE
1874 ):
1875 inserted_rows.extend(
1876 self._insert(
1877 item,
1878 fields=fields,
1879 using=self.db,
1880 on_conflict=on_conflict,
1881 update_fields=update_fields,
1882 unique_fields=unique_fields,
1883 returning_fields=self.model._meta.db_returning_fields,
1884 )
1885 )
1886 else:
1887 self._insert(
1888 item,
1889 fields=fields,
1890 using=self.db,
1891 on_conflict=on_conflict,
1892 update_fields=update_fields,
1893 unique_fields=unique_fields,
1894 )
1895 return inserted_rows
1897 def _chain(self):
1898 """
1899 Return a copy of the current QuerySet that's ready for another
1900 operation.
1901 """
1902 obj = self._clone()
1903 if obj._sticky_filter:
1904 obj.query.filter_is_sticky = True
1905 obj._sticky_filter = False
1906 return obj
1908 def _clone(self):
1909 """
1910 Return a copy of the current QuerySet. A lightweight alternative
1911 to deepcopy().
1912 """
1913 c = self.__class__(
1914 model=self.model,
1915 query=self.query.chain(),
1916 using=self._db,
1917 hints=self._hints,
1918 )
1919 c._sticky_filter = self._sticky_filter
1920 c._for_write = self._for_write
1921 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
1922 c._known_related_objects = self._known_related_objects
1923 c._iterable_class = self._iterable_class
1924 c._fields = self._fields
1925 return c
1927 def _fetch_all(self):
1928 if self._result_cache is None:
1929 self._result_cache = list(self._iterable_class(self))
1930 if self._prefetch_related_lookups and not self._prefetch_done:
1931 self._prefetch_related_objects()
1933 def _next_is_sticky(self):
1934 """
1935 Indicate that the next filter call and the one following that should
1936 be treated as a single filter. This is only important when it comes to
1937 determining when to reuse tables for many-to-many filters. Required so
1938 that we can filter naturally on the results of related managers.
1940 This doesn't return a clone of the current QuerySet (it returns
1941 "self"). The method is only used internally and should be immediately
1942 followed by a filter() that does create a clone.
1943 """
1944 self._sticky_filter = True
1945 return self
1947 def _merge_sanity_check(self, other):
1948 """Check that two QuerySet classes may be merged."""
1949 if self._fields is not None and (
1950 set(self.query.values_select) != set(other.query.values_select)
1951 or set(self.query.extra_select) != set(other.query.extra_select)
1952 or set(self.query.annotation_select) != set(other.query.annotation_select)
1953 ):
1954 raise TypeError(
1955 "Merging '%s' classes must involve the same values in each case."
1956 % self.__class__.__name__
1957 )
1959 def _merge_known_related_objects(self, other):
1960 """
1961 Keep track of all known related objects from either QuerySet instance.
1962 """
1963 for field, objects in other._known_related_objects.items():
1964 self._known_related_objects.setdefault(field, {}).update(objects)
1966 def resolve_expression(self, *args, **kwargs):
1967 if self._fields and len(self._fields) > 1:
1968 # values() queryset can only be used as nested queries
1969 # if they are set up to select only a single field.
1970 raise TypeError("Cannot use multi-field values as a filter value.")
1971 query = self.query.resolve_expression(*args, **kwargs)
1972 query._db = self._db
1973 return query
1975 resolve_expression.queryset_only = True
1977 def _add_hints(self, **hints):
1978 """
1979 Update hinting information for use by routers. Add new key/values or
1980 overwrite existing key/values.
1981 """
1982 self._hints.update(hints)
1984 def _has_filters(self):
1985 """
1986 Check if this QuerySet has any filtering going on. This isn't
1987 equivalent with checking if all objects are present in results, for
1988 example, qs[1:]._has_filters() -> False.
1989 """
1990 return self.query.has_filters()
1992 @staticmethod
1993 def _validate_values_are_expressions(values, method_name):
1994 invalid_args = sorted(
1995 str(arg) for arg in values if not hasattr(arg, "resolve_expression")
1996 )
1997 if invalid_args:
1998 raise TypeError(
1999 "QuerySet.%s() received non-expression(s): %s."
2000 % (
2001 method_name,
2002 ", ".join(invalid_args),
2003 )
2004 )
2006 def _not_support_combined_queries(self, operation_name):
2007 if self.query.combinator:
2008 raise NotSupportedError(
2009 "Calling QuerySet.%s() after %s() is not supported."
2010 % (operation_name, self.query.combinator)
2011 )
2013 def _check_operator_queryset(self, other, operator_):
2014 if self.query.combinator or other.query.combinator:
2015 raise TypeError(f"Cannot use {operator_} operator with combined queryset.")
2017 def _check_ordering_first_last_queryset_aggregation(self, method):
2018 if isinstance(self.query.group_by, tuple) and not any(
2019 col.output_field is self.model._meta.pk for col in self.query.group_by
2020 ):
2021 raise TypeError(
2022 f"Cannot use QuerySet.{method}() on an unordered queryset performing "
2023 f"aggregation. Add an ordering with order_by()."
2024 )
2027class InstanceCheckMeta(type):
2028 def __instancecheck__(self, instance):
2029 return isinstance(instance, QuerySet) and instance.query.is_empty()
2032class EmptyQuerySet(metaclass=InstanceCheckMeta):
2033 """
2034 Marker class to checking if a queryset is empty by .none():
2035 isinstance(qs.none(), EmptyQuerySet) -> True
2036 """
2038 def __init__(self, *args, **kwargs):
2039 raise TypeError("EmptyQuerySet can't be instantiated")
2042class RawQuerySet:
2043 """
2044 Provide an iterator which converts the results of raw SQL queries into
2045 annotated model instances.
2046 """
2048 def __init__(
2049 self,
2050 raw_query,
2051 model=None,
2052 query=None,
2053 params=(),
2054 translations=None,
2055 using=None,
2056 hints=None,
2057 ):
2058 self.raw_query = raw_query
2059 self.model = model
2060 self._db = using
2061 self._hints = hints or {}
2062 self.query = query or sql.RawQuery(sql=raw_query, using=self.db, params=params)
2063 self.params = params
2064 self.translations = translations or {}
2065 self._result_cache = None
2066 self._prefetch_related_lookups = ()
2067 self._prefetch_done = False
2069 def resolve_model_init_order(self):
2070 """Resolve the init field names and value positions."""
2071 converter = connections[self.db].introspection.identifier_converter
2072 model_init_fields = [
2073 f for f in self.model._meta.fields if converter(f.column) in self.columns
2074 ]
2075 annotation_fields = [
2076 (column, pos)
2077 for pos, column in enumerate(self.columns)
2078 if column not in self.model_fields
2079 ]
2080 model_init_order = [
2081 self.columns.index(converter(f.column)) for f in model_init_fields
2082 ]
2083 model_init_names = [f.attname for f in model_init_fields]
2084 return model_init_names, model_init_order, annotation_fields
2086 def prefetch_related(self, *lookups):
2087 """Same as QuerySet.prefetch_related()"""
2088 clone = self._clone()
2089 if lookups == (None,):
2090 clone._prefetch_related_lookups = ()
2091 else:
2092 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
2093 return clone
2095 def _prefetch_related_objects(self):
2096 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
2097 self._prefetch_done = True
2099 def _clone(self):
2100 """Same as QuerySet._clone()"""
2101 c = self.__class__(
2102 self.raw_query,
2103 model=self.model,
2104 query=self.query,
2105 params=self.params,
2106 translations=self.translations,
2107 using=self._db,
2108 hints=self._hints,
2109 )
2110 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
2111 return c
2113 def _fetch_all(self):
2114 if self._result_cache is None:
2115 self._result_cache = list(self.iterator())
2116 if self._prefetch_related_lookups and not self._prefetch_done:
2117 self._prefetch_related_objects()
2119 def __len__(self):
2120 self._fetch_all()
2121 return len(self._result_cache)
2123 def __bool__(self):
2124 self._fetch_all()
2125 return bool(self._result_cache)
2127 def __iter__(self):
2128 self._fetch_all()
2129 return iter(self._result_cache)
2131 def __aiter__(self):
2132 # Remember, __aiter__ itself is synchronous, it's the thing it returns
2133 # that is async!
2134 async def generator():
2135 await sync_to_async(self._fetch_all)()
2136 for item in self._result_cache:
2137 yield item
2139 return generator()
2141 def iterator(self):
2142 yield from RawModelIterable(self)
2144 def __repr__(self):
2145 return "<%s: %s>" % (self.__class__.__name__, self.query)
2147 def __getitem__(self, k):
2148 return list(self)[k]
2150 @property
2151 def db(self):
2152 """Return the database used if this query is executed now."""
2153 return self._db or router.db_for_read(self.model, **self._hints)
2155 def using(self, alias):
2156 """Select the database this RawQuerySet should execute against."""
2157 return RawQuerySet(
2158 self.raw_query,
2159 model=self.model,
2160 query=self.query.chain(using=alias),
2161 params=self.params,
2162 translations=self.translations,
2163 using=alias,
2164 )
2166 @cached_property
2167 def columns(self):
2168 """
2169 A list of model field names in the order they'll appear in the
2170 query results.
2171 """
2172 columns = self.query.get_columns()
2173 # Adjust any column names which don't match field names
2174 for query_name, model_name in self.translations.items():
2175 # Ignore translations for nonexistent column names
2176 try:
2177 index = columns.index(query_name)
2178 except ValueError:
2179 pass
2180 else:
2181 columns[index] = model_name
2182 return columns
2184 @cached_property
2185 def model_fields(self):
2186 """A dict mapping column names to model field names."""
2187 converter = connections[self.db].introspection.identifier_converter
2188 model_fields = {}
2189 for field in self.model._meta.fields:
2190 model_fields[converter(field.column)] = field
2191 return model_fields
2194class Prefetch:
2195 def __init__(self, lookup, queryset=None, to_attr=None):
2196 # `prefetch_through` is the path we traverse to perform the prefetch.
2197 self.prefetch_through = lookup
2198 # `prefetch_to` is the path to the attribute that stores the result.
2199 self.prefetch_to = lookup
2200 if queryset is not None and (
2201 isinstance(queryset, RawQuerySet)
2202 or (
2203 hasattr(queryset, "_iterable_class")
2204 and not issubclass(queryset._iterable_class, ModelIterable)
2205 )
2206 ):
2207 raise ValueError(
2208 "Prefetch querysets cannot use raw(), values(), and values_list()."
2209 )
2210 if to_attr:
2211 self.prefetch_to = LOOKUP_SEP.join(
2212 lookup.split(LOOKUP_SEP)[:-1] + [to_attr]
2213 )
2215 self.queryset = queryset
2216 self.to_attr = to_attr
2218 def __getstate__(self):
2219 obj_dict = self.__dict__.copy()
2220 if self.queryset is not None:
2221 queryset = self.queryset._chain()
2222 # Prevent the QuerySet from being evaluated
2223 queryset._result_cache = []
2224 queryset._prefetch_done = True
2225 obj_dict["queryset"] = queryset
2226 return obj_dict
2228 def add_prefix(self, prefix):
2229 self.prefetch_through = prefix + LOOKUP_SEP + self.prefetch_through
2230 self.prefetch_to = prefix + LOOKUP_SEP + self.prefetch_to
2232 def get_current_prefetch_to(self, level):
2233 return LOOKUP_SEP.join(self.prefetch_to.split(LOOKUP_SEP)[: level + 1])
2235 def get_current_to_attr(self, level):
2236 parts = self.prefetch_to.split(LOOKUP_SEP)
2237 to_attr = parts[level]
2238 as_attr = self.to_attr and level == len(parts) - 1
2239 return to_attr, as_attr
2241 def get_current_querysets(self, level):
2242 if (
2243 self.get_current_prefetch_to(level) == self.prefetch_to
2244 and self.queryset is not None
2245 ):
2246 return [self.queryset]
2247 return None
2249 def __eq__(self, other):
2250 if not isinstance(other, Prefetch):
2251 return NotImplemented
2252 return self.prefetch_to == other.prefetch_to
2254 def __hash__(self):
2255 return hash((self.__class__, self.prefetch_to))
2258def normalize_prefetch_lookups(lookups, prefix=None):
2259 """Normalize lookups into Prefetch objects."""
2260 ret = []
2261 for lookup in lookups:
2262 if not isinstance(lookup, Prefetch):
2263 lookup = Prefetch(lookup)
2264 if prefix:
2265 lookup.add_prefix(prefix)
2266 ret.append(lookup)
2267 return ret
2270def prefetch_related_objects(model_instances, *related_lookups):
2271 """
2272 Populate prefetched object caches for a list of model instances based on
2273 the lookups/Prefetch instances given.
2274 """
2275 if not model_instances:
2276 return # nothing to do
2278 # We need to be able to dynamically add to the list of prefetch_related
2279 # lookups that we look up (see below). So we need some book keeping to
2280 # ensure we don't do duplicate work.
2281 done_queries = {} # dictionary of things like 'foo__bar': [results]
2283 auto_lookups = set() # we add to this as we go through.
2284 followed_descriptors = set() # recursion protection
2286 all_lookups = normalize_prefetch_lookups(reversed(related_lookups))
2287 while all_lookups:
2288 lookup = all_lookups.pop()
2289 if lookup.prefetch_to in done_queries:
2290 if lookup.queryset is not None:
2291 raise ValueError(
2292 "'%s' lookup was already seen with a different queryset. "
2293 "You may need to adjust the ordering of your lookups."
2294 % lookup.prefetch_to
2295 )
2297 continue
2299 # Top level, the list of objects to decorate is the result cache
2300 # from the primary QuerySet. It won't be for deeper levels.
2301 obj_list = model_instances
2303 through_attrs = lookup.prefetch_through.split(LOOKUP_SEP)
2304 for level, through_attr in enumerate(through_attrs):
2305 # Prepare main instances
2306 if not obj_list:
2307 break
2309 prefetch_to = lookup.get_current_prefetch_to(level)
2310 if prefetch_to in done_queries:
2311 # Skip any prefetching, and any object preparation
2312 obj_list = done_queries[prefetch_to]
2313 continue
2315 # Prepare objects:
2316 good_objects = True
2317 for obj in obj_list:
2318 # Since prefetching can re-use instances, it is possible to have
2319 # the same instance multiple times in obj_list, so obj might
2320 # already be prepared.
2321 if not hasattr(obj, "_prefetched_objects_cache"):
2322 try:
2323 obj._prefetched_objects_cache = {}
2324 except (AttributeError, TypeError):
2325 # Must be an immutable object from
2326 # values_list(flat=True), for example (TypeError) or
2327 # a QuerySet subclass that isn't returning Model
2328 # instances (AttributeError), either in Django or a 3rd
2329 # party. prefetch_related() doesn't make sense, so quit.
2330 good_objects = False
2331 break
2332 if not good_objects:
2333 break
2335 # Descend down tree
2337 # We assume that objects retrieved are homogeneous (which is the premise
2338 # of prefetch_related), so what applies to first object applies to all.
2339 first_obj = obj_list[0]
2340 to_attr = lookup.get_current_to_attr(level)[0]
2341 prefetcher, descriptor, attr_found, is_fetched = get_prefetcher(
2342 first_obj, through_attr, to_attr
2343 )
2345 if not attr_found:
2346 raise AttributeError(
2347 "Cannot find '%s' on %s object, '%s' is an invalid "
2348 "parameter to prefetch_related()"
2349 % (
2350 through_attr,
2351 first_obj.__class__.__name__,
2352 lookup.prefetch_through,
2353 )
2354 )
2356 if level == len(through_attrs) - 1 and prefetcher is None:
2357 # Last one, this *must* resolve to something that supports
2358 # prefetching, otherwise there is no point adding it and the
2359 # developer asking for it has made a mistake.
2360 raise ValueError(
2361 "'%s' does not resolve to an item that supports "
2362 "prefetching - this is an invalid parameter to "
2363 "prefetch_related()." % lookup.prefetch_through
2364 )
2366 obj_to_fetch = None
2367 if prefetcher is not None:
2368 obj_to_fetch = [obj for obj in obj_list if not is_fetched(obj)]
2370 if obj_to_fetch:
2371 obj_list, additional_lookups = prefetch_one_level(
2372 obj_to_fetch,
2373 prefetcher,
2374 lookup,
2375 level,
2376 )
2377 # We need to ensure we don't keep adding lookups from the
2378 # same relationships to stop infinite recursion. So, if we
2379 # are already on an automatically added lookup, don't add
2380 # the new lookups from relationships we've seen already.
2381 if not (
2382 prefetch_to in done_queries
2383 and lookup in auto_lookups
2384 and descriptor in followed_descriptors
2385 ):
2386 done_queries[prefetch_to] = obj_list
2387 new_lookups = normalize_prefetch_lookups(
2388 reversed(additional_lookups), prefetch_to
2389 )
2390 auto_lookups.update(new_lookups)
2391 all_lookups.extend(new_lookups)
2392 followed_descriptors.add(descriptor)
2393 else:
2394 # Either a singly related object that has already been fetched
2395 # (e.g. via select_related), or hopefully some other property
2396 # that doesn't support prefetching but needs to be traversed.
2398 # We replace the current list of parent objects with the list
2399 # of related objects, filtering out empty or missing values so
2400 # that we can continue with nullable or reverse relations.
2401 new_obj_list = []
2402 for obj in obj_list:
2403 if through_attr in getattr(obj, "_prefetched_objects_cache", ()):
2404 # If related objects have been prefetched, use the
2405 # cache rather than the object's through_attr.
2406 new_obj = list(obj._prefetched_objects_cache.get(through_attr))
2407 else:
2408 try:
2409 new_obj = getattr(obj, through_attr)
2410 except exceptions.ObjectDoesNotExist:
2411 continue
2412 if new_obj is None:
2413 continue
2414 # We special-case `list` rather than something more generic
2415 # like `Iterable` because we don't want to accidentally match
2416 # user models that define __iter__.
2417 if isinstance(new_obj, list):
2418 new_obj_list.extend(new_obj)
2419 else:
2420 new_obj_list.append(new_obj)
2421 obj_list = new_obj_list
2424async def aprefetch_related_objects(model_instances, *related_lookups):
2425 """See prefetch_related_objects()."""
2426 return await sync_to_async(prefetch_related_objects)(
2427 model_instances, *related_lookups
2428 )
2431def get_prefetcher(instance, through_attr, to_attr):
2432 """
2433 For the attribute 'through_attr' on the given instance, find
2434 an object that has a get_prefetch_querysets().
2435 Return a 4 tuple containing:
2436 (the object with get_prefetch_querysets (or None),
2437 the descriptor object representing this relationship (or None),
2438 a boolean that is False if the attribute was not found at all,
2439 a function that takes an instance and returns a boolean that is True if
2440 the attribute has already been fetched for that instance)
2441 """
2443 def is_to_attr_fetched(model, to_attr):
2444 # Special case cached_property instances because hasattr() triggers
2445 # attribute computation and assignment.
2446 if isinstance(getattr(model, to_attr, None), cached_property):
2448 def has_cached_property(instance):
2449 return to_attr in instance.__dict__
2451 return has_cached_property
2453 def has_to_attr_attribute(instance):
2454 return hasattr(instance, to_attr)
2456 return has_to_attr_attribute
2458 prefetcher = None
2459 is_fetched = is_to_attr_fetched(instance.__class__, to_attr)
2461 # For singly related objects, we have to avoid getting the attribute
2462 # from the object, as this will trigger the query. So we first try
2463 # on the class, in order to get the descriptor object.
2464 rel_obj_descriptor = getattr(instance.__class__, through_attr, None)
2465 if rel_obj_descriptor is None:
2466 attr_found = hasattr(instance, through_attr)
2467 else:
2468 attr_found = True
2469 if rel_obj_descriptor:
2470 # singly related object, descriptor object has the
2471 # get_prefetch_querysets() method.
2472 if hasattr(rel_obj_descriptor, "get_prefetch_querysets"):
2473 prefetcher = rel_obj_descriptor
2474 # If to_attr is set, check if the value has already been set,
2475 # which is done with has_to_attr_attribute(). Do not use the
2476 # method from the descriptor, as the cache_name it defines
2477 # checks the field name, not the to_attr value.
2478 if through_attr == to_attr:
2479 is_fetched = rel_obj_descriptor.is_cached
2480 else:
2481 # descriptor doesn't support prefetching, so we go ahead and get
2482 # the attribute on the instance rather than the class to
2483 # support many related managers
2484 rel_obj = getattr(instance, through_attr)
2485 if hasattr(rel_obj, "get_prefetch_querysets"):
2486 prefetcher = rel_obj
2487 if through_attr == to_attr:
2489 def in_prefetched_cache(instance):
2490 return through_attr in instance._prefetched_objects_cache
2492 is_fetched = in_prefetched_cache
2493 return prefetcher, rel_obj_descriptor, attr_found, is_fetched
2496def prefetch_one_level(instances, prefetcher, lookup, level):
2497 """
2498 Helper function for prefetch_related_objects().
2500 Run prefetches on all instances using the prefetcher object,
2501 assigning results to relevant caches in instance.
2503 Return the prefetched objects along with any additional prefetches that
2504 must be done due to prefetch_related lookups found from default managers.
2505 """
2506 # prefetcher must have a method get_prefetch_querysets() which takes a list
2507 # of instances, and returns a tuple:
2509 # (queryset of instances of self.model that are related to passed in instances,
2510 # callable that gets value to be matched for returned instances,
2511 # callable that gets value to be matched for passed in instances,
2512 # boolean that is True for singly related objects,
2513 # cache or field name to assign to,
2514 # boolean that is True when the previous argument is a cache name vs a field name).
2516 # The 'values to be matched' must be hashable as they will be used
2517 # in a dictionary.
2518 (
2519 rel_qs,
2520 rel_obj_attr,
2521 instance_attr,
2522 single,
2523 cache_name,
2524 is_descriptor,
2525 ) = prefetcher.get_prefetch_querysets(
2526 instances, lookup.get_current_querysets(level)
2527 )
2528 # We have to handle the possibility that the QuerySet we just got back
2529 # contains some prefetch_related lookups. We don't want to trigger the
2530 # prefetch_related functionality by evaluating the query. Rather, we need
2531 # to merge in the prefetch_related lookups.
2532 # Copy the lookups in case it is a Prefetch object which could be reused
2533 # later (happens in nested prefetch_related).
2534 additional_lookups = [
2535 copy.copy(additional_lookup)
2536 for additional_lookup in getattr(rel_qs, "_prefetch_related_lookups", ())
2537 ]
2538 if additional_lookups:
2539 # Don't need to clone because the manager should have given us a fresh
2540 # instance, so we access an internal instead of using public interface
2541 # for performance reasons.
2542 rel_qs._prefetch_related_lookups = ()
2544 all_related_objects = list(rel_qs)
2546 rel_obj_cache = {}
2547 for rel_obj in all_related_objects:
2548 rel_attr_val = rel_obj_attr(rel_obj)
2549 rel_obj_cache.setdefault(rel_attr_val, []).append(rel_obj)
2551 to_attr, as_attr = lookup.get_current_to_attr(level)
2552 # Make sure `to_attr` does not conflict with a field.
2553 if as_attr and instances:
2554 # We assume that objects retrieved are homogeneous (which is the premise
2555 # of prefetch_related), so what applies to first object applies to all.
2556 model = instances[0].__class__
2557 try:
2558 model._meta.get_field(to_attr)
2559 except exceptions.FieldDoesNotExist:
2560 pass
2561 else:
2562 msg = "to_attr={} conflicts with a field on the {} model."
2563 raise ValueError(msg.format(to_attr, model.__name__))
2565 # Whether or not we're prefetching the last part of the lookup.
2566 leaf = len(lookup.prefetch_through.split(LOOKUP_SEP)) - 1 == level
2568 for obj in instances:
2569 instance_attr_val = instance_attr(obj)
2570 vals = rel_obj_cache.get(instance_attr_val, [])
2572 if single:
2573 val = vals[0] if vals else None
2574 if as_attr:
2575 # A to_attr has been given for the prefetch.
2576 setattr(obj, to_attr, val)
2577 elif is_descriptor:
2578 # cache_name points to a field name in obj.
2579 # This field is a descriptor for a related object.
2580 setattr(obj, cache_name, val)
2581 else:
2582 # No to_attr has been given for this prefetch operation and the
2583 # cache_name does not point to a descriptor. Store the value of
2584 # the field in the object's field cache.
2585 obj._state.fields_cache[cache_name] = val
2586 else:
2587 if as_attr:
2588 setattr(obj, to_attr, vals)
2589 else:
2590 manager = getattr(obj, to_attr)
2591 if leaf and lookup.queryset is not None:
2592 qs = manager._apply_rel_filters(lookup.queryset)
2593 else:
2594 qs = manager.get_queryset()
2595 qs._result_cache = vals
2596 # We don't want the individual qs doing prefetch_related now,
2597 # since we have merged this into the current work.
2598 qs._prefetch_done = True
2599 obj._prefetched_objects_cache[cache_name] = qs
2600 return all_related_objects, additional_lookups
2603class RelatedPopulator:
2604 """
2605 RelatedPopulator is used for select_related() object instantiation.
2607 The idea is that each select_related() model will be populated by a
2608 different RelatedPopulator instance. The RelatedPopulator instances get
2609 klass_info and select (computed in SQLCompiler) plus the used db as
2610 input for initialization. That data is used to compute which columns
2611 to use, how to instantiate the model, and how to populate the links
2612 between the objects.
2614 The actual creation of the objects is done in populate() method. This
2615 method gets row and from_obj as input and populates the select_related()
2616 model instance.
2617 """
2619 def __init__(self, klass_info, select, db):
2620 self.db = db
2621 # Pre-compute needed attributes. The attributes are:
2622 # - model_cls: the possibly deferred model class to instantiate
2623 # - either:
2624 # - cols_start, cols_end: usually the columns in the row are
2625 # in the same order model_cls.__init__ expects them, so we
2626 # can instantiate by model_cls(*row[cols_start:cols_end])
2627 # - reorder_for_init: When select_related descends to a child
2628 # class, then we want to reuse the already selected parent
2629 # data. However, in this case the parent data isn't necessarily
2630 # in the same order that Model.__init__ expects it to be, so
2631 # we have to reorder the parent data. The reorder_for_init
2632 # attribute contains a function used to reorder the field data
2633 # in the order __init__ expects it.
2634 # - pk_idx: the index of the primary key field in the reordered
2635 # model data. Used to check if a related object exists at all.
2636 # - init_list: the field attnames fetched from the database. For
2637 # deferred models this isn't the same as all attnames of the
2638 # model's fields.
2639 # - related_populators: a list of RelatedPopulator instances if
2640 # select_related() descends to related models from this model.
2641 # - local_setter, remote_setter: Methods to set cached values on
2642 # the object being populated and on the remote object. Usually
2643 # these are Field.set_cached_value() methods.
2644 select_fields = klass_info["select_fields"]
2645 from_parent = klass_info["from_parent"]
2646 if not from_parent:
2647 self.cols_start = select_fields[0]
2648 self.cols_end = select_fields[-1] + 1
2649 self.init_list = [
2650 f[0].target.attname for f in select[self.cols_start : self.cols_end]
2651 ]
2652 self.reorder_for_init = None
2653 else:
2654 attname_indexes = {
2655 select[idx][0].target.attname: idx for idx in select_fields
2656 }
2657 model_init_attnames = (
2658 f.attname for f in klass_info["model"]._meta.concrete_fields
2659 )
2660 self.init_list = [
2661 attname for attname in model_init_attnames if attname in attname_indexes
2662 ]
2663 self.reorder_for_init = operator.itemgetter(
2664 *[attname_indexes[attname] for attname in self.init_list]
2665 )
2667 self.model_cls = klass_info["model"]
2668 self.pk_idx = self.init_list.index(self.model_cls._meta.pk.attname)
2669 self.related_populators = get_related_populators(klass_info, select, self.db)
2670 self.local_setter = klass_info["local_setter"]
2671 self.remote_setter = klass_info["remote_setter"]
2673 def populate(self, row, from_obj):
2674 if self.reorder_for_init:
2675 obj_data = self.reorder_for_init(row)
2676 else:
2677 obj_data = row[self.cols_start : self.cols_end]
2678 if obj_data[self.pk_idx] is None:
2679 obj = None
2680 else:
2681 obj = self.model_cls.from_db(self.db, self.init_list, obj_data)
2682 for rel_iter in self.related_populators:
2683 rel_iter.populate(row, obj)
2684 self.local_setter(from_obj, obj)
2685 if obj is not None:
2686 self.remote_setter(obj, from_obj)
2689def get_related_populators(klass_info, select, db):
2690 iterators = []
2691 related_klass_infos = klass_info.get("related_klass_infos", [])
2692 for rel_klass_info in related_klass_infos:
2693 rel_cls = RelatedPopulator(rel_klass_info, select, db)
2694 iterators.append(rel_cls)
2695 return iterators