Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/sqlalchemy/sql/crud.py: 18%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# sql/crud.py
2# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
3# <see AUTHORS file>
4#
5# This module is part of SQLAlchemy and is released under
6# the MIT License: https://www.opensource.org/licenses/mit-license.php
7# mypy: allow-untyped-defs, allow-untyped-calls
9"""Functions used by compiler.py to determine the parameters rendered
10within INSERT and UPDATE statements.
12"""
14from __future__ import annotations
16import functools
17import operator
18import re
19from typing import Any
20from typing import Callable
21from typing import cast
22from typing import Dict
23from typing import Iterable
24from typing import List
25from typing import Literal
26from typing import MutableMapping
27from typing import NamedTuple
28from typing import Optional
29from typing import overload
30from typing import Sequence
31from typing import Set
32from typing import Tuple
33from typing import TYPE_CHECKING
34from typing import Union
36from . import coercions
37from . import dml
38from . import elements
39from . import roles
40from .base import _DefaultDescriptionTuple
41from .dml import isinsert as _compile_state_isinsert
42from .elements import ColumnClause
43from .schema import default_is_clause_element
44from .schema import default_is_sequence
45from .selectable import Select
46from .selectable import TableClause
47from .. import exc
48from .. import util
50if TYPE_CHECKING:
51 from .compiler import _BindNameForColProtocol
52 from .compiler import SQLCompiler
53 from .dml import _DMLColumnElement
54 from .dml import DMLState
55 from .dml import ValuesBase
56 from .elements import ColumnElement
57 from .elements import DMLTargetCopy
58 from .elements import KeyedColumnElement
59 from .schema import _SQLExprDefault
60 from .schema import Column
62REQUIRED = util.symbol(
63 "REQUIRED",
64 """
65Placeholder for the value within a :class:`.BindParameter`
66which is required to be present when the statement is passed
67to :meth:`_engine.Connection.execute`.
69This symbol is typically used when a :func:`_expression.insert`
70or :func:`_expression.update` statement is compiled without parameter
71values present.
73""",
74)
77def _as_dml_column(c: ColumnElement[Any]) -> ColumnClause[Any]:
78 if not isinstance(c, ColumnClause):
79 raise exc.CompileError(
80 f"Can't create DML statement against column expression {c!r}"
81 )
82 return c
85_CrudParamElement = Tuple[
86 "ColumnElement[Any]",
87 str, # column name
88 Optional[
89 Union[str, "_SQLExprDefault"]
90 ], # bound parameter string or SQL expression to apply
91 Iterable[str],
92]
93_CrudParamElementStr = Tuple[
94 "KeyedColumnElement[Any]",
95 str, # column name
96 str, # bound parameter string
97 Iterable[str],
98]
99_CrudParamElementSQLExpr = Tuple[
100 "ColumnClause[Any]",
101 str,
102 "_SQLExprDefault", # SQL expression to apply
103 Iterable[str],
104]
106_CrudParamSequence = List[_CrudParamElement]
109class _CrudParams(NamedTuple):
110 single_params: List[_CrudParamElementStr]
111 all_multi_params: List[Sequence[_CrudParamElementStr]]
112 is_default_metavalue_only: bool = False
113 use_insertmanyvalues: bool = False
114 use_sentinel_columns: Optional[Sequence[Column[Any]]] = None
117def _get_crud_params(
118 compiler: SQLCompiler,
119 stmt: ValuesBase,
120 compile_state: DMLState,
121 toplevel: bool,
122 **kw: Any,
123) -> _CrudParams:
124 """create a set of tuples representing column/string pairs for use
125 in an INSERT or UPDATE statement.
127 Also generates the Compiled object's postfetch, prefetch, and
128 returning column collections, used for default handling and ultimately
129 populating the CursorResult's prefetch_cols() and postfetch_cols()
130 collections.
132 """
134 # note: the _get_crud_params() system was written with the notion in mind
135 # that INSERT, UPDATE, DELETE are always the top level statement and
136 # that there is only one of them. With the addition of CTEs that can
137 # make use of DML, this assumption is no longer accurate; the DML
138 # statement is not necessarily the top-level "row returning" thing
139 # and it is also theoretically possible (fortunately nobody has asked yet)
140 # to have a single statement with multiple DMLs inside of it via CTEs.
142 # the current _get_crud_params() design doesn't accommodate these cases
143 # right now. It "just works" for a CTE that has a single DML inside of
144 # it, and for a CTE with multiple DML, it's not clear what would happen.
146 # overall, the "compiler.XYZ" collections here would need to be in a
147 # per-DML structure of some kind, and DefaultDialect would need to
148 # navigate these collections on a per-statement basis, with additional
149 # emphasis on the "toplevel returning data" statement. However we
150 # still need to run through _get_crud_params() for all DML as we have
151 # Python / SQL generated column defaults that need to be rendered.
153 # if there is user need for this kind of thing, it's likely a post 2.0
154 # kind of change as it would require deep changes to DefaultDialect
155 # as well as here.
157 compiler.postfetch = []
158 compiler.insert_prefetch = []
159 compiler.update_prefetch = []
160 compiler.implicit_returning = []
162 visiting_cte = kw.get("visiting_cte", None)
163 if visiting_cte is not None:
164 # for insert -> CTE -> insert, don't populate an incoming
165 # _crud_accumulate_bind_names collection; the INSERT we process here
166 # will not be inline within the VALUES of the enclosing INSERT as the
167 # CTE is placed on the outside. See issue #9173
168 kw.pop("accumulate_bind_names", None)
169 assert (
170 "accumulate_bind_names" not in kw
171 ), "Don't know how to handle insert within insert without a CTE"
173 bindmarkers: MutableMapping[ColumnElement[Any], DMLTargetCopy[Any]] = {}
174 kw["bindmarkers"] = bindmarkers
176 # getters - these are normally just column.key,
177 # but in the case of mysql multi-table update, the rules for
178 # .key must conditionally take tablename into account
179 (
180 _column_as_key,
181 _getattr_col_key,
182 _col_bind_name,
183 ) = _key_getters_for_crud_column(compiler, stmt, compile_state)
185 compiler._get_bind_name_for_col = _col_bind_name
187 if stmt._returning and stmt._return_defaults:
188 raise exc.CompileError(
189 "Can't compile statement that includes returning() and "
190 "return_defaults() simultaneously"
191 )
193 if compile_state.isdelete:
194 _setup_delete_return_defaults(
195 compiler,
196 stmt,
197 compile_state,
198 (),
199 _getattr_col_key,
200 _column_as_key,
201 _col_bind_name,
202 (),
203 (),
204 toplevel,
205 kw,
206 )
207 return _CrudParams([], [])
209 # no parameters in the statement, no parameters in the
210 # compiled params - return binds for all columns
211 if compiler.column_keys is None and compile_state._no_parameters:
212 return _CrudParams(
213 [
214 (
215 c,
216 compiler.preparer.format_column(c),
217 _create_bind_param(compiler, c, None, required=True),
218 (c.key,),
219 )
220 for c in stmt.table.columns
221 if not c._omit_from_statements
222 ],
223 [],
224 )
226 stmt_parameter_tuples: Optional[
227 List[Tuple[Union[str, ColumnClause[Any]], Any]]
228 ]
229 spd: Optional[MutableMapping[_DMLColumnElement, Any]]
231 if (
232 _compile_state_isinsert(compile_state)
233 and compile_state._has_multi_parameters
234 ):
235 mp = compile_state._multi_parameters
236 assert mp is not None
237 spd = mp[0]
238 stmt_parameter_tuples = list(spd.items())
239 spd_str_key = {_column_as_key(key) for key in spd}
240 elif compile_state._dict_parameters:
241 spd = compile_state._dict_parameters
242 stmt_parameter_tuples = list(spd.items())
243 spd_str_key = {_column_as_key(key) for key in spd}
244 else:
245 stmt_parameter_tuples = spd_str_key = None
247 # if we have statement parameters - set defaults in the
248 # compiled params
249 if compiler.column_keys is None:
250 parameters = {}
251 elif stmt_parameter_tuples:
252 assert spd_str_key is not None
253 parameters = {
254 _column_as_key(key): REQUIRED
255 for key in compiler.column_keys
256 if key not in spd_str_key
257 }
258 else:
259 parameters = {
260 _column_as_key(key): REQUIRED for key in compiler.column_keys
261 }
263 # create a list of column assignment clauses as tuples
264 values: List[_CrudParamElementStr] = []
266 if stmt_parameter_tuples is not None:
267 _get_stmt_parameter_tuples_params(
268 compiler,
269 compile_state,
270 parameters,
271 stmt_parameter_tuples,
272 _column_as_key,
273 values,
274 kw,
275 )
277 check_columns: Dict[str, ColumnClause[Any]] = {}
279 # special logic that only occurs for multi-table UPDATE
280 # statements
281 if dml.isupdate(compile_state) and compile_state.is_multitable:
282 _get_update_multitable_params(
283 compiler,
284 stmt,
285 compile_state,
286 stmt_parameter_tuples,
287 check_columns,
288 _col_bind_name,
289 _getattr_col_key,
290 values,
291 kw,
292 )
294 if _compile_state_isinsert(compile_state) and stmt._select_names:
295 # is an insert from select, is not a multiparams
297 assert not compile_state._has_multi_parameters
299 _scan_insert_from_select_cols(
300 compiler,
301 stmt,
302 compile_state,
303 parameters,
304 _getattr_col_key,
305 _column_as_key,
306 _col_bind_name,
307 check_columns,
308 values,
309 toplevel,
310 kw,
311 )
312 use_insertmanyvalues = False
313 use_sentinel_columns = None
314 else:
315 use_insertmanyvalues, use_sentinel_columns = _scan_cols(
316 compiler,
317 stmt,
318 compile_state,
319 parameters,
320 _getattr_col_key,
321 _column_as_key,
322 _col_bind_name,
323 check_columns,
324 values,
325 toplevel,
326 kw,
327 )
329 if parameters and stmt_parameter_tuples:
330 check = (
331 set(parameters)
332 .intersection(_column_as_key(k) for k, v in stmt_parameter_tuples)
333 .difference(check_columns)
334 )
335 if check:
337 if dml.isupdate(compile_state):
338 tables_mentioned = set(
339 c.table
340 for c, v in stmt_parameter_tuples
341 if isinstance(c, ColumnClause) and c.table is not None
342 ).difference([compile_state.dml_table])
344 multi_not_in_from = tables_mentioned.difference(
345 compile_state._extra_froms
346 )
348 if tables_mentioned and (
349 not compile_state.is_multitable
350 or not compiler.render_table_with_column_in_update_from
351 ):
352 if not compiler.render_table_with_column_in_update_from:
353 preamble = (
354 "Backend does not support additional "
355 "tables in the SET clause"
356 )
357 else:
358 preamble = (
359 "Statement is not a multi-table UPDATE statement"
360 )
362 raise exc.CompileError(
363 f"{preamble}; cannot "
364 f"""include columns from table(s) {
365 ", ".join(f"'{t.description}'"
366 for t in tables_mentioned)
367 } in SET clause"""
368 )
370 elif multi_not_in_from:
371 assert compiler.render_table_with_column_in_update_from
372 raise exc.CompileError(
373 f"Multi-table UPDATE statement does not include "
374 "table(s) "
375 f"""{
376 ", ".join(
377 f"'{t.description}'" for
378 t in multi_not_in_from)
379 }"""
380 )
382 raise exc.CompileError(
383 "Unconsumed column names: %s"
384 % (", ".join("%s" % (c,) for c in check))
385 )
387 is_default_metavalue_only = False
389 if (
390 _compile_state_isinsert(compile_state)
391 and compile_state._has_multi_parameters
392 ):
393 # is a multiparams, is not an insert from a select
394 assert not stmt._select_names
395 multi_extended_values = _extend_values_for_multiparams(
396 compiler,
397 stmt,
398 compile_state,
399 cast(
400 "Sequence[_CrudParamElementStr]",
401 values,
402 ),
403 cast("Callable[..., str]", _column_as_key),
404 kw,
405 )
407 if bindmarkers:
408 _replace_bindmarkers(
409 compiler,
410 _column_as_key,
411 bindmarkers,
412 compile_state,
413 values,
414 kw,
415 )
416 for m_v in multi_extended_values:
417 _replace_bindmarkers(
418 compiler,
419 _column_as_key,
420 bindmarkers,
421 compile_state,
422 m_v,
423 kw,
424 )
426 return _CrudParams(values, multi_extended_values)
427 elif (
428 not values
429 and compiler.for_executemany
430 and compiler.dialect.supports_default_metavalue
431 ):
432 # convert an "INSERT DEFAULT VALUES"
433 # into INSERT (firstcol) VALUES (DEFAULT) which can be turned
434 # into an in-place multi values. This supports
435 # insert_executemany_returning mode :)
436 values = [
437 (
438 _as_dml_column(stmt.table.columns[0]),
439 compiler.preparer.format_column(stmt.table.columns[0]),
440 compiler.dialect.default_metavalue_token,
441 (),
442 )
443 ]
444 is_default_metavalue_only = True
446 if bindmarkers:
447 _replace_bindmarkers(
448 compiler, _column_as_key, bindmarkers, compile_state, values, kw
449 )
450 return _CrudParams(
451 values,
452 [],
453 is_default_metavalue_only=is_default_metavalue_only,
454 use_insertmanyvalues=use_insertmanyvalues,
455 use_sentinel_columns=use_sentinel_columns,
456 )
459def _replace_bindmarkers(
460 compiler, _column_as_key, bindmarkers, compile_state, values, kw
461):
462 _expr_by_col_key = {
463 _column_as_key(col): compiled_str for col, _, compiled_str, _ in values
464 }
466 def replace_marker(m):
467 try:
468 return _expr_by_col_key[m.group(1)]
469 except KeyError as ke:
470 if dml.isupdate(compile_state):
471 return compiler.process(bindmarkers[m.group(1)].column, **kw)
472 else:
473 raise exc.CompileError(
474 f"Can't resolve referenced column name in "
475 f"INSERT statement: {m.group(1)!r}"
476 ) from ke
478 values[:] = [
479 (
480 col,
481 col_value,
482 re.sub(
483 r"__BINDMARKER_~~(.+?)~~",
484 replace_marker,
485 compiled_str,
486 ),
487 accumulated_bind_names,
488 )
489 for (
490 col,
491 col_value,
492 compiled_str,
493 accumulated_bind_names,
494 ) in values
495 ]
498@overload
499def _create_bind_param(
500 compiler: SQLCompiler,
501 col: ColumnElement[Any],
502 value: Any,
503 process: Literal[True] = ...,
504 required: bool = False,
505 name: Optional[str] = None,
506 force_anonymous: bool = False,
507 **kw: Any,
508) -> str: ...
511@overload
512def _create_bind_param(
513 compiler: SQLCompiler,
514 col: ColumnElement[Any],
515 value: Any,
516 **kw: Any,
517) -> str: ...
520def _create_bind_param(
521 compiler: SQLCompiler,
522 col: ColumnElement[Any],
523 value: Any,
524 process: bool = True,
525 required: bool = False,
526 name: Optional[str] = None,
527 force_anonymous: bool = False,
528 **kw: Any,
529) -> Union[str, elements.BindParameter[Any]]:
530 if force_anonymous:
531 name = None
532 elif name is None:
533 name = col.key
535 bindparam = elements.BindParameter(
536 name, value, type_=col.type, required=required
537 )
538 bindparam._is_crud = True
539 if process:
540 return bindparam._compiler_dispatch(compiler, **kw)
541 else:
542 return bindparam
545def _handle_values_anonymous_param(compiler, col, value, name, **kw):
546 # the insert() and update() constructs as of 1.4 will now produce anonymous
547 # bindparam() objects in the values() collections up front when given plain
548 # literal values. This is so that cache key behaviors, which need to
549 # produce bound parameters in deterministic order without invoking any
550 # compilation here, can be applied to these constructs when they include
551 # values() (but not yet multi-values, which are not included in caching
552 # right now).
553 #
554 # in order to produce the desired "crud" style name for these parameters,
555 # which will also be targetable in engine/default.py through the usual
556 # conventions, apply our desired name to these unique parameters by
557 # populating the compiler truncated names cache with the desired name,
558 # rather than having
559 # compiler.visit_bindparam()->compiler._truncated_identifier make up a
560 # name. Saves on call counts also.
562 # for INSERT/UPDATE that's a CTE, we don't need names to match to
563 # external parameters and these would also conflict in the case where
564 # multiple insert/update are combined together using CTEs
565 is_cte = "visiting_cte" in kw
567 if (
568 not is_cte
569 and value.unique
570 and isinstance(value.key, elements._truncated_label)
571 ):
572 compiler.truncated_names[("bindparam", value.key)] = name
574 if value.type._isnull:
575 # either unique parameter, or other bound parameters that were
576 # passed in directly
577 # set type to that of the column unconditionally
578 value = value._with_binary_element_type(col.type)
580 return value._compiler_dispatch(compiler, **kw)
583def _key_getters_for_crud_column(
584 compiler: SQLCompiler, stmt: ValuesBase, compile_state: DMLState
585) -> Tuple[
586 Callable[[Union[str, ColumnClause[Any]]], Union[str, Tuple[str, str]]],
587 Callable[[ColumnClause[Any]], Union[str, Tuple[str, str]]],
588 _BindNameForColProtocol,
589]:
590 if dml.isupdate(compile_state) and compile_state._extra_froms:
591 # when extra tables are present, refer to the columns
592 # in those extra tables as table-qualified, including in
593 # dictionaries and when rendering bind param names.
594 # the "main" table of the statement remains unqualified,
595 # allowing the most compatibility with a non-multi-table
596 # statement.
597 _et = set(compile_state._extra_froms)
599 c_key_role = functools.partial(
600 coercions.expect_as_key, roles.DMLColumnRole
601 )
603 def _column_as_key(
604 key: Union[ColumnClause[Any], str],
605 ) -> Union[str, Tuple[str, str]]:
606 str_key = c_key_role(key)
607 if hasattr(key, "table") and key.table in _et:
608 return (key.table.name, str_key) # type: ignore
609 else:
610 return str_key
612 def _getattr_col_key(
613 col: ColumnClause[Any],
614 ) -> Union[str, Tuple[str, str]]:
615 if col.table in _et:
616 return (col.table.name, col.key) # type: ignore
617 else:
618 return col.key
620 def _col_bind_name(col: ColumnClause[Any]) -> str:
621 if col.table in _et:
622 if TYPE_CHECKING:
623 assert isinstance(col.table, TableClause)
624 return "%s_%s" % (col.table.name, col.key)
625 else:
626 return col.key
628 else:
629 _column_as_key = functools.partial(
630 coercions.expect_as_key, roles.DMLColumnRole
631 )
632 _getattr_col_key = _col_bind_name = operator.attrgetter("key") # type: ignore # noqa: E501
634 return _column_as_key, _getattr_col_key, _col_bind_name
637def _scan_insert_from_select_cols(
638 compiler,
639 stmt,
640 compile_state,
641 parameters,
642 _getattr_col_key,
643 _column_as_key,
644 _col_bind_name,
645 check_columns,
646 values,
647 toplevel,
648 kw,
649):
650 cols = [stmt.table.c[_column_as_key(name)] for name in stmt._select_names]
652 assert compiler.stack[-1]["selectable"] is stmt
654 compiler.stack[-1]["insert_from_select"] = stmt.select
656 add_select_cols: List[_CrudParamElementSQLExpr] = []
657 if stmt.include_insert_from_select_defaults:
658 col_set = set(cols)
659 for col in stmt.table.columns:
660 # omit columns that were not in the SELECT statement.
661 # this will omit columns marked as omit_from_statements naturally,
662 # as long as that col was not explicit in the SELECT.
663 # if an omit_from_statements col has a "default" on it, then
664 # we need to include it, as these defaults should still fire off.
665 # but, if it has that default and it's the "sentinel" default,
666 # we don't do sentinel default operations for insert_from_select
667 # here so we again omit it.
668 if (
669 col not in col_set
670 and col.default
671 and not col.default.is_sentinel
672 ):
673 cols.append(col)
675 for c in cols:
676 col_key = _getattr_col_key(c)
677 if col_key in parameters and col_key not in check_columns:
678 parameters.pop(col_key)
679 values.append((c, compiler.preparer.format_column(c), None, ()))
680 else:
681 _append_param_insert_select_hasdefault(
682 compiler, stmt, c, add_select_cols, kw
683 )
685 if add_select_cols:
686 values.extend(add_select_cols)
687 ins_from_select = compiler.stack[-1]["insert_from_select"]
688 if not isinstance(ins_from_select, Select):
689 raise exc.CompileError(
690 f"Can't extend statement for INSERT..FROM SELECT to include "
691 f"additional default-holding column(s) "
692 f"""{
693 ', '.join(repr(key) for _, key, _, _ in add_select_cols)
694 }. Convert the selectable to a subquery() first, or pass """
695 "include_defaults=False to Insert.from_select() to skip these "
696 "columns."
697 )
698 ins_from_select = ins_from_select._generate()
699 # copy raw_columns
700 ins_from_select._raw_columns = list(ins_from_select._raw_columns) + [
701 expr for _, _, expr, _ in add_select_cols
702 ]
703 compiler.stack[-1]["insert_from_select"] = ins_from_select
706def _scan_cols(
707 compiler,
708 stmt,
709 compile_state,
710 parameters,
711 _getattr_col_key,
712 _column_as_key,
713 _col_bind_name,
714 check_columns,
715 values,
716 toplevel,
717 kw,
718):
719 (
720 need_pks,
721 implicit_returning,
722 implicit_return_defaults,
723 postfetch_lastrowid,
724 use_insertmanyvalues,
725 use_sentinel_columns,
726 ) = _get_returning_modifiers(compiler, stmt, compile_state, toplevel)
728 assert compile_state.isupdate or compile_state.isinsert
730 if compile_state._maintain_values_ordering:
731 parameter_ordering = [
732 _column_as_key(key) for key in compile_state._dict_parameters
733 ]
734 ordered_keys = set(parameter_ordering)
735 cols = [
736 stmt.table.c[key]
737 for key in parameter_ordering
738 if isinstance(key, str) and key in stmt.table.c
739 ] + [c for c in stmt.table.c if c.key not in ordered_keys]
741 else:
742 cols = stmt.table.columns
744 isinsert = _compile_state_isinsert(compile_state)
745 if isinsert and not compile_state._has_multi_parameters:
746 # new rules for #7998. fetch lastrowid or implicit returning
747 # for autoincrement column even if parameter is NULL, for DBs that
748 # override NULL param for primary key (sqlite, mysql/mariadb)
749 autoincrement_col = stmt.table._autoincrement_column
750 insert_null_pk_still_autoincrements = (
751 compiler.dialect.insert_null_pk_still_autoincrements
752 )
753 else:
754 autoincrement_col = insert_null_pk_still_autoincrements = None
756 if stmt._supplemental_returning:
757 supplemental_returning = set(stmt._supplemental_returning)
758 else:
759 supplemental_returning = set()
761 compiler_implicit_returning = compiler.implicit_returning
763 # TODO - see TODO(return_defaults_columns) below
764 # cols_in_params = set()
766 for c in cols:
767 # scan through every column in the target table
769 col_key = _getattr_col_key(c)
771 if col_key in parameters and col_key not in check_columns:
772 # parameter is present for the column. use that.
774 _append_param_parameter(
775 compiler,
776 stmt,
777 compile_state,
778 c,
779 col_key,
780 parameters,
781 _col_bind_name,
782 implicit_returning,
783 implicit_return_defaults,
784 postfetch_lastrowid,
785 values,
786 autoincrement_col,
787 insert_null_pk_still_autoincrements,
788 kw,
789 )
791 # TODO - see TODO(return_defaults_columns) below
792 # cols_in_params.add(c)
794 elif isinsert:
795 # no parameter is present and it's an insert.
797 if c.primary_key and need_pks:
798 # it's a primary key column, it will need to be generated by a
799 # default generator of some kind, and the statement expects
800 # inserted_primary_key to be available.
802 if implicit_returning:
803 # we can use RETURNING, find out how to invoke this
804 # column and get the value where RETURNING is an option.
805 # we can inline server-side functions in this case.
807 _append_param_insert_pk_returning(
808 compiler, stmt, c, values, kw
809 )
810 else:
811 # otherwise, find out how to invoke this column
812 # and get its value where RETURNING is not an option.
813 # if we have to invoke a server-side function, we need
814 # to pre-execute it. or if this is a straight
815 # autoincrement column and the dialect supports it
816 # we can use cursor.lastrowid.
818 _append_param_insert_pk_no_returning(
819 compiler, stmt, c, values, kw
820 )
822 elif c.default is not None:
823 # column has a default, but it's not a pk column, or it is but
824 # we don't need to get the pk back.
825 if not c.default.is_sentinel or (
826 use_sentinel_columns is not None
827 ):
828 _append_param_insert_hasdefault(
829 compiler, stmt, c, implicit_return_defaults, values, kw
830 )
832 elif c.server_default is not None:
833 # column has a DDL-level default, and is either not a pk
834 # column or we don't need the pk.
835 if implicit_return_defaults and c in implicit_return_defaults:
836 compiler_implicit_returning.append(c)
837 elif not c.primary_key:
838 compiler.postfetch.append(c)
840 elif implicit_return_defaults and c in implicit_return_defaults:
841 compiler_implicit_returning.append(c)
843 elif (
844 c.primary_key
845 and c is not stmt.table._autoincrement_column
846 and not c.nullable
847 ):
848 _warn_pk_with_no_anticipated_value(c)
850 elif compile_state.isupdate:
851 # no parameter is present and it's an insert.
853 _append_param_update(
854 compiler,
855 compile_state,
856 stmt,
857 c,
858 implicit_return_defaults,
859 values,
860 kw,
861 )
863 # adding supplemental cols to implicit_returning in table
864 # order so that order is maintained between multiple INSERT
865 # statements which may have different parameters included, but all
866 # have the same RETURNING clause
867 if (
868 c in supplemental_returning
869 and c not in compiler_implicit_returning
870 ):
871 compiler_implicit_returning.append(c)
873 if supplemental_returning:
874 # we should have gotten every col into implicit_returning,
875 # however supplemental returning can also have SQL functions etc.
876 # in it
877 remaining_supplemental = supplemental_returning.difference(
878 compiler_implicit_returning
879 )
880 compiler_implicit_returning.extend(
881 c
882 for c in stmt._supplemental_returning
883 if c in remaining_supplemental
884 )
886 # TODO(return_defaults_columns): there can still be more columns in
887 # _return_defaults_columns in the case that they are from something like an
888 # aliased of the table. we can add them here, however this breaks other ORM
889 # things. so this is for another day. see
890 # test/orm/dml/test_update_delete_where.py -> test_update_from_alias
892 # if stmt._return_defaults_columns:
893 # compiler_implicit_returning.extend(
894 # set(stmt._return_defaults_columns)
895 # .difference(compiler_implicit_returning)
896 # .difference(cols_in_params)
897 # )
899 return (use_insertmanyvalues, use_sentinel_columns)
902def _setup_delete_return_defaults(
903 compiler,
904 stmt,
905 compile_state,
906 parameters,
907 _getattr_col_key,
908 _column_as_key,
909 _col_bind_name,
910 check_columns,
911 values,
912 toplevel,
913 kw,
914):
915 _, _, implicit_return_defaults, *_ = _get_returning_modifiers(
916 compiler, stmt, compile_state, toplevel
917 )
919 if not implicit_return_defaults:
920 return
922 if stmt._return_defaults_columns:
923 compiler.implicit_returning.extend(implicit_return_defaults)
925 if stmt._supplemental_returning:
926 ir_set = set(compiler.implicit_returning)
927 compiler.implicit_returning.extend(
928 c for c in stmt._supplemental_returning if c not in ir_set
929 )
932def _append_param_parameter(
933 compiler,
934 stmt,
935 compile_state,
936 c,
937 col_key,
938 parameters,
939 _col_bind_name,
940 implicit_returning,
941 implicit_return_defaults,
942 postfetch_lastrowid,
943 values,
944 autoincrement_col,
945 insert_null_pk_still_autoincrements,
946 kw,
947):
948 value = parameters.pop(col_key)
950 has_visiting_cte = kw.get("visiting_cte") is not None
951 col_value = compiler.preparer.format_column(
952 c, use_table=compile_state.include_table_with_column_exprs
953 )
955 accumulated_bind_names: Set[str] = set()
957 if coercions._is_literal(value):
958 if (
959 insert_null_pk_still_autoincrements
960 and c.primary_key
961 and c is autoincrement_col
962 ):
963 # support use case for #7998, fetch autoincrement cols
964 # even if value was given.
966 if postfetch_lastrowid:
967 compiler.postfetch_lastrowid = True
968 elif implicit_returning:
969 compiler.implicit_returning.append(c)
971 value = _create_bind_param(
972 compiler,
973 c,
974 value,
975 required=value is REQUIRED,
976 name=(
977 _col_bind_name(c)
978 if not _compile_state_isinsert(compile_state)
979 or not compile_state._has_multi_parameters
980 else "%s_m0" % _col_bind_name(c)
981 ),
982 accumulate_bind_names=accumulated_bind_names,
983 force_anonymous=has_visiting_cte,
984 **kw,
985 )
986 elif value._is_bind_parameter:
987 if (
988 insert_null_pk_still_autoincrements
989 and value.value is None
990 and c.primary_key
991 and c is autoincrement_col
992 ):
993 # support use case for #7998, fetch autoincrement cols
994 # even if value was given
995 if implicit_returning:
996 compiler.implicit_returning.append(c)
997 elif compiler.dialect.postfetch_lastrowid:
998 compiler.postfetch_lastrowid = True
1000 value = _handle_values_anonymous_param(
1001 compiler,
1002 c,
1003 value,
1004 name=(
1005 _col_bind_name(c)
1006 if not _compile_state_isinsert(compile_state)
1007 or not compile_state._has_multi_parameters
1008 else "%s_m0" % _col_bind_name(c)
1009 ),
1010 accumulate_bind_names=accumulated_bind_names,
1011 **kw,
1012 )
1013 else:
1014 # value is a SQL expression
1015 value = compiler.process(
1016 value.self_group(),
1017 accumulate_bind_names=accumulated_bind_names,
1018 **kw,
1019 )
1021 if compile_state.isupdate:
1022 if implicit_return_defaults and c in implicit_return_defaults:
1023 compiler.implicit_returning.append(c)
1025 else:
1026 compiler.postfetch.append(c)
1027 else:
1028 if c.primary_key:
1029 if implicit_returning:
1030 compiler.implicit_returning.append(c)
1031 elif compiler.dialect.postfetch_lastrowid:
1032 compiler.postfetch_lastrowid = True
1034 elif implicit_return_defaults and (c in implicit_return_defaults):
1035 compiler.implicit_returning.append(c)
1037 else:
1038 # postfetch specifically means, "we can SELECT the row we just
1039 # inserted by primary key to get back the server generated
1040 # defaults". so by definition this can't be used to get the
1041 # primary key value back, because we need to have it ahead of
1042 # time.
1044 compiler.postfetch.append(c)
1046 values.append((c, col_value, value, accumulated_bind_names))
1049def _append_param_insert_pk_returning(compiler, stmt, c, values, kw):
1050 """Create a primary key expression in the INSERT statement where
1051 we want to populate result.inserted_primary_key and RETURNING
1052 is available.
1054 """
1055 if c.default is not None:
1056 if c.default.is_sequence:
1057 if compiler.dialect.supports_sequences and (
1058 not c.default.optional
1059 or not compiler.dialect.sequences_optional
1060 ):
1061 accumulated_bind_names: Set[str] = set()
1062 values.append(
1063 (
1064 c,
1065 compiler.preparer.format_column(c),
1066 compiler.process(
1067 c.default,
1068 accumulate_bind_names=accumulated_bind_names,
1069 **kw,
1070 ),
1071 accumulated_bind_names,
1072 )
1073 )
1074 compiler.implicit_returning.append(c)
1075 elif c.default.is_clause_element:
1076 accumulated_bind_names = set()
1077 values.append(
1078 (
1079 c,
1080 compiler.preparer.format_column(c),
1081 compiler.process(
1082 c.default.arg.self_group(),
1083 accumulate_bind_names=accumulated_bind_names,
1084 **kw,
1085 ),
1086 accumulated_bind_names,
1087 )
1088 )
1089 compiler.implicit_returning.append(c)
1090 else:
1091 # client side default. OK we can't use RETURNING, need to
1092 # do a "prefetch", which in fact fetches the default value
1093 # on the Python side
1094 values.append(
1095 (
1096 c,
1097 compiler.preparer.format_column(c),
1098 _create_insert_prefetch_bind_param(compiler, c, **kw),
1099 (c.key,),
1100 )
1101 )
1102 elif c is stmt.table._autoincrement_column or c.server_default is not None:
1103 compiler.implicit_returning.append(c)
1104 elif not c.nullable:
1105 # no .default, no .server_default, not autoincrement, we have
1106 # no indication this primary key column will have any value
1107 _warn_pk_with_no_anticipated_value(c)
1110def _append_param_insert_pk_no_returning(compiler, stmt, c, values, kw):
1111 """Create a primary key expression in the INSERT statement where
1112 we want to populate result.inserted_primary_key and we cannot use
1113 RETURNING.
1115 Depending on the kind of default here we may create a bound parameter
1116 in the INSERT statement and pre-execute a default generation function,
1117 or we may use cursor.lastrowid if supported by the dialect.
1120 """
1122 if (
1123 # column has a Python-side default
1124 c.default is not None
1125 and (
1126 # and it either is not a sequence, or it is and we support
1127 # sequences and want to invoke it
1128 not c.default.is_sequence
1129 or (
1130 compiler.dialect.supports_sequences
1131 and (
1132 not c.default.optional
1133 or not compiler.dialect.sequences_optional
1134 )
1135 )
1136 )
1137 ) or (
1138 # column is the "autoincrement column"
1139 c is stmt.table._autoincrement_column
1140 and (
1141 # dialect can't use cursor.lastrowid
1142 not compiler.dialect.postfetch_lastrowid
1143 and (
1144 # column has a Sequence and we support those
1145 (
1146 c.default is not None
1147 and c.default.is_sequence
1148 and compiler.dialect.supports_sequences
1149 )
1150 or
1151 # column has no default on it, but dialect can run the
1152 # "autoincrement" mechanism explicitly, e.g. PostgreSQL
1153 # SERIAL we know the sequence name
1154 (
1155 c.default is None
1156 and compiler.dialect.preexecute_autoincrement_sequences
1157 )
1158 )
1159 )
1160 ):
1161 # do a pre-execute of the default
1162 values.append(
1163 (
1164 c,
1165 compiler.preparer.format_column(c),
1166 _create_insert_prefetch_bind_param(compiler, c, **kw),
1167 (c.key,),
1168 )
1169 )
1170 elif (
1171 c.default is None
1172 and c.server_default is None
1173 and not c.nullable
1174 and c is not stmt.table._autoincrement_column
1175 ):
1176 # no .default, no .server_default, not autoincrement, we have
1177 # no indication this primary key column will have any value
1178 _warn_pk_with_no_anticipated_value(c)
1179 elif compiler.dialect.postfetch_lastrowid:
1180 # finally, where it seems like there will be a generated primary key
1181 # value and we haven't set up any other way to fetch it, and the
1182 # dialect supports cursor.lastrowid, switch on the lastrowid flag so
1183 # that the DefaultExecutionContext calls upon cursor.lastrowid
1184 compiler.postfetch_lastrowid = True
1187def _append_param_insert_hasdefault(
1188 compiler, stmt, c, implicit_return_defaults, values, kw
1189):
1190 if c.default.is_sequence:
1191 if compiler.dialect.supports_sequences and (
1192 not c.default.optional or not compiler.dialect.sequences_optional
1193 ):
1194 accumulated_bind_names: Set[str] = set()
1195 values.append(
1196 (
1197 c,
1198 compiler.preparer.format_column(c),
1199 compiler.process(
1200 c.default,
1201 accumulate_bind_names=accumulated_bind_names,
1202 **kw,
1203 ),
1204 accumulated_bind_names,
1205 )
1206 )
1207 if implicit_return_defaults and c in implicit_return_defaults:
1208 compiler.implicit_returning.append(c)
1209 elif not c.primary_key:
1210 compiler.postfetch.append(c)
1211 elif c.default.is_clause_element:
1212 accumulated_bind_names = set()
1213 values.append(
1214 (
1215 c,
1216 compiler.preparer.format_column(c),
1217 compiler.process(
1218 c.default.arg.self_group(),
1219 accumulate_bind_names=accumulated_bind_names,
1220 **kw,
1221 ),
1222 accumulated_bind_names,
1223 )
1224 )
1226 if implicit_return_defaults and c in implicit_return_defaults:
1227 compiler.implicit_returning.append(c)
1228 elif not c.primary_key:
1229 # don't add primary key column to postfetch
1230 compiler.postfetch.append(c)
1231 else:
1232 values.append(
1233 (
1234 c,
1235 compiler.preparer.format_column(c),
1236 _create_insert_prefetch_bind_param(compiler, c, **kw),
1237 (c.key,),
1238 )
1239 )
1242def _append_param_insert_select_hasdefault(
1243 compiler: SQLCompiler,
1244 stmt: ValuesBase,
1245 c: ColumnClause[Any],
1246 values: List[_CrudParamElementSQLExpr],
1247 kw: Dict[str, Any],
1248) -> None:
1249 if default_is_sequence(c.default):
1250 if compiler.dialect.supports_sequences and (
1251 not c.default.optional or not compiler.dialect.sequences_optional
1252 ):
1253 values.append(
1254 (
1255 c,
1256 compiler.preparer.format_column(c),
1257 c.default.next_value(),
1258 (),
1259 )
1260 )
1261 elif default_is_clause_element(c.default):
1262 values.append(
1263 (
1264 c,
1265 compiler.preparer.format_column(c),
1266 c.default.arg.self_group(),
1267 (),
1268 )
1269 )
1270 else:
1271 values.append(
1272 (
1273 c,
1274 compiler.preparer.format_column(c),
1275 _create_insert_prefetch_bind_param(
1276 compiler, c, process=False, **kw
1277 ),
1278 (c.key,),
1279 )
1280 )
1283def _append_param_update(
1284 compiler, compile_state, stmt, c, implicit_return_defaults, values, kw
1285):
1286 include_table = compile_state.include_table_with_column_exprs
1287 if c.onupdate is not None and not c.onupdate.is_sequence:
1288 if c.onupdate.is_clause_element:
1289 values.append(
1290 (
1291 c,
1292 compiler.preparer.format_column(
1293 c,
1294 use_table=include_table,
1295 ),
1296 compiler.process(c.onupdate.arg.self_group(), **kw),
1297 (),
1298 )
1299 )
1300 if implicit_return_defaults and c in implicit_return_defaults:
1301 compiler.implicit_returning.append(c)
1302 else:
1303 compiler.postfetch.append(c)
1304 else:
1305 values.append(
1306 (
1307 c,
1308 compiler.preparer.format_column(
1309 c,
1310 use_table=include_table,
1311 ),
1312 _create_update_prefetch_bind_param(compiler, c, **kw),
1313 (c.key,),
1314 )
1315 )
1316 elif c.server_onupdate is not None:
1317 if implicit_return_defaults and c in implicit_return_defaults:
1318 compiler.implicit_returning.append(c)
1319 else:
1320 compiler.postfetch.append(c)
1321 elif (
1322 implicit_return_defaults
1323 and (stmt._return_defaults_columns or not stmt._return_defaults)
1324 and c in implicit_return_defaults
1325 ):
1326 compiler.implicit_returning.append(c)
1329@overload
1330def _create_insert_prefetch_bind_param(
1331 compiler: SQLCompiler,
1332 c: ColumnElement[Any],
1333 process: Literal[True] = ...,
1334 **kw: Any,
1335) -> str: ...
1338@overload
1339def _create_insert_prefetch_bind_param(
1340 compiler: SQLCompiler,
1341 c: ColumnElement[Any],
1342 process: Literal[False],
1343 **kw: Any,
1344) -> elements.BindParameter[Any]: ...
1347def _create_insert_prefetch_bind_param(
1348 compiler: SQLCompiler,
1349 c: ColumnElement[Any],
1350 process: bool = True,
1351 name: Optional[str] = None,
1352 **kw: Any,
1353) -> Union[elements.BindParameter[Any], str]:
1354 param = _create_bind_param(
1355 compiler, c, None, process=process, name=name, **kw
1356 )
1357 compiler.insert_prefetch.append(c) # type: ignore
1358 return param
1361@overload
1362def _create_update_prefetch_bind_param(
1363 compiler: SQLCompiler,
1364 c: ColumnElement[Any],
1365 process: Literal[True] = ...,
1366 **kw: Any,
1367) -> str: ...
1370@overload
1371def _create_update_prefetch_bind_param(
1372 compiler: SQLCompiler,
1373 c: ColumnElement[Any],
1374 process: Literal[False],
1375 **kw: Any,
1376) -> elements.BindParameter[Any]: ...
1379def _create_update_prefetch_bind_param(
1380 compiler: SQLCompiler,
1381 c: ColumnElement[Any],
1382 process: bool = True,
1383 name: Optional[str] = None,
1384 **kw: Any,
1385) -> Union[elements.BindParameter[Any], str]:
1386 param = _create_bind_param(
1387 compiler, c, None, process=process, name=name, **kw
1388 )
1389 compiler.update_prefetch.append(c) # type: ignore
1390 return param
1393class _multiparam_column(elements.ColumnElement[Any]):
1394 _is_multiparam_column = True
1396 def __init__(self, original, index):
1397 self.index = index
1398 self.key = "%s_m%d" % (original.key, index + 1)
1399 self.original = original
1400 self.default = original.default
1401 self.type = original.type
1403 def compare(self, other, **kw):
1404 raise NotImplementedError()
1406 def _copy_internals(self, **kw):
1407 raise NotImplementedError()
1409 def __eq__(self, other):
1410 return (
1411 isinstance(other, _multiparam_column)
1412 and other.key == self.key
1413 and other.original == self.original
1414 )
1416 @util.memoized_property
1417 def _default_description_tuple(self) -> _DefaultDescriptionTuple:
1418 """used by default.py -> _process_execute_defaults()"""
1420 return _DefaultDescriptionTuple._from_column_default(self.default)
1422 @util.memoized_property
1423 def _onupdate_description_tuple(self) -> _DefaultDescriptionTuple:
1424 """used by default.py -> _process_execute_defaults()"""
1426 return _DefaultDescriptionTuple._from_column_default(self.onupdate)
1429def _process_multiparam_default_bind(
1430 compiler: SQLCompiler,
1431 stmt: ValuesBase,
1432 c: KeyedColumnElement[Any],
1433 index: int,
1434 kw: Dict[str, Any],
1435) -> str:
1436 if not c.default:
1437 raise exc.CompileError(
1438 "INSERT value for column %s is explicitly rendered as a bound"
1439 "parameter in the VALUES clause; "
1440 "a Python-side value or SQL expression is required" % c
1441 )
1442 elif default_is_clause_element(c.default):
1443 return compiler.process(c.default.arg.self_group(), **kw)
1444 elif c.default.is_sequence:
1445 # these conditions would have been established
1446 # by append_param_insert_(?:hasdefault|pk_returning|pk_no_returning)
1447 # in order for us to be here, so these don't need to be
1448 # checked
1449 # assert compiler.dialect.supports_sequences and (
1450 # not c.default.optional
1451 # or not compiler.dialect.sequences_optional
1452 # )
1453 return compiler.process(c.default, **kw)
1454 else:
1455 col = _multiparam_column(c, index)
1456 assert isinstance(stmt, dml.Insert)
1457 return _create_insert_prefetch_bind_param(
1458 compiler, col, process=True, **kw
1459 )
1462def _get_update_multitable_params(
1463 compiler,
1464 stmt,
1465 compile_state,
1466 stmt_parameter_tuples,
1467 check_columns,
1468 _col_bind_name,
1469 _getattr_col_key,
1470 values,
1471 kw,
1472):
1473 normalized_params = {
1474 coercions.expect(roles.DMLColumnRole, c): param
1475 for c, param in stmt_parameter_tuples or ()
1476 }
1478 include_table = compile_state.include_table_with_column_exprs
1480 affected_tables = set()
1481 for t in compile_state._extra_froms:
1482 # extra gymnastics to support the probably-shouldnt-have-supported
1483 # case of "UPDATE table AS alias SET table.foo = bar", but it's
1484 # supported
1485 we_shouldnt_be_here_if_columns_found = (
1486 not include_table
1487 and not compile_state.dml_table.is_derived_from(t)
1488 )
1490 for c in t.c:
1491 if c in normalized_params:
1493 if we_shouldnt_be_here_if_columns_found:
1494 raise exc.CompileError(
1495 "Backend does not support additional tables "
1496 "in the SET "
1497 "clause; cannot include columns from table(s) "
1498 f"'{t.description}' in "
1499 "SET clause"
1500 )
1502 affected_tables.add(t)
1504 check_columns[_getattr_col_key(c)] = c
1505 value = normalized_params[c]
1507 col_value = compiler.process(c, include_table=include_table)
1508 if coercions._is_literal(value):
1509 value = _create_bind_param(
1510 compiler,
1511 c,
1512 value,
1513 required=value is REQUIRED,
1514 name=_col_bind_name(c),
1515 **kw, # TODO: no test coverage for literal binds here
1516 )
1517 accumulated_bind_names: Iterable[str] = (c.key,)
1518 elif value._is_bind_parameter:
1519 cbn = _col_bind_name(c)
1520 value = _handle_values_anonymous_param(
1521 compiler, c, value, name=cbn, **kw
1522 )
1523 accumulated_bind_names = (cbn,)
1524 else:
1525 compiler.postfetch.append(c)
1526 value = compiler.process(value.self_group(), **kw)
1527 accumulated_bind_names = ()
1528 values.append((c, col_value, value, accumulated_bind_names))
1530 # determine tables which are actually to be updated - process onupdate
1531 # and server_onupdate for these
1532 for t in affected_tables:
1533 for c in t.c:
1534 if c in normalized_params:
1535 continue
1536 elif c.onupdate is not None and not c.onupdate.is_sequence:
1537 if c.onupdate.is_clause_element:
1538 values.append(
1539 (
1540 c,
1541 compiler.process(c, include_table=include_table),
1542 compiler.process(
1543 c.onupdate.arg.self_group(), **kw
1544 ),
1545 (),
1546 )
1547 )
1548 compiler.postfetch.append(c)
1549 else:
1550 values.append(
1551 (
1552 c,
1553 compiler.process(c, include_table=include_table),
1554 _create_update_prefetch_bind_param(
1555 compiler, c, name=_col_bind_name(c), **kw
1556 ),
1557 (c.key,),
1558 )
1559 )
1560 elif c.server_onupdate is not None:
1561 compiler.postfetch.append(c)
1564def _extend_values_for_multiparams(
1565 compiler: SQLCompiler,
1566 stmt: ValuesBase,
1567 compile_state: DMLState,
1568 initial_values: Sequence[_CrudParamElementStr],
1569 _column_as_key: Callable[..., str],
1570 kw: Dict[str, Any],
1571) -> List[Sequence[_CrudParamElementStr]]:
1572 values_0 = initial_values
1573 values = [initial_values]
1575 has_visiting_cte = kw.get("visiting_cte") is not None
1576 mp = compile_state._multi_parameters
1577 assert mp is not None
1578 for i, row in enumerate(mp[1:]):
1579 extension: List[_CrudParamElementStr] = []
1581 row = {_column_as_key(key): v for key, v in row.items()}
1583 for col, col_expr, param, accumulated_names in values_0:
1584 if col.key in row:
1585 key = col.key
1587 if coercions._is_literal(row[key]):
1588 new_param = _create_bind_param(
1589 compiler,
1590 col,
1591 row[key],
1592 name=("%s_m%d" % (col.key, i + 1)),
1593 force_anonymous=has_visiting_cte,
1594 **kw,
1595 )
1596 else:
1597 new_param = compiler.process(row[key].self_group(), **kw)
1598 else:
1599 new_param = _process_multiparam_default_bind(
1600 compiler, stmt, col, i, kw
1601 )
1603 extension.append((col, col_expr, new_param, accumulated_names))
1605 values.append(extension)
1607 return values
1610def _get_stmt_parameter_tuples_params(
1611 compiler,
1612 compile_state,
1613 parameters,
1614 stmt_parameter_tuples,
1615 _column_as_key,
1616 values,
1617 kw,
1618):
1619 for k, v in stmt_parameter_tuples:
1620 colkey = _column_as_key(k)
1621 if colkey is not None:
1622 parameters.setdefault(colkey, v)
1623 else:
1624 # a non-Column expression on the left side;
1625 # add it to values() in an "as-is" state,
1626 # coercing right side to bound param
1628 # note one of the main use cases for this is array slice
1629 # updates on PostgreSQL, as the left side is also an expression.
1631 col_expr = compiler.process(
1632 k, include_table=compile_state.include_table_with_column_exprs
1633 )
1635 if coercions._is_literal(v):
1636 v = compiler.process(
1637 elements.BindParameter(None, v, type_=k.type), **kw
1638 )
1639 else:
1640 if v._is_bind_parameter and v.type._isnull:
1641 # either unique parameter, or other bound parameters that
1642 # were passed in directly
1643 # set type to that of the column unconditionally
1644 v = v._with_binary_element_type(k.type)
1646 v = compiler.process(v.self_group(), **kw)
1648 # TODO: not sure if accumulated_bind_names applies here
1649 values.append((k, col_expr, v, ()))
1652def _get_returning_modifiers(compiler, stmt, compile_state, toplevel):
1653 """determines RETURNING strategy, if any, for the statement.
1655 This is where it's determined what we need to fetch from the
1656 INSERT or UPDATE statement after it's invoked.
1658 """
1660 dialect = compiler.dialect
1662 need_pks = (
1663 toplevel
1664 and _compile_state_isinsert(compile_state)
1665 and not stmt._inline
1666 and (
1667 not compiler.for_executemany
1668 or (dialect.insert_executemany_returning and stmt._return_defaults)
1669 )
1670 and not stmt._returning
1671 # and (not stmt._returning or stmt._return_defaults)
1672 and not compile_state._has_multi_parameters
1673 )
1675 # check if we have access to simple cursor.lastrowid. we can use that
1676 # after the INSERT if that's all we need.
1677 postfetch_lastrowid = (
1678 need_pks
1679 and dialect.postfetch_lastrowid
1680 and stmt.table._autoincrement_column is not None
1681 )
1683 # see if we want to add RETURNING to an INSERT in order to get
1684 # primary key columns back. This would be instead of postfetch_lastrowid
1685 # if that's set.
1686 implicit_returning = (
1687 # statement itself can veto it
1688 need_pks
1689 # the dialect can veto it if it just doesn't support RETURNING
1690 # with INSERT
1691 and dialect.insert_returning
1692 # user-defined implicit_returning on Table can veto it
1693 and compile_state._primary_table.implicit_returning
1694 # the compile_state can veto it (SQlite uses this to disable
1695 # RETURNING for an ON CONFLICT insert, as SQLite does not return
1696 # for rows that were updated, which is wrong)
1697 and compile_state._supports_implicit_returning
1698 and (
1699 # since we support MariaDB and SQLite which also support lastrowid,
1700 # decide if we should use lastrowid or RETURNING. for insert
1701 # that didn't call return_defaults() and has just one set of
1702 # parameters, we can use lastrowid. this is more "traditional"
1703 # and a lot of weird use cases are supported by it.
1704 # SQLite lastrowid times 3x faster than returning,
1705 # Mariadb lastrowid 2x faster than returning
1706 (not postfetch_lastrowid or dialect.favor_returning_over_lastrowid)
1707 or compile_state._has_multi_parameters
1708 or stmt._return_defaults
1709 )
1710 )
1711 if implicit_returning:
1712 postfetch_lastrowid = False
1714 if _compile_state_isinsert(compile_state):
1715 should_implicit_return_defaults = (
1716 implicit_returning and stmt._return_defaults
1717 )
1718 explicit_returning = (
1719 should_implicit_return_defaults
1720 or stmt._returning
1721 or stmt._supplemental_returning
1722 )
1723 use_insertmanyvalues = (
1724 toplevel
1725 and compiler.for_executemany
1726 and dialect.use_insertmanyvalues
1727 and (
1728 explicit_returning
1729 or (
1730 dialect.use_insertmanyvalues_wo_returning
1731 # Disable insertmanyvalues_wo_returning when there's a
1732 # post-values clause like ON CONFLICT DO UPDATE.
1733 # This is a performance optimization flag and the batching
1734 # doesn't work correctly with these clauses. See #13130.
1735 and stmt._post_values_clause is None
1736 )
1737 )
1738 )
1740 use_sentinel_columns = None
1741 if (
1742 use_insertmanyvalues
1743 and explicit_returning
1744 and stmt._sort_by_parameter_order
1745 ):
1746 use_sentinel_columns = compiler._get_sentinel_column_for_table(
1747 stmt.table
1748 )
1750 elif compile_state.isupdate:
1751 should_implicit_return_defaults = (
1752 stmt._return_defaults
1753 and compile_state._primary_table.implicit_returning
1754 and compile_state._supports_implicit_returning
1755 and dialect.update_returning
1756 )
1757 use_insertmanyvalues = False
1758 use_sentinel_columns = None
1759 elif compile_state.isdelete:
1760 should_implicit_return_defaults = (
1761 stmt._return_defaults
1762 and compile_state._primary_table.implicit_returning
1763 and compile_state._supports_implicit_returning
1764 and dialect.delete_returning
1765 )
1766 use_insertmanyvalues = False
1767 use_sentinel_columns = None
1768 else:
1769 should_implicit_return_defaults = False # pragma: no cover
1770 use_insertmanyvalues = False
1771 use_sentinel_columns = None
1773 if should_implicit_return_defaults:
1774 if not stmt._return_defaults_columns:
1775 # TODO: this is weird. See #9685 where we have to
1776 # take an extra step to prevent this from happening. why
1777 # would this ever be *all* columns? but if we set to blank, then
1778 # that seems to break things also in the ORM. So we should
1779 # try to clean this up and figure out what return_defaults
1780 # needs to do w/ the ORM etc. here
1781 implicit_return_defaults = set(stmt.table.c)
1782 else:
1783 implicit_return_defaults = set(stmt._return_defaults_columns)
1784 else:
1785 implicit_return_defaults = None
1787 return (
1788 need_pks,
1789 implicit_returning or should_implicit_return_defaults,
1790 implicit_return_defaults,
1791 postfetch_lastrowid,
1792 use_insertmanyvalues,
1793 use_sentinel_columns,
1794 )
1797def _warn_pk_with_no_anticipated_value(c):
1798 msg = (
1799 "Column '%s.%s' is marked as a member of the "
1800 "primary key for table '%s', "
1801 "but has no Python-side or server-side default generator indicated, "
1802 "nor does it indicate 'autoincrement=True' or 'nullable=True', "
1803 "and no explicit value is passed. "
1804 "Primary key columns typically may not store NULL."
1805 % (c.table.fullname, c.name, c.table.fullname)
1806 )
1807 if len(c.table.primary_key) > 1:
1808 msg += (
1809 " Note that as of SQLAlchemy 1.1, 'autoincrement=True' must be "
1810 "indicated explicitly for composite (e.g. multicolumn) primary "
1811 "keys if AUTO_INCREMENT/SERIAL/IDENTITY "
1812 "behavior is expected for one of the columns in the primary key. "
1813 "CREATE TABLE statements are impacted by this change as well on "
1814 "most backends."
1815 )
1816 util.warn(msg)