Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/keras/src/layers/core/dense.py: 21%
90 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Contains the Dense layer."""
18import tensorflow.compat.v2 as tf
20from keras.src import activations
21from keras.src import backend
22from keras.src import constraints
23from keras.src import initializers
24from keras.src import regularizers
25from keras.src.dtensor import utils
26from keras.src.engine.base_layer import Layer
27from keras.src.engine.input_spec import InputSpec
29# isort: off
30from tensorflow.python.util.tf_export import keras_export
33@keras_export("keras.layers.Dense")
34class Dense(Layer):
35 """Just your regular densely-connected NN layer.
37 `Dense` implements the operation:
38 `output = activation(dot(input, kernel) + bias)`
39 where `activation` is the element-wise activation function
40 passed as the `activation` argument, `kernel` is a weights matrix
41 created by the layer, and `bias` is a bias vector created by the layer
42 (only applicable if `use_bias` is `True`). These are all attributes of
43 `Dense`.
45 Note: If the input to the layer has a rank greater than 2, then `Dense`
46 computes the dot product between the `inputs` and the `kernel` along the
47 last axis of the `inputs` and axis 0 of the `kernel` (using `tf.tensordot`).
48 For example, if input has dimensions `(batch_size, d0, d1)`, then we create
49 a `kernel` with shape `(d1, units)`, and the `kernel` operates along axis 2
50 of the `input`, on every sub-tensor of shape `(1, 1, d1)` (there are
51 `batch_size * d0` such sub-tensors). The output in this case will have
52 shape `(batch_size, d0, units)`.
54 Besides, layer attributes cannot be modified after the layer has been called
55 once (except the `trainable` attribute).
56 When a popular kwarg `input_shape` is passed, then keras will create
57 an input layer to insert before the current layer. This can be treated
58 equivalent to explicitly defining an `InputLayer`.
60 Example:
62 >>> # Create a `Sequential` model and add a Dense layer as the first layer.
63 >>> model = tf.keras.models.Sequential()
64 >>> model.add(tf.keras.Input(shape=(16,)))
65 >>> model.add(tf.keras.layers.Dense(32, activation='relu'))
66 >>> # Now the model will take as input arrays of shape (None, 16)
67 >>> # and output arrays of shape (None, 32).
68 >>> # Note that after the first layer, you don't need to specify
69 >>> # the size of the input anymore:
70 >>> model.add(tf.keras.layers.Dense(32))
71 >>> model.output_shape
72 (None, 32)
74 Args:
75 units: Positive integer, dimensionality of the output space.
76 activation: Activation function to use.
77 If you don't specify anything, no activation is applied
78 (ie. "linear" activation: `a(x) = x`).
79 use_bias: Boolean, whether the layer uses a bias vector.
80 kernel_initializer: Initializer for the `kernel` weights matrix.
81 bias_initializer: Initializer for the bias vector.
82 kernel_regularizer: Regularizer function applied to
83 the `kernel` weights matrix.
84 bias_regularizer: Regularizer function applied to the bias vector.
85 activity_regularizer: Regularizer function applied to
86 the output of the layer (its "activation").
87 kernel_constraint: Constraint function applied to
88 the `kernel` weights matrix.
89 bias_constraint: Constraint function applied to the bias vector.
91 Input shape:
92 N-D tensor with shape: `(batch_size, ..., input_dim)`.
93 The most common situation would be
94 a 2D input with shape `(batch_size, input_dim)`.
96 Output shape:
97 N-D tensor with shape: `(batch_size, ..., units)`.
98 For instance, for a 2D input with shape `(batch_size, input_dim)`,
99 the output would have shape `(batch_size, units)`.
100 """
102 @utils.allow_initializer_layout
103 def __init__(
104 self,
105 units,
106 activation=None,
107 use_bias=True,
108 kernel_initializer="glorot_uniform",
109 bias_initializer="zeros",
110 kernel_regularizer=None,
111 bias_regularizer=None,
112 activity_regularizer=None,
113 kernel_constraint=None,
114 bias_constraint=None,
115 **kwargs,
116 ):
117 super().__init__(activity_regularizer=activity_regularizer, **kwargs)
119 self.units = int(units) if not isinstance(units, int) else units
120 if self.units < 0:
121 raise ValueError(
122 "Received an invalid value for `units`, expected "
123 f"a positive integer. Received: units={units}"
124 )
125 self.activation = activations.get(activation)
126 self.use_bias = use_bias
127 self.kernel_initializer = initializers.get(kernel_initializer)
128 self.bias_initializer = initializers.get(bias_initializer)
129 self.kernel_regularizer = regularizers.get(kernel_regularizer)
130 self.bias_regularizer = regularizers.get(bias_regularizer)
131 self.kernel_constraint = constraints.get(kernel_constraint)
132 self.bias_constraint = constraints.get(bias_constraint)
134 self.input_spec = InputSpec(min_ndim=2)
135 self.supports_masking = True
137 def build(self, input_shape):
138 dtype = tf.as_dtype(self.dtype or backend.floatx())
139 if not (dtype.is_floating or dtype.is_complex):
140 raise TypeError(
141 "A Dense layer can only be built with a floating-point "
142 f"dtype. Received: dtype={dtype}"
143 )
145 input_shape = tf.TensorShape(input_shape)
146 last_dim = tf.compat.dimension_value(input_shape[-1])
147 if last_dim is None:
148 raise ValueError(
149 "The last dimension of the inputs to a Dense layer "
150 "should be defined. Found None. "
151 f"Full input shape received: {input_shape}"
152 )
153 self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim})
154 self.kernel = self.add_weight(
155 "kernel",
156 shape=[last_dim, self.units],
157 initializer=self.kernel_initializer,
158 regularizer=self.kernel_regularizer,
159 constraint=self.kernel_constraint,
160 dtype=self.dtype,
161 trainable=True,
162 )
163 if self.use_bias:
164 self.bias = self.add_weight(
165 "bias",
166 shape=[
167 self.units,
168 ],
169 initializer=self.bias_initializer,
170 regularizer=self.bias_regularizer,
171 constraint=self.bias_constraint,
172 dtype=self.dtype,
173 trainable=True,
174 )
175 else:
176 self.bias = None
177 self.built = True
179 def call(self, inputs):
180 if inputs.dtype.base_dtype != self._compute_dtype_object.base_dtype:
181 inputs = tf.cast(inputs, dtype=self._compute_dtype_object)
183 is_ragged = isinstance(inputs, tf.RaggedTensor)
184 if is_ragged:
185 # In case we encounter a RaggedTensor with a fixed last dimension
186 # (last dimension not ragged), we can flatten the input and restore
187 # the ragged dimensions at the end.
188 if tf.compat.dimension_value(inputs.shape[-1]) is None:
189 raise ValueError(
190 "Dense layer only supports RaggedTensors when the "
191 "innermost dimension is non-ragged. Received: "
192 f"inputs.shape={inputs.shape}."
193 )
194 original_inputs = inputs
195 if inputs.flat_values.shape.rank > 1:
196 inputs = inputs.flat_values
197 else:
198 # Innermost partition is encoded using uniform_row_length.
199 # (This is unusual, but we can handle it.)
200 if inputs.shape.rank == 2:
201 inputs = inputs.to_tensor()
202 is_ragged = False
203 else:
204 for _ in range(original_inputs.ragged_rank - 1):
205 inputs = inputs.values
206 inputs = inputs.to_tensor()
207 original_inputs = tf.RaggedTensor.from_nested_row_splits(
208 inputs, original_inputs.nested_row_splits[:-1]
209 )
211 rank = inputs.shape.rank
212 if rank == 2 or rank is None:
213 # We use embedding_lookup_sparse as a more efficient matmul
214 # operation for large sparse input tensors. The op will result in a
215 # sparse gradient, as opposed to
216 # sparse_ops.sparse_tensor_dense_matmul which results in dense
217 # gradients. This can lead to sigfinicant speedups, see b/171762937.
218 if isinstance(inputs, tf.SparseTensor):
219 # We need to fill empty rows, as the op assumes at least one id
220 # per row.
221 inputs, _ = tf.sparse.fill_empty_rows(inputs, 0)
222 # We need to do some munging of our input to use the embedding
223 # lookup as a matrix multiply. We split our input matrix into
224 # separate ids and weights tensors. The values of the ids tensor
225 # should be the column indices of our input matrix and the
226 # values of the weights tensor can continue to the actual matrix
227 # weights. The column arrangement of ids and weights will be
228 # summed over and does not matter. See the documentation for
229 # sparse_ops.sparse_tensor_dense_matmul a more detailed
230 # explanation of the inputs to both ops.
231 ids = tf.SparseTensor(
232 indices=inputs.indices,
233 values=inputs.indices[:, 1],
234 dense_shape=inputs.dense_shape,
235 )
236 weights = inputs
237 outputs = tf.nn.embedding_lookup_sparse(
238 self.kernel, ids, weights, combiner="sum"
239 )
240 else:
241 outputs = tf.matmul(a=inputs, b=self.kernel)
242 # Broadcast kernel to inputs.
243 else:
244 outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])
245 # Reshape the output back to the original ndim of the input.
246 if not tf.executing_eagerly():
247 shape = inputs.shape.as_list()
248 output_shape = shape[:-1] + [self.kernel.shape[-1]]
249 outputs.set_shape(output_shape)
251 if self.use_bias:
252 outputs = tf.nn.bias_add(outputs, self.bias)
254 if self.activation is not None:
255 outputs = self.activation(outputs)
257 if is_ragged:
258 outputs = original_inputs.with_flat_values(outputs)
260 return outputs
262 def compute_output_shape(self, input_shape):
263 input_shape = tf.TensorShape(input_shape)
264 input_shape = input_shape.with_rank_at_least(2)
265 if tf.compat.dimension_value(input_shape[-1]) is None:
266 raise ValueError(
267 "The last dimension of the input shape of a Dense layer "
268 "should be defined. Found None. "
269 f"Received: input_shape={input_shape}"
270 )
271 return input_shape[:-1].concatenate(self.units)
273 def get_config(self):
274 config = super().get_config()
275 config.update(
276 {
277 "units": self.units,
278 "activation": activations.serialize(self.activation),
279 "use_bias": self.use_bias,
280 "kernel_initializer": initializers.serialize(
281 self.kernel_initializer
282 ),
283 "bias_initializer": initializers.serialize(
284 self.bias_initializer
285 ),
286 "kernel_regularizer": regularizers.serialize(
287 self.kernel_regularizer
288 ),
289 "bias_regularizer": regularizers.serialize(
290 self.bias_regularizer
291 ),
292 "activity_regularizer": regularizers.serialize(
293 self.activity_regularizer
294 ),
295 "kernel_constraint": constraints.serialize(
296 self.kernel_constraint
297 ),
298 "bias_constraint": constraints.serialize(self.bias_constraint),
299 }
300 )
301 return config