Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/ops/init_ops_v2.py: 38%

1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================== 

15"""Initializers for TF 2.""" 

16import math 

17 

18import numpy as np 

19 

20from tensorflow.python.framework import constant_op 

21from tensorflow.python.framework import dtypes 

22from tensorflow.python.ops import array_ops 

23from tensorflow.python.ops import gen_linalg_ops 

24from tensorflow.python.ops import linalg_ops_impl 

25from tensorflow.python.ops import math_ops 

26from tensorflow.python.ops import random_ops 

27from tensorflow.python.ops import stateless_random_ops 

28from tensorflow.python.ops.init_ops import _compute_fans 

29from tensorflow.python.util.tf_export import tf_export 

30 

31_PARTITION_SHAPE = "partition_shape" 

32_PARTITION_OFFSET = "partition_offset" 

33 

34 

35class Initializer: 

36 """Initializer base class: all initializers inherit from this class. 

37 

38 Initializers should implement a `__call__` method with the following 

39 signature: 

40 

41 ```python 

42 def __call__(self, shape, dtype=None, **kwargs): 

43 # returns a tensor of shape `shape` and dtype `dtype` 

44 # containing values drawn from a distribution of your choice. 

45 ``` 

46 """ 

47 

48 def __call__(self, shape, dtype=None, **kwargs): 

49 """Returns a tensor object initialized as specified by the initializer. 

50 

51 Args: 

52 shape: Shape of the tensor. 

53 dtype: Optional dtype of the tensor. If not provided will return tensor 

54 of `tf.float32`. 

55 **kwargs: Additional keyword arguments. Accepted values: 

56 `partition_shape` and `partition_offset`. Used when creating a single 

57 partition in a partitioned variable. `partition_shape` is the shape of 

58 the partition (i.e. the shape of the returned tensor) and 

59 `partition_offset` is a tuple of `int` specifying the offset of this 

60 partition w.r.t each axis. For example, a tensor of shape `(30, 100)` 

61 can be partitioned into two partitions: `p0` of shape `(10, 100)` and 

62 `p1` of shape `(20, 100)`; if the initializer is called with 

63 `partition_shape=(20, 100)` and `partition_offset=(10, 0)`, it should 

64 return the value for `p1`. 

65 """ 

66 raise NotImplementedError 

67 

68 def get_config(self): 

69 """Returns the configuration of the initializer as a JSON-serializable dict. 

70 

71 Returns: 

72 A JSON-serializable Python dict. 

73 """ 

74 return {} 

75 

76 @classmethod 

77 def from_config(cls, config): 

78 """Instantiates an initializer from a configuration dictionary. 

79 

80 Example: 

81 

82 ```python 

83 initializer = RandomUniform(-1, 1) 

84 config = initializer.get_config() 

85 initializer = RandomUniform.from_config(config) 

86 ``` 

87 

88 Args: 

89 config: A Python dictionary. 

90 It will typically be the output of `get_config`. 

91 

92 Returns: 

93 An Initializer instance. 

94 """ 

95 config.pop("dtype", None) 

96 return cls(**config) 

97 

98 def _validate_kwargs(self, kwargs, support_partition=True): 

99 for kwarg in kwargs: 

100 if kwarg not in [_PARTITION_SHAPE, _PARTITION_OFFSET]: 

101 raise TypeError( 

102 "Keyword argument should be one of " 

103 f"{list([_PARTITION_SHAPE, _PARTITION_OFFSET])}. Received: {kwarg}") 

104 elif not support_partition: 

105 raise ValueError( 

106 f"{self.__class__.__name__} initializer doesn't support " 

107 "partition-related arguments") 

108 

109 

110@tf_export("zeros_initializer", v1=[]) 

111class Zeros(Initializer): 

112 """Initializer that generates tensors initialized to 0. 

113 

114 Initializers allow you to pre-specify an initialization strategy, encoded in 

115 the Initializer object, without knowing the shape and dtype of the variable 

116 being initialized. 

117 

118 Examples: 

119 

120 >>> def make_variables(k, initializer): 

121 ... return (tf.Variable(initializer(shape=[k], dtype=tf.float32)), 

122 ... tf.Variable(initializer(shape=[k, k], dtype=tf.float32))) 

123 >>> v1, v2 = make_variables(3, tf.zeros_initializer()) 

124 >>> v1 

125 <tf.Variable ... shape=(3,) ... numpy=array([0., 0., 0.], dtype=float32)> 

126 >>> v2 

127 <tf.Variable ... shape=(3, 3) ... numpy= 

128 array([[0., 0., 0.], 

129 [0., 0., 0.], 

130 [0., 0., 0.]], dtype=float32)> 

131 >>> make_variables(4, tf.random_uniform_initializer(minval=-1., maxval=1.)) 

132 (<tf.Variable...shape=(4,) dtype=float32...>, <tf.Variable...shape=(4, 4) ... 

133 """ 

134 

135 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

136 """Returns a tensor object initialized as specified by the initializer. 

137 

138 Args: 

139 shape: Shape of the tensor. 

140 dtype: Optional dtype of the tensor. Only numeric or boolean dtypes are 

141 supported. 

142 **kwargs: Additional keyword arguments. 

143 

144 Raises: 

145 ValuesError: If the dtype is not numeric or boolean. 

146 """ 

147 self._validate_kwargs(kwargs) 

148 dtype = dtypes.as_dtype(dtype) 

149 if not dtype.is_numpy_compatible or dtype == dtypes.string: 

150 raise ValueError("Argument `dtype` expected to be numeric or boolean. " 

151 f"Received {dtype}.") 

152 if _PARTITION_SHAPE in kwargs: 

153 shape = kwargs[_PARTITION_SHAPE] 

154 return array_ops.zeros(shape, dtype) 

155 

156 

157@tf_export("ones_initializer", v1=[]) 

158class Ones(Initializer): 

159 """Initializer that generates tensors initialized to 1. 

160 

161 Initializers allow you to pre-specify an initialization strategy, encoded in 

162 the Initializer object, without knowing the shape and dtype of the variable 

163 being initialized. 

164 

165 Examples: 

166 

167 >>> def make_variables(k, initializer): 

168 ... return (tf.Variable(initializer(shape=[k], dtype=tf.float32)), 

169 ... tf.Variable(initializer(shape=[k, k], dtype=tf.float32))) 

170 >>> v1, v2 = make_variables(3, tf.ones_initializer()) 

171 >>> v1 

172 <tf.Variable ... shape=(3,) ... numpy=array([1., 1., 1.], dtype=float32)> 

173 >>> v2 

174 <tf.Variable ... shape=(3, 3) ... numpy= 

175 array([[1., 1., 1.], 

176 [1., 1., 1.], 

177 [1., 1., 1.]], dtype=float32)> 

178 >>> make_variables(4, tf.random_uniform_initializer(minval=-1., maxval=1.)) 

179 (<tf.Variable...shape=(4,) dtype=float32...>, <tf.Variable...shape=(4, 4) ... 

180 """ 

181 

182 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

183 """Returns a tensor object initialized as specified by the initializer. 

184 

185 Args: 

186 shape: Shape of the tensor. 

187 dtype: Optional dtype of the tensor. Only numeric or boolean dtypes are 

188 supported. 

189 **kwargs: Additional keyword arguments. 

190 

191 Raises: 

192 ValuesError: If the dtype is not numeric or boolean. 

193 """ 

194 self._validate_kwargs(kwargs) 

195 dtype = dtypes.as_dtype(dtype) 

196 if not dtype.is_numpy_compatible or dtype == dtypes.string: 

197 raise ValueError("Argument `dtype` expected to be numeric or boolean. " 

198 f"Received {dtype}.") 

199 if _PARTITION_SHAPE in kwargs: 

200 shape = kwargs[_PARTITION_SHAPE] 

201 return array_ops.ones(shape, dtype) 

202 

203 

204@tf_export("constant_initializer", v1=[]) 

205class Constant(Initializer): 

206 """Initializer that generates tensors with constant values. 

207 

208 Initializers allow you to pre-specify an initialization strategy, encoded in 

209 the Initializer object, without knowing the shape and dtype of the variable 

210 being initialized. 

211 

212 `tf.constant_initializer` returns an object which when called returns a tensor 

213 populated with the `value` specified in the constructor. This `value` must be 

214 convertible to the requested `dtype`. 

215 

216 The argument `value` can be a scalar constant value, or a list of 

217 values. Scalars broadcast to whichever shape is requested from the 

218 initializer. 

219 

220 If `value` is a list, then the length of the list must be equal to the number 

221 of elements implied by the desired shape of the tensor. If the total number of 

222 elements in `value` is not equal to the number of elements required by the 

223 tensor shape, the initializer will raise a `TypeError`. 

224 

225 Examples: 

226 

227 >>> def make_variables(k, initializer): 

228 ... return (tf.Variable(initializer(shape=[k], dtype=tf.float32)), 

229 ... tf.Variable(initializer(shape=[k, k], dtype=tf.float32))) 

230 >>> v1, v2 = make_variables(3, tf.constant_initializer(2.)) 

231 >>> v1 

232 <tf.Variable ... shape=(3,) ... numpy=array([2., 2., 2.], dtype=float32)> 

233 >>> v2 

234 <tf.Variable ... shape=(3, 3) ... numpy= 

235 array([[2., 2., 2.], 

236 [2., 2., 2.], 

237 [2., 2., 2.]], dtype=float32)> 

238 >>> make_variables(4, tf.random_uniform_initializer(minval=-1., maxval=1.)) 

239 (<tf.Variable...shape=(4,) dtype=float32...>, <tf.Variable...shape=(4, 4) ... 

240 

241 >>> value = [0, 1, 2, 3, 4, 5, 6, 7] 

242 >>> init = tf.constant_initializer(value) 

243 >>> # Fitting shape 

244 >>> tf.Variable(init(shape=[2, 4], dtype=tf.float32)) 

245 <tf.Variable ... 

246 array([[0., 1., 2., 3.], 

247 [4., 5., 6., 7.]], dtype=float32)> 

248 >>> # Larger shape 

249 >>> tf.Variable(init(shape=[3, 4], dtype=tf.float32)) 

250 Traceback (most recent call last): 

251 ... 

252 TypeError: ...value has 8 elements, shape is (3, 4) with 12 elements... 

253 >>> # Smaller shape 

254 >>> tf.Variable(init(shape=[2, 3], dtype=tf.float32)) 

255 Traceback (most recent call last): 

256 ... 

257 TypeError: ...value has 8 elements, shape is (2, 3) with 6 elements... 

258 

259 Args: 

260 value: A Python scalar, list or tuple of values, or a N-dimensional numpy 

261 array. All elements of the initialized variable will be set to the 

262 corresponding value in the `value` argument. 

263 

264 Raises: 

265 TypeError: If the input `value` is not one of the expected types. 

266 """ 

267 

268 def __init__(self, value=0): 

269 if not (np.isscalar(value) or isinstance(value, (list, tuple, np.ndarray))): 

270 raise TypeError( 

271 f"Invalid type for initial value: {type(value).__name__}. Expected " 

272 "Python scalar, list or tuple of values, or numpy.ndarray.") 

273 self.value = value 

274 

275 def __call__(self, shape, dtype=None, **kwargs): 

276 """Returns a tensor object initialized as specified by the initializer. 

277 

278 Args: 

279 shape: Shape of the tensor. 

280 dtype: Optional dtype of the tensor. If not provided the dtype of the 

281 tensor created will be the type of the inital value. 

282 **kwargs: Additional keyword arguments. 

283 

284 Raises: 

285 TypeError: If the initializer cannot create a tensor of the requested 

286 dtype. 

287 """ 

288 self._validate_kwargs(kwargs, support_partition=False) 

289 if dtype is not None: 

290 dtype = dtypes.as_dtype(dtype) 

291 return constant_op.constant(self.value, dtype=dtype, shape=shape) 

292 

293 def get_config(self): 

294 return {"value": self.value} 

295 

296 

297@tf_export("random_uniform_initializer", v1=[]) 

298class RandomUniform(Initializer): 

299 """Initializer that generates tensors with a uniform distribution. 

300 

301 Initializers allow you to pre-specify an initialization strategy, encoded in 

302 the Initializer object, without knowing the shape and dtype of the variable 

303 being initialized. 

304 

305 Examples: 

306 

307 >>> def make_variables(k, initializer): 

308 ... return (tf.Variable(initializer(shape=[k], dtype=tf.float32)), 

309 ... tf.Variable(initializer(shape=[k, k], dtype=tf.float32))) 

310 >>> v1, v2 = make_variables(3, tf.ones_initializer()) 

311 >>> v1 

312 <tf.Variable ... shape=(3,) ... numpy=array([1., 1., 1.], dtype=float32)> 

313 >>> v2 

314 <tf.Variable ... shape=(3, 3) ... numpy= 

315 array([[1., 1., 1.], 

316 [1., 1., 1.], 

317 [1., 1., 1.]], dtype=float32)> 

318 >>> make_variables(4, tf.random_uniform_initializer(minval=-1., maxval=1.)) 

319 (<tf.Variable...shape=(4,) dtype=float32...>, <tf.Variable...shape=(4, 4) ... 

320 

321 Args: 

322 minval: A python scalar or a scalar tensor. Lower bound of the range of 

323 random values to generate (inclusive). 

324 maxval: A python scalar or a scalar tensor. Upper bound of the range of 

325 random values to generate (exclusive). 

326 seed: A Python integer. Used to create random seeds. See 

327 `tf.random.set_seed` for behavior. 

328 """ 

329 

330 def __init__(self, minval=-0.05, maxval=0.05, seed=None): 

331 self.minval = minval 

332 self.maxval = maxval 

333 self.seed = seed 

334 self._random_generator = _RandomGenerator(seed) 

335 

336 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

337 """Returns a tensor object initialized as specified by the initializer. 

338 

339 Args: 

340 shape: Shape of the tensor. 

341 dtype: Optional dtype of the tensor. Only floating point and integer 

342 types are supported. 

343 **kwargs: Additional keyword arguments. 

344 

345 Raises: 

346 ValueError: If the dtype is not numeric. 

347 """ 

348 self._validate_kwargs(kwargs) 

349 dtype = dtypes.as_dtype(dtype) 

350 if not dtype.is_floating and not dtype.is_integer: 

351 raise ValueError("Argument `dtype` expected to be numeric or boolean. " 

352 f"Received {dtype}.") 

353 if _PARTITION_SHAPE in kwargs: 

354 shape = kwargs[_PARTITION_SHAPE] 

355 return self._random_generator.random_uniform(shape, self.minval, 

356 self.maxval, dtype) 

357 

358 def get_config(self): 

359 return { 

360 "minval": self.minval, 

361 "maxval": self.maxval, 

362 "seed": self.seed 

363 } 

364 

365 

366@tf_export("random_normal_initializer", v1=[]) 

367class RandomNormal(Initializer): 

368 """Initializer that generates tensors with a normal distribution. 

369 

370 Initializers allow you to pre-specify an initialization strategy, encoded in 

371 the Initializer object, without knowing the shape and dtype of the variable 

372 being initialized. 

373 

374 Examples: 

375 

376 >>> def make_variables(k, initializer): 

377 ... return (tf.Variable(initializer(shape=[k], dtype=tf.float32)), 

378 ... tf.Variable(initializer(shape=[k, k], dtype=tf.float32))) 

379 >>> v1, v2 = make_variables(3, 

380 ... tf.random_normal_initializer(mean=1., stddev=2.)) 

381 >>> v1 

382 <tf.Variable ... shape=(3,) ... numpy=array([...], dtype=float32)> 

383 >>> v2 

384 <tf.Variable ... shape=(3, 3) ... numpy= 

385 ... 

386 >>> make_variables(4, tf.random_uniform_initializer(minval=-1., maxval=1.)) 

387 (<tf.Variable...shape=(4,) dtype=float32...>, <tf.Variable...shape=(4, 4) ... 

388 

389 Args: 

390 mean: a python scalar or a scalar tensor. Mean of the random values to 

391 generate. 

392 stddev: a python scalar or a scalar tensor. Standard deviation of the random 

393 values to generate. 

394 seed: A Python integer. Used to create random seeds. See 

395 `tf.random.set_seed` for behavior. 

396 

397 """ 

398 

399 def __init__(self, mean=0.0, stddev=0.05, seed=None): 

400 self.mean = mean 

401 self.stddev = stddev 

402 self.seed = seed 

403 self._random_generator = _RandomGenerator(seed) 

404 

405 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

406 """Returns a tensor object initialized as specified by the initializer. 

407 

408 Args: 

409 shape: Shape of the tensor. 

410 dtype: Optional dtype of the tensor. Only floating point types are 

411 supported. 

412 **kwargs: Additional keyword arguments. 

413 

414 Raises: 

415 ValueError: If the dtype is not floating point 

416 """ 

417 self._validate_kwargs(kwargs) 

418 dtype = _assert_float_dtype(dtype) 

419 if _PARTITION_SHAPE in kwargs: 

420 shape = kwargs[_PARTITION_SHAPE] 

421 return self._random_generator.random_normal(shape, self.mean, self.stddev, 

422 dtype) 

423 

424 def get_config(self): 

425 return { 

426 "mean": self.mean, 

427 "stddev": self.stddev, 

428 "seed": self.seed 

429 } 

430 

431 

432class TruncatedNormal(Initializer): 

433 """Initializer that generates a truncated normal distribution. 

434 

435 Initializers allow you to pre-specify an initialization strategy, encoded in 

436 the Initializer object, without knowing the shape and dtype of the variable 

437 being initialized. 

438 

439 These values are similar to values from a `tf.initializers.RandomNormal` 

440 except that values more than two standard deviations from the mean are 

441 discarded and re-drawn. This is the recommended initializer for neural network 

442 weights and filters. 

443 

444 Examples: 

445 

446 >>> def make_variables(k, initializer): 

447 ... return (tf.Variable(initializer(shape=[k], dtype=tf.float32)), 

448 ... tf.Variable(initializer(shape=[k, k], dtype=tf.float32))) 

449 >>> v1, v2 = make_variables( 

450 ... 3, tf.initializers.TruncatedNormal(mean=1., stddev=2.)) 

451 >>> v1 

452 <tf.Variable ... shape=(3,) ... numpy=array([...], dtype=float32)> 

453 >>> v2 

454 <tf.Variable ... shape=(3, 3) ... numpy= 

455 ... 

456 >>> make_variables(4, tf.initializers.RandomUniform(minval=-1., maxval=1.)) 

457 (<tf.Variable...shape=(4,) dtype=float32...>, <tf.Variable...shape=(4, 4) ... 

458 

459 Args: 

460 mean: a python scalar or a scalar tensor. Mean of the random values 

461 to generate. 

462 stddev: a python scalar or a scalar tensor. Standard deviation of the 

463 random values to generate. 

464 seed: A Python integer. Used to create random seeds. See 

465 `tf.random.set_seed` for behavior. 

466 """ 

467 

468 def __init__(self, mean=0.0, stddev=0.05, seed=None): 

469 self.mean = mean 

470 self.stddev = stddev 

471 self.seed = seed 

472 self._random_generator = _RandomGenerator(seed) 

473 

474 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

475 """Returns a tensor object initialized as specified by the initializer. 

476 

477 Args: 

478 shape: Shape of the tensor. 

479 dtype: Optional dtype of the tensor. Only floating point types are 

480 supported. 

481 **kwargs: Additional keyword arguments. 

482 

483 Raises: 

484 ValueError: If the dtype is not floating point 

485 """ 

486 self._validate_kwargs(kwargs) 

487 dtype = _assert_float_dtype(dtype) 

488 if _PARTITION_SHAPE in kwargs: 

489 shape = kwargs[_PARTITION_SHAPE] 

490 return self._random_generator.truncated_normal(shape, self.mean, 

491 self.stddev, dtype) 

492 

493 def get_config(self): 

494 return { 

495 "mean": self.mean, 

496 "stddev": self.stddev, 

497 "seed": self.seed 

498 } 

499 

500 

501class VarianceScaling(Initializer): 

502 """Initializer capable of adapting its scale to the shape of weights tensors. 

503 

504 Initializers allow you to pre-specify an initialization strategy, encoded in 

505 the Initializer object, without knowing the shape and dtype of the variable 

506 being initialized. 

507 

508 With `distribution="truncated_normal" or "untruncated_normal"`, samples are 

509 drawn from a truncated/untruncated normal distribution with a mean of zero and 

510 a standard deviation (after truncation, if used) `stddev = sqrt(scale / n)` 

511 where n is: 

512 

513 - number of input units in the weight tensor, if mode = "fan_in" 

514 - number of output units, if mode = "fan_out" 

515 - average of the numbers of input and output units, if mode = "fan_avg" 

516 

517 With `distribution="uniform"`, samples are drawn from a uniform distribution 

518 within [-limit, limit], with `limit = sqrt(3 * scale / n)`. 

519 

520 Examples: 

521 

522 >>> def make_variables(k, initializer): 

523 ... return (tf.Variable(initializer(shape=[k], dtype=tf.float32)), 

524 ... tf.Variable(initializer(shape=[k, k], dtype=tf.float32))) 

525 >>> v1, v2 = make_variables(3, tf.initializers.VarianceScaling(scale=1.)) 

526 >>> v1 

527 <tf.Variable ... shape=(3,) ... numpy=array([...], dtype=float32)> 

528 >>> v2 

529 <tf.Variable ... shape=(3, 3) ... numpy= 

530 ... 

531 >>> make_variables(4, tf.initializers.VarianceScaling(distribution='uniform')) 

532 (<tf.Variable...shape=(4,) dtype=float32...>, <tf.Variable...shape=(4, 4) ... 

533 

534 Args: 

535 scale: Scaling factor (positive float). 

536 mode: One of "fan_in", "fan_out", "fan_avg". 

537 distribution: Random distribution to use. One of "truncated_normal", 

538 "untruncated_normal" and "uniform". 

539 seed: A Python integer. Used to create random seeds. See 

540 `tf.random.set_seed` for behavior. 

541 

542 Raises: 

543 ValueError: In case of an invalid value for the "scale", mode" or 

544 "distribution" arguments. 

545 """ 

546 

547 def __init__(self, 

548 scale=1.0, 

549 mode="fan_in", 

550 distribution="truncated_normal", 

551 seed=None): 

552 if scale <= 0.: 

553 raise ValueError("Argument `scale` must be a positive float. Received: " 

554 f"{scale}") 

555 if mode not in {"fan_in", "fan_out", "fan_avg"}: 

556 raise ValueError("Argument `mode` should be one of ('fan_in', 'fan_out', " 

557 f"'fan_avg'). Received: {mode}") 

558 distribution = distribution.lower() 

559 # Compatibility with keras-team/keras. 

560 if distribution == "normal": 

561 distribution = "truncated_normal" 

562 if distribution not in {"uniform", "truncated_normal", 

563 "untruncated_normal"}: 

564 raise ValueError("Argument `distribution` should be one of ('uniform', " 

565 "'truncated_normal', 'untruncated_normal'). Received: " 

566 f"{distribution}") 

567 self.scale = scale 

568 self.mode = mode 

569 self.distribution = distribution 

570 self.seed = seed 

571 self._random_generator = _RandomGenerator(seed) 

572 

573 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

574 """Returns a tensor object initialized as specified by the initializer. 

575 

576 Args: 

577 shape: Shape of the tensor. 

578 dtype: Optional dtype of the tensor. Only floating point types are 

579 supported. 

580 **kwargs: Additional keyword arguments. 

581 

582 Raises: 

583 ValueError: If the dtype is not floating point 

584 """ 

585 self._validate_kwargs(kwargs) 

586 dtype = _assert_float_dtype(dtype) 

587 scale = self.scale 

588 fan_in, fan_out = _compute_fans(shape) 

589 if _PARTITION_SHAPE in kwargs: 

590 shape = kwargs[_PARTITION_SHAPE] 

591 if self.mode == "fan_in": 

592 scale /= max(1., fan_in) 

593 elif self.mode == "fan_out": 

594 scale /= max(1., fan_out) 

595 else: 

596 scale /= max(1., (fan_in + fan_out) / 2.) 

597 if self.distribution == "truncated_normal": 

598 # constant from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) 

599 stddev = math.sqrt(scale) / .87962566103423978 

600 return self._random_generator.truncated_normal(shape, 0.0, stddev, dtype) 

601 elif self.distribution == "untruncated_normal": 

602 stddev = math.sqrt(scale) 

603 return self._random_generator.random_normal(shape, 0.0, stddev, dtype) 

604 else: 

605 limit = math.sqrt(3.0 * scale) 

606 return self._random_generator.random_uniform(shape, -limit, limit, dtype) 

607 

608 def get_config(self): 

609 return { 

610 "scale": self.scale, 

611 "mode": self.mode, 

612 "distribution": self.distribution, 

613 "seed": self.seed 

614 } 

615 

616 

617class Orthogonal(Initializer): 

618 """Initializer that generates an orthogonal matrix. 

619 

620 Initializers allow you to pre-specify an initialization strategy, encoded in 

621 the Initializer object, without knowing the shape and dtype of the variable 

622 being initialized. 

623 

624 If the shape of the tensor to initialize is two-dimensional, it is initialized 

625 with an orthogonal matrix obtained from the QR decomposition of a matrix of 

626 random numbers drawn from a normal distribution. 

627 If the matrix has fewer rows than columns then the output will have orthogonal 

628 rows. Otherwise, the output will have orthogonal columns. 

629 

630 If the shape of the tensor to initialize is more than two-dimensional, 

631 a matrix of shape `(shape[0] * ... * shape[n - 2], shape[n - 1])` 

632 is initialized, where `n` is the length of the shape vector. 

633 The matrix is subsequently reshaped to give a tensor of the desired shape. 

634 

635 Examples: 

636 

637 >>> def make_variables(k, initializer): 

638 ... return (tf.Variable(initializer(shape=[k, k], dtype=tf.float32)), 

639 ... tf.Variable(initializer(shape=[k, k, k], dtype=tf.float32))) 

640 >>> v1, v2 = make_variables(3, tf.initializers.Orthogonal()) 

641 >>> v1 

642 <tf.Variable ... shape=(3, 3) ... 

643 >>> v2 

644 <tf.Variable ... shape=(3, 3, 3) ... 

645 >>> make_variables(4, tf.initializers.Orthogonal(gain=0.5)) 

646 (<tf.Variable ... shape=(4, 4) dtype=float32... 

647 <tf.Variable ... shape=(4, 4, 4) dtype=float32... 

648 

649 Args: 

650 gain: multiplicative factor to apply to the orthogonal matrix 

651 seed: A Python integer. Used to create random seeds. See 

652 `tf.random.set_seed` for behavior. 

653 

654 References: 

655 [Saxe et al., 2014](https://openreview.net/forum?id=_wzZwKpTDF_9C) 

656 ([pdf](https://arxiv.org/pdf/1312.6120.pdf)) 

657 """ 

658 

659 def __init__(self, gain=1.0, seed=None): 

660 self.gain = gain 

661 self.seed = seed 

662 self._random_generator = _RandomGenerator(seed) 

663 

664 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

665 """Returns a tensor object initialized as specified by the initializer. 

666 

667 Args: 

668 shape: Shape of the tensor. 

669 dtype: Optional dtype of the tensor. Only floating point types are 

670 supported. 

671 **kwargs: Additional keyword arguments. 

672 

673 Raises: 

674 ValueError: If the dtype is not floating point or the input shape is not 

675 valid. 

676 """ 

677 self._validate_kwargs(kwargs, support_partition=False) 

678 dtype = _assert_float_dtype(dtype) 

679 # Check the shape 

680 if len(shape) < 2: 

681 raise ValueError("The tensor to initialize, specified by argument `shape`" 

682 " must be at least two-dimensional. Received shape=" 

683 f"{shape}") 

684 # Flatten the input shape with the last dimension remaining 

685 # its original shape so it works for conv2d 

686 num_rows = 1 

687 for dim in shape[:-1]: 

688 num_rows *= dim 

689 num_cols = shape[-1] 

690 flat_shape = (max(num_cols, num_rows), min(num_cols, num_rows)) 

691 

692 # Generate a random matrix 

693 a = self._random_generator.random_normal(flat_shape, dtype=dtype) 

694 # Compute the qr factorization 

695 q, r = gen_linalg_ops.qr(a, full_matrices=False) 

696 # Make Q uniform 

697 d = array_ops.diag_part(r) 

698 q *= math_ops.sign(d) 

699 if num_rows < num_cols: 

700 q = array_ops.matrix_transpose(q) 

701 return self.gain * array_ops.reshape(q, shape) 

702 

703 def get_config(self): 

704 return {"gain": self.gain, "seed": self.seed} 

705 

706 

707class Identity(Initializer): 

708 """Initializer that generates the identity matrix. 

709 

710 Initializers allow you to pre-specify an initialization strategy, encoded in 

711 the Initializer object, without knowing the shape and dtype of the variable 

712 being initialized. 

713 

714 Only usable for generating 2D matrices. 

715 

716 Examples: 

717 

718 >>> def make_variable(k, initializer): 

719 ... return tf.Variable(initializer(shape=[k, k], dtype=tf.float32)) 

720 >>> make_variable(2, tf.initializers.Identity()) 

721 <tf.Variable ... shape=(2, 2) dtype=float32, numpy= 

722 array([[1., 0.], 

723 [0., 1.]], dtype=float32)> 

724 >>> make_variable(3, tf.initializers.Identity(gain=0.5)) 

725 <tf.Variable ... shape=(3, 3) dtype=float32, numpy= 

726 array([[0.5, 0. , 0. ], 

727 [0. , 0.5, 0. ], 

728 [0. , 0. , 0.5]], dtype=float32)> 

729 

730 Args: 

731 gain: Multiplicative factor to apply to the identity matrix. 

732 """ 

733 

734 def __init__(self, gain=1.0): 

735 self.gain = gain 

736 

737 def __call__(self, shape, dtype=dtypes.float32, **kwargs): 

738 """Returns a tensor object initialized as specified by the initializer. 

739 

740 Args: 

741 shape: Shape of the tensor. 

742 dtype: Optional dtype of the tensor. Only floating point types are 

743 supported. 

744 **kwargs: Additional keyword arguments. 

745 

746 Raises: 

747 ValueError: If the dtype is not floating point 

748 ValueError: If the requested shape does not have exactly two axes. 

749 """ 

750 self._validate_kwargs(kwargs, support_partition=False) 

751 dtype = _assert_float_dtype(dtype) 

752 if len(shape) != 2: 

753 raise ValueError("The tensor to initialize, specified by argument `shape`" 

754 " must be at least two-dimensional. Received shape=" 

755 f"{shape}") 

756 initializer = linalg_ops_impl.eye(*shape, dtype=dtype) 

757 return self.gain * initializer 

758 

759 def get_config(self): 

760 return {"gain": self.gain} 

761 

762 

763class GlorotUniform(VarianceScaling): 

764 """The Glorot uniform initializer, also called Xavier uniform initializer. 

765 

766 Initializers allow you to pre-specify an initialization strategy, encoded in 

767 the Initializer object, without knowing the shape and dtype of the variable 

768 being initialized. 

769 

770 Draws samples from a uniform distribution within [-limit, limit] where `limit` 

771 is `sqrt(6 / (fan_in + fan_out))` where `fan_in` is the number of input units 

772 in the weight tensor and `fan_out` is the number of output units in the weight 

773 tensor. 

774 

775 Examples: 

776 

777 >>> def make_variables(k, initializer): 

778 ... return (tf.Variable(initializer(shape=[k, k], dtype=tf.float32)), 

779 ... tf.Variable(initializer(shape=[k, k, k], dtype=tf.float32))) 

780 >>> v1, v2 = make_variables(3, tf.initializers.GlorotUniform()) 

781 >>> v1 

782 <tf.Variable ... shape=(3, 3) ... 

783 >>> v2 

784 <tf.Variable ... shape=(3, 3, 3) ... 

785 >>> make_variables(4, tf.initializers.RandomNormal()) 

786 (<tf.Variable ... shape=(4, 4) dtype=float32... 

787 <tf.Variable ... shape=(4, 4, 4) dtype=float32... 

788 

789 Args: 

790 seed: A Python integer. Used to create random seeds. See 

791 `tf.random.set_seed` for behavior. 

792 

793 References: 

794 [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html) 

795 ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) 

796 """ 

797 

798 def __init__(self, seed=None): 

799 super(GlorotUniform, self).__init__( 

800 scale=1.0, 

801 mode="fan_avg", 

802 distribution="uniform", 

803 seed=seed) 

804 

805 def get_config(self): 

806 return {"seed": self.seed} 

807 

808 

809class GlorotNormal(VarianceScaling): 

810 """The Glorot normal initializer, also called Xavier normal initializer. 

811 

812 Initializers allow you to pre-specify an initialization strategy, encoded in 

813 the Initializer object, without knowing the shape and dtype of the variable 

814 being initialized. 

815 

816 Draws samples from a truncated normal distribution centered on 0 with `stddev 

817 = sqrt(2 / (fan_in + fan_out))` where `fan_in` is the number of input units in 

818 the weight tensor and `fan_out` is the number of output units in the weight 

819 tensor. 

820 

821 Examples: 

822 

823 >>> def make_variables(k, initializer): 

824 ... return (tf.Variable(initializer(shape=[k, k], dtype=tf.float32)), 

825 ... tf.Variable(initializer(shape=[k, k, k], dtype=tf.float32))) 

826 >>> v1, v2 = make_variables(3, tf.initializers.GlorotNormal()) 

827 >>> v1 

828 <tf.Variable ... shape=(3, 3) ... 

829 >>> v2 

830 <tf.Variable ... shape=(3, 3, 3) ... 

831 >>> make_variables(4, tf.initializers.RandomNormal()) 

832 (<tf.Variable ... shape=(4, 4) dtype=float32... 

833 <tf.Variable ... shape=(4, 4, 4) dtype=float32... 

834 

835 Args: 

836 seed: A Python integer. Used to create random seeds. See 

837 `tf.random.set_seed` for behavior. 

838 

839 References: 

840 [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html) 

841 ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) 

842 """ 

843 

844 def __init__(self, seed=None): 

845 super(GlorotNormal, self).__init__( 

846 scale=1.0, 

847 mode="fan_avg", 

848 distribution="truncated_normal", 

849 seed=seed) 

850 

851 def get_config(self): 

852 return {"seed": self.seed} 

853 

854 

855# Aliases. 

856 

857# pylint: disable=invalid-name 

858zeros_initializer = Zeros 

859ones_initializer = Ones 

860constant_initializer = Constant 

861random_uniform_initializer = RandomUniform 

862random_normal_initializer = RandomNormal 

863truncated_normal_initializer = TruncatedNormal 

864variance_scaling_initializer = VarianceScaling 

865glorot_uniform_initializer = GlorotUniform 

866glorot_normal_initializer = GlorotNormal 

867orthogonal_initializer = Orthogonal 

868identity_initializer = Identity 

869# pylint: enable=invalid-name 

870 

871 

872def lecun_normal(seed=None): 

873 """LeCun normal initializer. 

874 

875 Initializers allow you to pre-specify an initialization strategy, encoded in 

876 the Initializer object, without knowing the shape and dtype of the variable 

877 being initialized. 

878 

879 Draws samples from a truncated normal distribution centered on 0 with `stddev 

880 = sqrt(1 / fan_in)` where `fan_in` is the number of input units in the weight 

881 tensor. 

882 

883 Examples: 

884 

885 >>> def make_variables(k, initializer): 

886 ... return (tf.Variable(initializer(shape=[k, k], dtype=tf.float32)), 

887 ... tf.Variable(initializer(shape=[k, k, k], dtype=tf.float32))) 

888 >>> v1, v2 = make_variables(3, tf.initializers.lecun_normal()) 

889 >>> v1 

890 <tf.Variable ... shape=(3, 3) ... 

891 >>> v2 

892 <tf.Variable ... shape=(3, 3, 3) ... 

893 >>> make_variables(4, tf.initializers.RandomNormal()) 

894 (<tf.Variable ... shape=(4, 4) dtype=float32... 

895 <tf.Variable ... shape=(4, 4, 4) dtype=float32... 

896 

897 Args: 

898 seed: A Python integer. Used to seed the random generator. 

899 

900 Returns: 

901 A callable Initializer with `shape` and `dtype` arguments which generates a 

902 tensor. 

903 

904 References: 

905 - Self-Normalizing Neural Networks, 

906 [Klambauer et al., 2017] 

907 (https://papers.nips.cc/paper/6698-self-normalizing-neural-networks) 

908 ([pdf] 

909 (https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf)) 

910 - Efficient Backprop, 

911 [Lecun et al., 1998](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) 

912 """ 

913 return VarianceScaling( 

914 scale=1., mode="fan_in", distribution="truncated_normal", seed=seed) 

915 

916 

917def lecun_uniform(seed=None): 

918 """LeCun uniform initializer. 

919 

920 Initializers allow you to pre-specify an initialization strategy, encoded in 

921 the Initializer object, without knowing the shape and dtype of the variable 

922 being initialized. 

923 

924 Draws samples from a uniform distribution within [-limit, limit] where `limit` 

925 is `sqrt(3 / fan_in)` where `fan_in` is the number of input units in the 

926 weight tensor. 

927 

928 Examples: 

929 

930 >>> def make_variables(k, initializer): 

931 ... return (tf.Variable(initializer(shape=[k, k], dtype=tf.float32)), 

932 ... tf.Variable(initializer(shape=[k, k, k], dtype=tf.float32))) 

933 >>> v1, v2 = make_variables(3, tf.initializers.lecun_uniform()) 

934 >>> v1 

935 <tf.Variable ... shape=(3, 3) ... 

936 >>> v2 

937 <tf.Variable ... shape=(3, 3, 3) ... 

938 >>> make_variables(4, tf.initializers.RandomNormal()) 

939 (<tf.Variable ... shape=(4, 4) dtype=float32... 

940 <tf.Variable ... shape=(4, 4, 4) dtype=float32... 

941 

942 Args: 

943 seed: A Python integer. Used to seed the random generator. 

944 

945 Returns: 

946 A callable Initializer with `shape` and `dtype` arguments which generates a 

947 tensor. 

948 

949 References: 

950 - Self-Normalizing Neural Networks, 

951 [Klambauer et al., 2017](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks) # pylint: disable=line-too-long 

952 ([pdf](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf)) 

953 - Efficient Backprop, 

954 [Lecun et al., 1998](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) 

955 """ 

956 return VarianceScaling( 

957 scale=1., mode="fan_in", distribution="uniform", seed=seed) 

958 

959 

960def he_normal(seed=None): 

961 """He normal initializer. 

962 

963 Initializers allow you to pre-specify an initialization strategy, encoded in 

964 the Initializer object, without knowing the shape and dtype of the variable 

965 being initialized. 

966 

967 It draws samples from a truncated normal distribution centered on 0 with 

968 `stddev = sqrt(2 / fan_in)` where `fan_in` is the number of input units in the 

969 weight tensor. 

970 

971 Examples: 

972 

973 >>> def make_variables(k, initializer): 

974 ... return (tf.Variable(initializer(shape=[k, k], dtype=tf.float32)), 

975 ... tf.Variable(initializer(shape=[k, k, k], dtype=tf.float32))) 

976 >>> v1, v2 = make_variables(3, tf.initializers.he_normal()) 

977 >>> v1 

978 <tf.Variable ... shape=(3, 3) ... 

979 >>> v2 

980 <tf.Variable ... shape=(3, 3, 3) ... 

981 >>> make_variables(4, tf.initializers.RandomNormal()) 

982 (<tf.Variable ... shape=(4, 4) dtype=float32... 

983 <tf.Variable ... shape=(4, 4, 4) dtype=float32... 

984 

985 Args: 

986 seed: A Python integer. Used to seed the random generator. 

987 

988 Returns: 

989 A callable Initializer with `shape` and `dtype` arguments which generates a 

990 tensor. 

991 

992 References: 

993 [He et al., 2015](https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html) # pylint: disable=line-too-long 

994 ([pdf](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf)) 

995 """ 

996 return VarianceScaling( 

997 scale=2., mode="fan_in", distribution="truncated_normal", seed=seed) 

998 

999 

1000def he_uniform(seed=None): 

1001 """He uniform variance scaling initializer. 

1002 

1003 Initializers allow you to pre-specify an initialization strategy, encoded in 

1004 the Initializer object, without knowing the shape and dtype of the variable 

1005 being initialized. 

1006 

1007 Draws samples from a uniform distribution within [-limit, limit] where `limit` 

1008 is `sqrt(6 / fan_in)` where `fan_in` is the number of input units in the 

1009 weight tensor. 

1010 

1011 Examples: 

1012 

1013 >>> def make_variables(k, initializer): 

1014 ... return (tf.Variable(initializer(shape=[k, k], dtype=tf.float32)), 

1015 ... tf.Variable(initializer(shape=[k, k, k], dtype=tf.float32))) 

1016 >>> v1, v2 = make_variables(3, tf.initializers.he_uniform()) 

1017 >>> v1 

1018 <tf.Variable ... shape=(3, 3) ... 

1019 >>> v2 

1020 <tf.Variable ... shape=(3, 3, 3) ... 

1021 >>> make_variables(4, tf.initializers.RandomNormal()) 

1022 (<tf.Variable ... shape=(4, 4) dtype=float32... 

1023 <tf.Variable ... shape=(4, 4, 4) dtype=float32... 

1024 

1025 Args: 

1026 seed: A Python integer. Used to seed the random generator. 

1027 

1028 Returns: 

1029 A callable Initializer with `shape` and `dtype` arguments which generates a 

1030 tensor. 

1031 

1032 References: 

1033 [He et al., 2015](https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html) # pylint: disable=line-too-long 

1034 ([pdf](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf)) 

1035 """ 

1036 return VarianceScaling( 

1037 scale=2., mode="fan_in", distribution="uniform", seed=seed) 

1038 

1039 

1040# Utility functions. 

1041 

1042 

1043def _assert_float_dtype(dtype): 

1044 """Validate and return floating point type based on `dtype`. 

1045 

1046 `dtype` must be a floating point type. 

1047 

1048 Args: 

1049 dtype: The data type to validate. 

1050 

1051 Returns: 

1052 Validated type. 

1053 

1054 Raises: 

1055 ValueError: if `dtype` is not a floating point type. 

1056 """ 

1057 dtype = dtypes.as_dtype(dtype) 

1058 if not dtype.is_floating: 

1059 raise ValueError("Argument `dtype` is expected to be floating point. " 

1060 f"Received: {dtype}.") 

1061 return dtype 

1062 

1063 

1064class _RandomGenerator: 

1065 """Random generator that selects appropriate random ops.""" 

1066 

1067 def __init__(self, seed=None): 

1068 super(_RandomGenerator, self).__init__() 

1069 if seed is not None: 

1070 # Stateless random ops requires 2-int seed. 

1071 self.seed = [seed, 0] 

1072 else: 

1073 self.seed = None 

1074 

1075 def random_normal(self, shape, mean=0.0, stddev=1, dtype=dtypes.float32): 

1076 """A deterministic random normal if seed is passed.""" 

1077 if self.seed: 

1078 op = stateless_random_ops.stateless_random_normal 

1079 else: 

1080 op = random_ops.random_normal 

1081 return op( 

1082 shape=shape, mean=mean, stddev=stddev, dtype=dtype, seed=self.seed) 

1083 

1084 def random_uniform(self, shape, minval, maxval, dtype):