Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/keras/src/layers/rnn/dropout_rnn_cell_mixin.py: 39%

1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================== 

15"""Mixin holding dropout fields for RNN cells.""" 

16 

17 

18import tensorflow.compat.v2 as tf 

19from tensorflow.tools.docs import doc_controls 

20 

21from keras.src import backend 

22 

23 

24@doc_controls.do_not_generate_docs 

25class DropoutRNNCellMixin: 

26 """Object that hold dropout related fields for RNN Cell. 

27 

28 This class is not a standalone RNN cell. It suppose to be used with a RNN 

29 cell by multiple inheritance. Any cell that mix with class should have 

30 following fields: 

31 dropout: a float number within range [0, 1). The ratio that the input 

32 tensor need to dropout. 

33 recurrent_dropout: a float number within range [0, 1). The ratio that the 

34 recurrent state weights need to dropout. 

35 _random_generator: A backend.RandomGenerator instance, which will be used 

36 to produce outputs based on the inputs and dropout rate. 

37 This object will create and cache created dropout masks, and reuse them for 

38 the incoming data, so that the same mask is used for every batch input. 

39 """ 

40 

41 def __init__(self, *args, **kwargs): 

42 self._create_non_trackable_mask_cache() 

43 super().__init__(*args, **kwargs) 

44 

45 @tf.__internal__.tracking.no_automatic_dependency_tracking 

46 def _create_non_trackable_mask_cache(self): 

47 """Create the cache for dropout and recurrent dropout mask. 

48 

49 Note that the following two masks will be used in "graph function" mode, 

50 e.g. these masks are symbolic tensors. In eager mode, the `eager_*_mask` 

51 tensors will be generated differently than in the "graph function" case, 

52 and they will be cached. 

53 

54 Also note that in graph mode, we still cache those masks only because 

55 the RNN could be created with `unroll=True`. In that case, the 

56 `cell.call()` function will be invoked multiple times, and we want to 

57 ensure same mask is used every time. 

58 

59 Also the caches are created without tracking. Since they are not 

60 picklable by python when deepcopy, we don't want 

61 `layer._obj_reference_counts_dict` to track it by default. 

62 """ 

63 self._dropout_mask_cache = backend.ContextValueCache( 

64 self._create_dropout_mask 

65 ) 

66 self._recurrent_dropout_mask_cache = backend.ContextValueCache( 

67 self._create_recurrent_dropout_mask 

68 ) 

69 

70 def reset_dropout_mask(self): 

71 """Reset the cached dropout masks if any. 

72 

73 This is important for the RNN layer to invoke this in it `call()` method 

74 so that the cached mask is cleared before calling the `cell.call()`. The 

75 mask should be cached across the timestep within the same batch, but 

76 shouldn't be cached between batches. Otherwise it will introduce 

77 unreasonable bias against certain index of data within the batch. 

78 """ 

79 self._dropout_mask_cache.clear() 

80 

81 def reset_recurrent_dropout_mask(self): 

82 """Reset the cached recurrent dropout masks if any. 

83 

84 This is important for the RNN layer to invoke this in it call() method 

85 so that the cached mask is cleared before calling the cell.call(). The 

86 mask should be cached across the timestep within the same batch, but 

87 shouldn't be cached between batches. Otherwise it will introduce 

88 unreasonable bias against certain index of data within the batch. 

89 """ 

90 self._recurrent_dropout_mask_cache.clear() 

91 

92 def _create_dropout_mask(self, inputs, training, count=1): 

93 return _generate_dropout_mask( 

94 self._random_generator, 

95 tf.ones_like(inputs), 

96 self.dropout, 

97 training=training, 

98 count=count, 

99 ) 

100 

101 def _create_recurrent_dropout_mask(self, inputs, training, count=1): 

102 return _generate_dropout_mask( 

103 self._random_generator, 

104 tf.ones_like(inputs), 

105 self.recurrent_dropout, 

106 training=training, 

107 count=count, 

108 ) 

109 

110 def get_dropout_mask_for_cell(self, inputs, training, count=1): 

111 """Get the dropout mask for RNN cell's input. 

112 

113 It will create mask based on context if there isn't any existing cached 

114 mask. If a new mask is generated, it will update the cache in the cell. 

115 

116 Args: 

117 inputs: The input tensor whose shape will be used to generate dropout 

118 mask. 

119 training: Boolean tensor, whether its in training mode, dropout will 

120 be ignored in non-training mode. 

121 count: Int, how many dropout mask will be generated. It is useful for 

122 cell that has internal weights fused together. 

123 Returns: 

124 List of mask tensor, generated or cached mask based on context. 

125 """ 

126 if self.dropout == 0: 

127 return None 

128 init_kwargs = dict(inputs=inputs, training=training, count=count) 

129 return self._dropout_mask_cache.setdefault(kwargs=init_kwargs) 

130 

131 def get_recurrent_dropout_mask_for_cell(self, inputs, training, count=1): 

132 """Get the recurrent dropout mask for RNN cell. 

133 

134 It will create mask based on context if there isn't any existing cached 

135 mask. If a new mask is generated, it will update the cache in the cell. 

136 

137 Args: 

138 inputs: The input tensor whose shape will be used to generate dropout 

139 mask. 

140 training: Boolean tensor, whether its in training mode, dropout will 

141 be ignored in non-training mode. 

142 count: Int, how many dropout mask will be generated. It is useful for 

143 cell that has internal weights fused together. 

144 Returns: 

145 List of mask tensor, generated or cached mask based on context. 

146 """ 

147 if self.recurrent_dropout == 0: 

148 return None 

149 init_kwargs = dict(inputs=inputs, training=training, count=count) 

150 return self._recurrent_dropout_mask_cache.setdefault(kwargs=init_kwargs) 

151 

152 def __getstate__(self): 

153 # Used for deepcopy. The caching can't be pickled by python, since it 

154 # will contain tensor and graph. 

155 state = super().__getstate__() 

156 state.pop("_dropout_mask_cache", None) 

157 state.pop("_recurrent_dropout_mask_cache", None) 

158 return state 

159 

160 def __setstate__(self, state): 

161 state["_dropout_mask_cache"] = backend.ContextValueCache( 

162 self._create_dropout_mask 

163 ) 

164 state["_recurrent_dropout_mask_cache"] = backend.ContextValueCache( 

165 self._create_recurrent_dropout_mask 

166 ) 

167 super().__setstate__(state) 

168 

169 

170def _generate_dropout_mask(generator, ones, rate, training=None, count=1): 

171 def dropped_inputs(): 

172 return generator.dropout(ones, rate) 

173 

174 if count > 1: 

175 return [ 

176 backend.in_train_phase(dropped_inputs, ones, training=training) 

177 for _ in range(count) 

178 ] 

179 return backend.in_train_phase(dropped_inputs, ones, training=training) 

180