Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/redis/_cache.py: 42%

1import copy 

2import random 

3import time 

4from abc import ABC, abstractmethod 

5from collections import OrderedDict, defaultdict 

6from enum import Enum 

7from typing import List 

8 

9from redis.typing import KeyT, ResponseT 

10 

11DEFAULT_EVICTION_POLICY = "lru" 

12 

13 

14DEFAULT_BLACKLIST = [ 

15 "BF.CARD", 

16 "BF.DEBUG", 

17 "BF.EXISTS", 

18 "BF.INFO", 

19 "BF.MEXISTS", 

20 "BF.SCANDUMP", 

21 "CF.COMPACT", 

22 "CF.COUNT", 

23 "CF.DEBUG", 

24 "CF.EXISTS", 

25 "CF.INFO", 

26 "CF.MEXISTS", 

27 "CF.SCANDUMP", 

28 "CMS.INFO", 

29 "CMS.QUERY", 

30 "DUMP", 

31 "EXPIRETIME", 

32 "FT.AGGREGATE", 

33 "FT.ALIASADD", 

34 "FT.ALIASDEL", 

35 "FT.ALIASUPDATE", 

36 "FT.CURSOR", 

37 "FT.EXPLAIN", 

38 "FT.EXPLAINCLI", 

39 "FT.GET", 

40 "FT.INFO", 

41 "FT.MGET", 

42 "FT.PROFILE", 

43 "FT.SEARCH", 

44 "FT.SPELLCHECK", 

45 "FT.SUGGET", 

46 "FT.SUGLEN", 

47 "FT.SYNDUMP", 

48 "FT.TAGVALS", 

49 "FT._ALIASADDIFNX", 

50 "FT._ALIASDELIFX", 

51 "HRANDFIELD", 

52 "JSON.DEBUG", 

53 "PEXPIRETIME", 

54 "PFCOUNT", 

55 "PTTL", 

56 "SRANDMEMBER", 

57 "TDIGEST.BYRANK", 

58 "TDIGEST.BYREVRANK", 

59 "TDIGEST.CDF", 

60 "TDIGEST.INFO", 

61 "TDIGEST.MAX", 

62 "TDIGEST.MIN", 

63 "TDIGEST.QUANTILE", 

64 "TDIGEST.RANK", 

65 "TDIGEST.REVRANK", 

66 "TDIGEST.TRIMMED_MEAN", 

67 "TOPK.INFO", 

68 "TOPK.LIST", 

69 "TOPK.QUERY", 

70 "TOUCH", 

71 "TTL", 

72] 

73 

74 

75DEFAULT_WHITELIST = [ 

76 "BITCOUNT", 

77 "BITFIELD_RO", 

78 "BITPOS", 

79 "EXISTS", 

80 "GEODIST", 

81 "GEOHASH", 

82 "GEOPOS", 

83 "GEORADIUSBYMEMBER_RO", 

84 "GEORADIUS_RO", 

85 "GEOSEARCH", 

86 "GET", 

87 "GETBIT", 

88 "GETRANGE", 

89 "HEXISTS", 

90 "HGET", 

91 "HGETALL", 

92 "HKEYS", 

93 "HLEN", 

94 "HMGET", 

95 "HSTRLEN", 

96 "HVALS", 

97 "JSON.ARRINDEX", 

98 "JSON.ARRLEN", 

99 "JSON.GET", 

100 "JSON.MGET", 

101 "JSON.OBJKEYS", 

102 "JSON.OBJLEN", 

103 "JSON.RESP", 

104 "JSON.STRLEN", 

105 "JSON.TYPE", 

106 "LCS", 

107 "LINDEX", 

108 "LLEN", 

109 "LPOS", 

110 "LRANGE", 

111 "MGET", 

112 "SCARD", 

113 "SDIFF", 

114 "SINTER", 

115 "SINTERCARD", 

116 "SISMEMBER", 

117 "SMEMBERS", 

118 "SMISMEMBER", 

119 "SORT_RO", 

120 "STRLEN", 

121 "SUBSTR", 

122 "SUNION", 

123 "TS.GET", 

124 "TS.INFO", 

125 "TS.RANGE", 

126 "TS.REVRANGE", 

127 "TYPE", 

128 "XLEN", 

129 "XPENDING", 

130 "XRANGE", 

131 "XREAD", 

132 "XREVRANGE", 

133 "ZCARD", 

134 "ZCOUNT", 

135 "ZDIFF", 

136 "ZINTER", 

137 "ZINTERCARD", 

138 "ZLEXCOUNT", 

139 "ZMSCORE", 

140 "ZRANGE", 

141 "ZRANGEBYLEX", 

142 "ZRANGEBYSCORE", 

143 "ZRANK", 

144 "ZREVRANGE", 

145 "ZREVRANGEBYLEX", 

146 "ZREVRANGEBYSCORE", 

147 "ZREVRANK", 

148 "ZSCORE", 

149 "ZUNION", 

150] 

151 

152_RESPONSE = "response" 

153_KEYS = "keys" 

154_CTIME = "ctime" 

155_ACCESS_COUNT = "access_count" 

156 

157 

158class EvictionPolicy(Enum): 

159 LRU = "lru" 

160 LFU = "lfu" 

161 RANDOM = "random" 

162 

163 

164class AbstractCache(ABC): 

165 """ 

166 An abstract base class for client caching implementations. 

167 If you want to implement your own cache you must support these methods. 

168 """ 

169 

170 @abstractmethod 

171 def set(self, command: str, response: ResponseT, keys_in_command: List[KeyT]): 

172 pass 

173 

174 @abstractmethod 

175 def get(self, command: str) -> ResponseT: 

176 pass 

177 

178 @abstractmethod 

179 def delete_command(self, command: str): 

180 pass 

181 

182 @abstractmethod 

183 def delete_many(self, commands): 

184 pass 

185 

186 @abstractmethod 

187 def flush(self): 

188 pass 

189 

190 @abstractmethod 

191 def invalidate_key(self, key: KeyT): 

192 pass 

193 

194 

195class _LocalCache(AbstractCache): 

196 """ 

197 A caching mechanism for storing redis commands and their responses. 

198 

199 Args: 

200 max_size (int): The maximum number of commands to be stored in the cache. 

201 ttl (int): The time-to-live for each command in seconds. 

202 eviction_policy (EvictionPolicy): The eviction policy to use for removing commands when the cache is full. 

203 

204 Attributes: 

205 max_size (int): The maximum number of commands to be stored in the cache. 

206 ttl (int): The time-to-live for each command in seconds. 

207 eviction_policy (EvictionPolicy): The eviction policy used for cache management. 

208 cache (OrderedDict): The ordered dictionary to store commands and their metadata. 

209 key_commands_map (defaultdict): A mapping of keys to the set of commands that use each key. 

210 commands_ttl_list (list): A list to keep track of the commands in the order they were added. # noqa 

211 """ 

212 

213 def __init__( 

214 self, 

215 max_size: int = 10000, 

216 ttl: int = 0, 

217 eviction_policy: EvictionPolicy = DEFAULT_EVICTION_POLICY, 

218 **kwargs, 

219 ): 

220 self.max_size = max_size 

221 self.ttl = ttl 

222 self.eviction_policy = eviction_policy 

223 self.cache = OrderedDict() 

224 self.key_commands_map = defaultdict(set) 

225 self.commands_ttl_list = [] 

226 

227 def set(self, command: str, response: ResponseT, keys_in_command: List[KeyT]): 

228 """ 

229 Set a redis command and its response in the cache. 

230 

231 Args: 

232 command (str): The redis command. 

233 response (ResponseT): The response associated with the command. 

234 keys_in_command (List[KeyT]): The list of keys used in the command. 

235 """ 

236 if len(self.cache) >= self.max_size: 

237 self._evict() 

238 self.cache[command] = { 

239 _RESPONSE: response, 

240 _KEYS: keys_in_command, 

241 _CTIME: time.monotonic(), 

242 _ACCESS_COUNT: 0, # Used only for LFU 

243 } 

244 self._update_key_commands_map(keys_in_command, command) 

245 self.commands_ttl_list.append(command) 

246 

247 def get(self, command: str) -> ResponseT: 

248 """ 

249 Get the response for a redis command from the cache. 

250 

251 Args: 

252 command (str): The redis command. 

253 

254 Returns: 

255 ResponseT: The response associated with the command, or None if the command is not in the cache. # noqa 

256 """ 

257 if command in self.cache: 

258 if self._is_expired(command): 

259 self.delete_command(command) 

260 return 

261 self._update_access(command) 

262 return copy.deepcopy(self.cache[command]["response"]) 

263 

264 def delete_command(self, command: str): 

265 """ 

266 Delete a redis command and its metadata from the cache. 

267 

268 Args: 

269 command (str): The redis command to be deleted. 

270 """ 

271 if command in self.cache: 

272 keys_in_command = self.cache[command].get("keys") 

273 self._del_key_commands_map(keys_in_command, command) 

274 self.commands_ttl_list.remove(command) 

275 del self.cache[command] 

276 

277 def delete_many(self, commands): 

278 pass 

279 

280 def flush(self): 

281 """Clear the entire cache, removing all redis commands and metadata.""" 

282 self.cache.clear() 

283 self.key_commands_map.clear() 

284 self.commands_ttl_list = [] 

285 

286 def _is_expired(self, command: str) -> bool: 

287 """ 

288 Check if a redis command has expired based on its time-to-live. 

289 

290 Args: 

291 command (str): The redis command. 

292 

293 Returns: 

294 bool: True if the command has expired, False otherwise. 

295 """ 

296 if self.ttl == 0: 

297 return False 

298 return time.monotonic() - self.cache[command]["ctime"] > self.ttl 

299 

300 def _update_access(self, command: str): 

301 """ 

302 Update the access information for a redis command based on the eviction policy. 

303 

304 Args: 

305 command (str): The redis command. 

306 """ 

307 if self.eviction_policy == EvictionPolicy.LRU.value: 

308 self.cache.move_to_end(command) 

309 elif self.eviction_policy == EvictionPolicy.LFU.value: 

310 self.cache[command]["access_count"] = ( 

311 self.cache.get(command, {}).get("access_count", 0) + 1 

312 ) 

313 self.cache.move_to_end(command) 

314 elif self.eviction_policy == EvictionPolicy.RANDOM.value: 

315 pass # Random eviction doesn't require updates 

316 

317 def _evict(self): 

318 """Evict a redis command from the cache based on the eviction policy.""" 

319 if self._is_expired(self.commands_ttl_list[0]): 

320 self.delete_command(self.commands_ttl_list[0]) 

321 elif self.eviction_policy == EvictionPolicy.LRU.value: 

322 self.cache.popitem(last=False) 

323 elif self.eviction_policy == EvictionPolicy.LFU.value: 

324 min_access_command = min( 

325 self.cache, key=lambda k: self.cache[k].get("access_count", 0) 

326 ) 

327 self.cache.pop(min_access_command) 

328 elif self.eviction_policy == EvictionPolicy.RANDOM.value: 

329 random_command = random.choice(list(self.cache.keys())) 

330 self.cache.pop(random_command) 

331 

332 def _update_key_commands_map(self, keys: List[KeyT], command: str): 

333 """ 

334 Update the key_commands_map with command that uses the keys. 

335 

336 Args: 

337 keys (List[KeyT]): The list of keys used in the command. 

338 command (str): The redis command. 

339 """ 

340 for key in keys: 

341 self.key_commands_map[key].add(command) 

342 

343 def _del_key_commands_map(self, keys: List[KeyT], command: str): 

344 """ 

345 Remove a redis command from the key_commands_map. 

346 

347 Args: 

348 keys (List[KeyT]): The list of keys used in the redis command. 

349 command (str): The redis command. 

350 """ 

351 for key in keys: 

352 self.key_commands_map[key].remove(command) 

353 

354 def invalidate_key(self, key: KeyT): 

355 """ 

356 Invalidate (delete) all redis commands associated with a specific key. 

357 

358 Args: 

359 key (KeyT): The key to be invalidated. 

360 """ 

361 if key not in self.key_commands_map: 

362 return 

363 commands = list(self.key_commands_map[key]) 

364 for command in commands: 

365 self.delete_command(command)