Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/fsspec/dircache.py: 30%

1import time 

2from collections import OrderedDict 

3from collections.abc import MutableMapping 

4 

5 

6class DirCache(MutableMapping): 

7 """ 

8 Caching of directory listings, in a structure like:: 

9 

10 {"path0": [ 

11 {"name": "path0/file0", 

12 "size": 123, 

13 "type": "file", 

14 ... 

15 }, 

16 {"name": "path0/file1", 

17 }, 

18 ... 

19 ], 

20 "path1": [...] 

21 } 

22 

23 Parameters to this class control listing expiry or indeed turn 

24 caching off. 

25 """ 

26 

27 def __init__( 

28 self, 

29 use_listings_cache=True, 

30 listings_expiry_time=None, 

31 max_paths=None, 

32 **kwargs, 

33 ): 

34 """ 

35 Parameters 

36 ---------- 

37 use_listings_cache: bool 

38 If False, this cache never returns items, but always reports KeyError, 

39 and setting items has no effect. 

40 listings_expiry_time: int or float (optional) 

41 Time in seconds that a listing is considered valid. If None, 

42 listings do not expire. Time is measured from when the entry was set. 

43 max_paths: int (optional) 

44 The maximum number of directory listings to retain in the cache. 

45 When the cache exceeds this limit, the least recently used 

46 (set or accessed) listings are evicted. If None, 0, or negative, 

47 there is no limit. 

48 """ 

49 # max_paths is normalized to be either positive or None. 

50 self.max_paths = max_paths if max_paths and max_paths > 0 else None 

51 self._cache = OrderedDict() if self.max_paths else {} 

52 self._times = {} 

53 self.use_listings_cache = use_listings_cache 

54 self.listings_expiry_time = listings_expiry_time 

55 

56 def __getitem__(self, item): 

57 if not self.use_listings_cache: 

58 raise KeyError(item) 

59 

60 if self.listings_expiry_time is not None and item in self._cache: 

61 if time.time() - self._times.get(item, 0) > self.listings_expiry_time: 

62 del self[item] 

63 raise KeyError(item) 

64 

65 val = self._cache[item] 

66 if self.max_paths: 

67 self._cache.move_to_end(item) 

68 return val 

69 

70 def clear(self): 

71 self._cache.clear() 

72 self._times.clear() 

73 

74 def __len__(self): 

75 return len(self._cache) 

76 

77 def __contains__(self, item): 

78 if not self.use_listings_cache or item not in self._cache: 

79 return False 

80 

81 if self.listings_expiry_time is not None: 

82 if time.time() - self._times.get(item, 0) > self.listings_expiry_time: 

83 del self[item] 

84 return False 

85 

86 return True 

87 

88 def __setitem__(self, key, value): 

89 if not self.use_listings_cache: 

90 return 

91 

92 self._cache[key] = value 

93 if self.listings_expiry_time is not None: 

94 self._times[key] = time.time() 

95 

96 if self.max_paths: 

97 self._cache.move_to_end(key) 

98 if len(self._cache) > self.max_paths: 

99 oldest, _ = self._cache.popitem(last=False) 

100 self._times.pop(oldest, None) 

101 

102 def __delitem__(self, key): 

103 del self._cache[key] 

104 self._times.pop(key, None) 

105 

106 def __iter__(self): 

107 if not self.use_listings_cache: 

108 return 

109 

110 if self.listings_expiry_time is None: 

111 yield from self._cache 

112 return 

113 

114 now = time.time() 

115 for key in list(self._cache): 

116 if now - self._times.get(key, 0) > self.listings_expiry_time: 

117 del self[key] 

118 else: 

119 yield key 

120 

121 def __reduce__(self): 

122 return ( 

123 DirCache, 

124 (self.use_listings_cache, self.listings_expiry_time, self.max_paths), 

125 )