Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/gitdb/db/loose.py: 61%

1# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors 

2# 

3# This module is part of GitDB and is released under 

4# the New BSD License: https://opensource.org/license/bsd-3-clause/ 

5from gitdb.db.base import ( 

6 FileDBBase, 

7 ObjectDBR, 

8 ObjectDBW 

9) 

10 

11from gitdb.exc import ( 

12 BadObject, 

13 AmbiguousObjectName 

14) 

15 

16from gitdb.stream import ( 

17 DecompressMemMapReader, 

18 FDCompressedSha1Writer, 

19 FDStream, 

20 Sha1Writer 

21) 

22 

23from gitdb.base import ( 

24 OStream, 

25 OInfo 

26) 

27 

28from gitdb.util import ( 

29 file_contents_ro_filepath, 

30 ENOENT, 

31 hex_to_bin, 

32 bin_to_hex, 

33 exists, 

34 chmod, 

35 isfile, 

36 remove, 

37 rename, 

38 dirname, 

39 basename, 

40 join 

41) 

42 

43from gitdb.fun import ( 

44 chunk_size, 

45 loose_object_header_info, 

46 write_object, 

47 stream_copy 

48) 

49 

50from gitdb.utils.encoding import force_bytes 

51 

52import tempfile 

53import os 

54import sys 

55 

56 

57__all__ = ('LooseObjectDB', ) 

58 

59 

60class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): 

61 

62 """A database which operates on loose object files""" 

63 

64 # CONFIGURATION 

65 # chunks in which data will be copied between streams 

66 stream_chunk_size = chunk_size 

67 

68 # On windows we need to keep it writable, otherwise it cannot be removed 

69 # either 

70 new_objects_mode = int("444", 8) 

71 if os.name == 'nt': 

72 new_objects_mode = int("644", 8) 

73 

74 def __init__(self, root_path): 

75 super().__init__(root_path) 

76 self._hexsha_to_file = dict() 

77 # Additional Flags - might be set to 0 after the first failure 

78 # Depending on the root, this might work for some mounts, for others not, which 

79 # is why it is per instance 

80 self._fd_open_flags = getattr(os, 'O_NOATIME', 0) 

81 

82 #{ Interface 

83 def object_path(self, hexsha): 

84 """ 

85 :return: path at which the object with the given hexsha would be stored, 

86 relative to the database root""" 

87 return join(hexsha[:2], hexsha[2:]) 

88 

89 def readable_db_object_path(self, hexsha): 

90 """ 

91 :return: readable object path to the object identified by hexsha 

92 :raise BadObject: If the object file does not exist""" 

93 try: 

94 return self._hexsha_to_file[hexsha] 

95 except KeyError: 

96 pass 

97 # END ignore cache misses 

98 

99 # try filesystem 

100 path = self.db_path(self.object_path(hexsha)) 

101 if exists(path): 

102 self._hexsha_to_file[hexsha] = path 

103 return path 

104 # END handle cache 

105 raise BadObject(hexsha) 

106 

107 def partial_to_complete_sha_hex(self, partial_hexsha): 

108 """:return: 20 byte binary sha1 string which matches the given name uniquely 

109 :param name: hexadecimal partial name (bytes or ascii string) 

110 :raise AmbiguousObjectName: 

111 :raise BadObject: """ 

112 candidate = None 

113 for binsha in self.sha_iter(): 

114 if bin_to_hex(binsha).startswith(force_bytes(partial_hexsha)): 

115 # it can't ever find the same object twice 

116 if candidate is not None: 

117 raise AmbiguousObjectName(partial_hexsha) 

118 candidate = binsha 

119 # END for each object 

120 if candidate is None: 

121 raise BadObject(partial_hexsha) 

122 return candidate 

123 

124 #} END interface 

125 

126 def _map_loose_object(self, sha): 

127 """ 

128 :return: memory map of that file to allow random read access 

129 :raise BadObject: if object could not be located""" 

130 db_path = self.db_path(self.object_path(bin_to_hex(sha))) 

131 try: 

132 return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) 

133 except OSError as e: 

134 if e.errno != ENOENT: 

135 # try again without noatime 

136 try: 

137 return file_contents_ro_filepath(db_path) 

138 except OSError as new_e: 

139 raise BadObject(sha) from new_e 

140 # didn't work because of our flag, don't try it again 

141 self._fd_open_flags = 0 

142 else: 

143 raise BadObject(sha) from e 

144 # END handle error 

145 # END exception handling 

146 

147 def set_ostream(self, stream): 

148 """:raise TypeError: if the stream does not support the Sha1Writer interface""" 

149 if stream is not None and not isinstance(stream, Sha1Writer): 

150 raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) 

151 return super().set_ostream(stream) 

152 

153 def info(self, sha): 

154 m = self._map_loose_object(sha) 

155 try: 

156 typ, size = loose_object_header_info(m) 

157 return OInfo(sha, typ, size) 

158 finally: 

159 if hasattr(m, 'close'): 

160 m.close() 

161 # END assure release of system resources 

162 

163 def stream(self, sha): 

164 m = self._map_loose_object(sha) 

165 type, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True) 

166 return OStream(sha, type, size, stream) 

167 

168 def has_object(self, sha): 

169 try: 

170 self.readable_db_object_path(bin_to_hex(sha)) 

171 return True 

172 except BadObject: 

173 return False 

174 # END check existence 

175 

176 def store(self, istream): 

177 """note: The sha we produce will be hex by nature""" 

178 tmp_path = None 

179 writer = self.ostream() 

180 if writer is None: 

181 # open a tmp file to write the data to 

182 fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) 

183 

184 if istream.binsha is None: 

185 writer = FDCompressedSha1Writer(fd) 

186 else: 

187 writer = FDStream(fd) 

188 # END handle direct stream copies 

189 # END handle custom writer 

190 

191 try: 

192 try: 

193 if istream.binsha is not None: 

194 # copy as much as possible, the actual uncompressed item size might 

195 # be smaller than the compressed version 

196 stream_copy(istream.read, writer.write, sys.maxsize, self.stream_chunk_size) 

197 else: 

198 # write object with header, we have to make a new one 

199 write_object(istream.type, istream.size, istream.read, writer.write, 

200 chunk_size=self.stream_chunk_size) 

201 # END handle direct stream copies 

202 finally: 

203 if tmp_path: 

204 writer.close() 

205 # END assure target stream is closed 

206 except: 

207 if tmp_path: 

208 os.remove(tmp_path) 

209 raise 

210 # END assure tmpfile removal on error 

211 

212 hexsha = None 

213 if istream.binsha: 

214 hexsha = istream.hexsha 

215 else: 

216 hexsha = writer.sha(as_hex=True) 

217 # END handle sha 

218 

219 if tmp_path: 

220 obj_path = self.db_path(self.object_path(hexsha)) 

221 obj_dir = dirname(obj_path) 

222 os.makedirs(obj_dir, exist_ok=True) 

223 # END handle destination directory 

224 # rename onto existing doesn't work on NTFS 

225 if isfile(obj_path): 

226 remove(tmp_path) 

227 else: 

228 rename(tmp_path, obj_path) 

229 # end rename only if needed 

230 

231 # make sure its readable for all ! It started out as rw-- tmp file 

232 # but needs to be rwrr 

233 chmod(obj_path, self.new_objects_mode) 

234 # END handle dry_run 

235 

236 istream.binsha = hex_to_bin(hexsha) 

237 return istream 

238 

239 def sha_iter(self): 

240 # find all files which look like an object, extract sha from there 

241 for root, dirs, files in os.walk(self.root_path()): 

242 root_base = basename(root) 

243 if len(root_base) != 2: 

244 continue 

245 

246 for f in files: 

247 if len(f) != 38: 

248 continue 

249 yield hex_to_bin(root_base + f) 

250 # END for each file 

251 # END for each walk iteration 

252 

253 def size(self): 

254 return len(tuple(self.sha_iter()))