Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/joblib/disk.py: 27%

1"""

2Disk management utilities.

3"""

5# Authors: Gael Varoquaux <gael dot varoquaux at normalesup dot org>

6# Lars Buitinck

8# License: BSD Style, 3 clauses.

11import os

12import sys

13import time

14import errno

15import shutil

17from multiprocessing import util

20try:

21 WindowsError

22except NameError:

23 WindowsError = OSError

26def disk_used(path):

27 """ Return the disk usage in a directory."""

28 size = 0

29 for file in os.listdir(path) + ['.']:

30 stat = os.stat(os.path.join(path, file))

31 if hasattr(stat, 'st_blocks'):

32 size += stat.st_blocks * 512

33 else:

34 # on some platform st_blocks is not available (e.g., Windows)

35 # approximate by rounding to next multiple of 512

36 size += (stat.st_size // 512 + 1) * 512

37 # We need to convert to int to avoid having longs on some systems (we

38 # don't want longs to avoid problems we SQLite)

39 return int(size / 1024.)

42def memstr_to_bytes(text):

43 """ Convert a memory text to its value in bytes.

44 """

45 kilo = 1024

46 units = dict(K=kilo, M=kilo ** 2, G=kilo ** 3)

47 try:

48 size = int(units[text[-1]] * float(text[:-1]))

49 except (KeyError, ValueError) as e:

50 raise ValueError(

51 "Invalid literal for size give: %s (type %s) should be "

52 "alike '10G', '500M', '50K'." % (text, type(text))) from e

53 return size

56def mkdirp(d):

57 """Ensure directory d exists (like mkdir -p on Unix)

58 No guarantee that the directory is writable.

59 """

60 try:

61 os.makedirs(d)

62 except OSError as e:

63 if e.errno != errno.EEXIST:

64 raise

67# if a rmtree operation fails in rm_subdirs, wait for this much time (in secs),

68# then retry up to RM_SUBDIRS_N_RETRY times. If it still fails, raise the

69# exception. this mechanism ensures that the sub-process gc have the time to

70# collect and close the memmaps before we fail.

71RM_SUBDIRS_RETRY_TIME = 0.1

72RM_SUBDIRS_N_RETRY = 10

75def rm_subdirs(path, onerror=None):

76 """Remove all subdirectories in this path.

78 The directory indicated by `path` is left in place, and its subdirectories

79 are erased.

81 If onerror is set, it is called to handle the error with arguments (func,

82 path, exc_info) where func is os.listdir, os.remove, or os.rmdir;

83 path is the argument to that function that caused it to fail; and

84 exc_info is a tuple returned by sys.exc_info(). If onerror is None,

85 an exception is raised.

86 """

88 # NOTE this code is adapted from the one in shutil.rmtree, and is

89 # just as fast

91 names = []

92 try:

93 names = os.listdir(path)

94 except os.error:

95 if onerror is not None:

96 onerror(os.listdir, path, sys.exc_info())

97 else:

98 raise

100 for name in names:

101 fullname = os.path.join(path, name)

102 delete_folder(fullname, onerror=onerror)

103

104

105def delete_folder(folder_path, onerror=None, allow_non_empty=True):

106 """Utility function to cleanup a temporary folder if it still exists."""

107 if os.path.isdir(folder_path):

108 if onerror is not None:

109 shutil.rmtree(folder_path, False, onerror)

110 else:

111 # allow the rmtree to fail once, wait and re-try.

112 # if the error is raised again, fail

113 err_count = 0

114 while True:

115 files = os.listdir(folder_path)

116 try:

117 if len(files) == 0 or allow_non_empty:

118 shutil.rmtree(

119 folder_path, ignore_errors=False, onerror=None

120 )

121 util.debug(

122 "Successfully deleted {}".format(folder_path))

123 break

124 else:

125 raise OSError(

126 "Expected empty folder {} but got {} "

127 "files.".format(folder_path, len(files))

128 )

129 except (OSError, WindowsError):

130 err_count += 1

131 if err_count > RM_SUBDIRS_N_RETRY:

132 # the folder cannot be deleted right now. It maybe

133 # because some temporary files have not been deleted

134 # yet.

135 raise

136 time.sleep(RM_SUBDIRS_RETRY_TIME)