Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/platform/gfile.py: 92%

1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================== 

15 

16"""Import router for file_io.""" 

17# pylint: disable=unused-import 

18from tensorflow.python.lib.io.file_io import copy as Copy 

19from tensorflow.python.lib.io.file_io import create_dir as MkDir 

20from tensorflow.python.lib.io.file_io import delete_file as Remove 

21from tensorflow.python.lib.io.file_io import delete_recursively as DeleteRecursively 

22from tensorflow.python.lib.io.file_io import file_exists as Exists 

23from tensorflow.python.lib.io.file_io import FileIO as _FileIO 

24from tensorflow.python.lib.io.file_io import get_matching_files as Glob 

25from tensorflow.python.lib.io.file_io import is_directory as IsDirectory 

26from tensorflow.python.lib.io.file_io import list_directory as ListDirectory 

27from tensorflow.python.lib.io.file_io import recursive_create_dir as MakeDirs 

28from tensorflow.python.lib.io.file_io import rename as Rename 

29from tensorflow.python.lib.io.file_io import stat as Stat 

30from tensorflow.python.lib.io.file_io import walk as Walk 

31# pylint: enable=unused-import 

32from tensorflow.python.util.deprecation import deprecated 

33from tensorflow.python.util.tf_export import tf_export 

34 

35 

36@tf_export('io.gfile.GFile', v1=['gfile.GFile', 'gfile.Open', 'io.gfile.GFile']) 

37class GFile(_FileIO): 

38 r"""File I/O wrappers without thread locking. 

39 

40 The main roles of the `tf.io.gfile` module are: 

41 

42 1. To provide an API that is close to Python's file I/O objects, and 

43 2. To provide an implementation based on TensorFlow's C++ FileSystem API. 

44 

45 The C++ FileSystem API supports multiple file system implementations, 

46 including local files, Google Cloud Storage (using a `gs://` prefix, and 

47 HDFS (using an `hdfs://` prefix). TensorFlow exports these as `tf.io.gfile`, 

48 so that you can use these implementations for saving and loading checkpoints, 

49 writing to TensorBoard logs, and accessing training data (among other uses). 

50 However, if all your files are local, you can use the regular Python file 

51 API without any problem. 

52 

53 *Note*: though similar to Python's I/O implementation, there are semantic 

54 differences to make `tf.io.gfile` more efficient for backing filesystems. For 

55 example, a write mode file will not be opened until the first write call to 

56 minimize RPC invocations in network filesystems. 

57 

58 Once you obtain a `GFile` object, you can use it in most ways as you would any 

59 Python's file object: 

60 

61 >>> with open("/tmp/x", "w") as f: 

62 ... f.write("asdf") 

63 4 

64 >>> with tf.io.gfile.GFile("/tmp/x") as f: 

65 ... f.read() 

66 'asdf' 

67 

68 The difference is that you can specify URI schemes to use other filesystems 

69 (e.g., `gs://` for GCS, `s3://` for S3, etc.), if they are supported. Using 

70 `file://` as an example, we have: 

71 

72 >>> with tf.io.gfile.GFile("file:///tmp/x", "w") as f: 

73 ... f.write("qwert") 

74 ... f.write("asdf") 

75 >>> tf.io.gfile.GFile("file:///tmp/x").read() 

76 'qwertasdf' 

77 

78 You can also read all lines of a file directly: 

79 

80 >>> with tf.io.gfile.GFile("file:///tmp/x", "w") as f: 

81 ... f.write("asdf\n") 

82 ... f.write("qwer\n") 

83 >>> tf.io.gfile.GFile("/tmp/x").readlines() 

84 ['asdf\n', 'qwer\n'] 

85 

86 You can iterate over the lines: 

87 

88 >>> with tf.io.gfile.GFile("file:///tmp/x", "w") as f: 

89 ... f.write("asdf\n") 

90 ... f.write("qwer\n") 

91 >>> for line in tf.io.gfile.GFile("/tmp/x"): 

92 ... print(line[:-1]) # removes the end of line character 

93 asdf 

94 qwer 

95 

96 Random access read is possible if the underlying filesystem supports it: 

97 

98 >>> with open("/tmp/x", "w") as f: 

99 ... f.write("asdfqwer") 

100 >>> f = tf.io.gfile.GFile("/tmp/x") 

101 >>> f.read(3) 

102 'asd' 

103 >>> f.seek(4) 

104 >>> f.tell() 

105 4 

106 >>> f.read(3) 

107 'qwe' 

108 >>> f.tell() 

109 7 

110 >>> f.close() 

111 """ 

112 

113 def __init__(self, name, mode='r'): 

114 super(GFile, self).__init__(name=name, mode=mode) 

115 

116 

117@tf_export(v1=['gfile.FastGFile']) 

118class FastGFile(_FileIO): 

119 """File I/O wrappers without thread locking. 

120 

121 Note, that this is somewhat like builtin Python file I/O, but 

122 there are semantic differences to make it more efficient for 

123 some backing filesystems. For example, a write mode file will 

124 not be opened until the first write call (to minimize RPC 

125 invocations in network filesystems). 

126 """ 

127 

128 @deprecated(None, 'Use tf.gfile.GFile.') 

129 def __init__(self, name, mode='r'): 

130 super(FastGFile, self).__init__(name=name, mode=mode) 

131 

132 

133# Does not alias to Open so that we use our version of GFile to strip 

134# 'b' mode. 

135Open = GFile