Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/smart_open/__init__.py: 69%

1# -*- coding: utf-8 -*- 

2# 

3# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com> 

4# 

5# This code is distributed under the terms and conditions 

6# from the MIT License (MIT). 

7# 

8 

9""" 

10Utilities for streaming to/from several file-like data storages: S3 / HDFS / local 

11filesystem / compressed files, and many more, using a simple, Pythonic API. 

12 

13The streaming makes heavy use of generators and pipes, to avoid loading 

14full file contents into memory, allowing work with arbitrarily large files. 

15 

16The main functions are: 

17 

18* `open()`, which opens the given file for reading/writing 

19* `parse_uri()` 

20* `s3_iter_bucket()`, which goes over all keys in an S3 bucket in parallel 

21* `register_compressor()`, which registers callbacks for transparent compressor handling 

22 

23""" 

24 

25import logging 

26 

27# 

28# Prevent regression of #474 and #475 

29# 

30logger = logging.getLogger(__name__) 

31logger.addHandler(logging.NullHandler()) 

32 

33from smart_open import version # noqa: E402 

34from .smart_open_lib import open, parse_uri, smart_open, register_compressor # noqa: E402 

35 

36_WARNING = """smart_open.s3_iter_bucket is deprecated and will stop functioning 

37in a future version. Please import iter_bucket from the smart_open.s3 module instead: 

38 

39 from smart_open.s3 import iter_bucket as s3_iter_bucket 

40 

41""" 

42_WARNED = False 

43 

44 

45def s3_iter_bucket( 

46 bucket_name, 

47 prefix='', 

48 accept_key=None, 

49 key_limit=None, 

50 workers=16, 

51 retries=3, 

52 **session_kwargs 

53): 

54 """Deprecated. Use smart_open.s3.iter_bucket instead.""" 

55 global _WARNED 

56 from .s3 import iter_bucket 

57 if not _WARNED: 

58 logger.warning(_WARNING) 

59 _WARNED = True 

60 return iter_bucket( 

61 bucket_name=bucket_name, 

62 prefix=prefix, 

63 accept_key=accept_key, 

64 key_limit=key_limit, 

65 workers=workers, 

66 retries=retries, 

67 session_kwargs=session_kwargs 

68 ) 

69 

70 

71__all__ = [ 

72 'open', 

73 'parse_uri', 

74 'register_compressor', 

75 's3_iter_bucket', 

76 'smart_open', 

77] 

78 

79__version__ = version.__version__