Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/smart_open/__init__.py: 68%

1# -*- coding: utf-8 -*- 

2# 

3# Copyright (C) 2019 Radim Rehurek <me@radimrehurek.com> 

4# 

5# This code is distributed under the terms and conditions 

6# from the MIT License (MIT). 

7# 

8 

9""" 

10Utilities for streaming to/from several file-like data storages: S3 / HDFS / local 

11filesystem / compressed files, and many more, using a simple, Pythonic API. 

12 

13The streaming makes heavy use of generators and pipes, to avoid loading 

14full file contents into memory, allowing work with arbitrarily large files. 

15 

16The main functions are: 

17 

18* `open()`, which opens the given file for reading/writing 

19* `parse_uri()` 

20* `s3_iter_bucket()`, which goes over all keys in an S3 bucket in parallel 

21* `register_compressor()`, which registers callbacks for transparent compressor handling 

22 

23""" 

24 

25import contextlib 

26import logging 

27from importlib.metadata import PackageNotFoundError, version 

28 

29with contextlib.suppress(PackageNotFoundError): 

30 __version__ = version("smart_open") 

31# 

32# Prevent regression of #474 and #475 

33# 

34logger = logging.getLogger(__name__) 

35logger.addHandler(logging.NullHandler()) 

36 

37from .smart_open_lib import open, parse_uri, smart_open, register_compressor # noqa: E402 

38 

39_WARNING = """smart_open.s3_iter_bucket is deprecated and will stop functioning 

40in a future version. Please import iter_bucket from the smart_open.s3 module instead: 

41 

42 from smart_open.s3 import iter_bucket as s3_iter_bucket 

43 

44""" 

45_WARNED = False 

46 

47 

48def s3_iter_bucket( 

49 bucket_name, 

50 prefix='', 

51 accept_key=None, 

52 key_limit=None, 

53 workers=16, 

54 retries=3, 

55 **session_kwargs 

56): 

57 """Deprecated. Use smart_open.s3.iter_bucket instead.""" 

58 global _WARNED 

59 from .s3 import iter_bucket 

60 if not _WARNED: 

61 logger.warning(_WARNING) 

62 _WARNED = True 

63 return iter_bucket( 

64 bucket_name=bucket_name, 

65 prefix=prefix, 

66 accept_key=accept_key, 

67 key_limit=key_limit, 

68 workers=workers, 

69 retries=retries, 

70 session_kwargs=session_kwargs 

71 ) 

72 

73 

74__all__ = [ 

75 'open', 

76 'parse_uri', 

77 'register_compressor', 

78 's3_iter_bucket', 

79 'smart_open', 

80]