Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/scrapy/http/headers.py: 35%

1from __future__ import annotations 

2 

3from collections.abc import Mapping 

4from typing import ( 

5 TYPE_CHECKING, 

6 Any, 

7 AnyStr, 

8 Dict, 

9 Iterable, 

10 List, 

11 Optional, 

12 Tuple, 

13 Union, 

14 cast, 

15) 

16 

17from w3lib.http import headers_dict_to_raw 

18 

19from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict 

20from scrapy.utils.python import to_unicode 

21 

22if TYPE_CHECKING: 

23 # typing.Self requires Python 3.11 

24 from typing_extensions import Self 

25 

26 

27_RawValueT = Union[bytes, str, int] 

28 

29 

30# isn't fully compatible typing-wise with either dict or CaselessDict, 

31# but it needs refactoring anyway, see also https://github.com/scrapy/scrapy/pull/5146 

32class Headers(CaselessDict): 

33 """Case insensitive http headers dictionary""" 

34 

35 def __init__( 

36 self, 

37 seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, 

38 encoding: str = "utf-8", 

39 ): 

40 self.encoding: str = encoding 

41 super().__init__(seq) 

42 

43 def update( # type: ignore[override] 

44 self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]] 

45 ) -> None: 

46 seq = seq.items() if isinstance(seq, Mapping) else seq 

47 iseq: Dict[bytes, List[bytes]] = {} 

48 for k, v in seq: 

49 iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) 

50 super().update(iseq) 

51 

52 def normkey(self, key: AnyStr) -> bytes: # type: ignore[override] 

53 """Normalize key to bytes""" 

54 return self._tobytes(key.title()) 

55 

56 def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]: 

57 """Normalize values to bytes""" 

58 _value: Iterable[_RawValueT] 

59 if value is None: 

60 _value = [] 

61 elif isinstance(value, (str, bytes)): 

62 _value = [value] 

63 elif hasattr(value, "__iter__"): 

64 _value = value 

65 else: 

66 _value = [value] 

67 

68 return [self._tobytes(x) for x in _value] 

69 

70 def _tobytes(self, x: _RawValueT) -> bytes: 

71 if isinstance(x, bytes): 

72 return x 

73 if isinstance(x, str): 

74 return x.encode(self.encoding) 

75 if isinstance(x, int): 

76 return str(x).encode(self.encoding) 

77 raise TypeError(f"Unsupported value type: {type(x)}") 

78 

79 def __getitem__(self, key: AnyStr) -> Optional[bytes]: 

80 try: 

81 return cast(List[bytes], super().__getitem__(key))[-1] 

82 except IndexError: 

83 return None 

84 

85 def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: 

86 try: 

87 return cast(List[bytes], super().get(key, def_val))[-1] 

88 except IndexError: 

89 return None 

90 

91 def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]: 

92 try: 

93 return cast(List[bytes], super().__getitem__(key)) 

94 except KeyError: 

95 if def_val is not None: 

96 return self.normvalue(def_val) 

97 return [] 

98 

99 def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None: 

100 self[key] = list_ 

101 

102 def setlistdefault( 

103 self, key: AnyStr, default_list: Iterable[_RawValueT] = () 

104 ) -> Any: 

105 return self.setdefault(key, default_list) 

106 

107 def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None: 

108 lst = self.getlist(key) 

109 lst.extend(self.normvalue(value)) 

110 self[key] = lst 

111 

112 def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override] 

113 return ((k, self.getlist(k)) for k in self.keys()) 

114 

115 def values(self) -> List[Optional[bytes]]: # type: ignore[override] 

116 return [self[k] for k in self.keys()] 

117 

118 def to_string(self) -> bytes: 

119 # cast() can be removed if the headers_dict_to_raw() hint is improved 

120 return cast(bytes, headers_dict_to_raw(self)) 

121 

122 def to_unicode_dict(self) -> CaseInsensitiveDict: 

123 """Return headers as a CaseInsensitiveDict with str keys 

124 and str values. Multiple values are joined with ','. 

125 """ 

126 return CaseInsensitiveDict( 

127 ( 

128 to_unicode(key, encoding=self.encoding), 

129 to_unicode(b",".join(value), encoding=self.encoding), 

130 ) 

131 for key, value in self.items() 

132 ) 

133 

134 def __copy__(self) -> Self: 

135 return self.__class__(self) 

136 

137 copy = __copy__