Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dateparser/date.py: 79%

1import collections

2from collections.abc import Set

3from datetime import datetime, timedelta

5import regex as re

6from dateutil.relativedelta import relativedelta

7from tzlocal import get_localzone

9from dateparser.conf import apply_settings, check_settings

10from dateparser.custom_language_detection.language_mapping import map_languages

11from dateparser.date_parser import date_parser

12from dateparser.freshness_date_parser import freshness_date_parser

13from dateparser.languages.loader import LocaleDataLoader

14from dateparser.parser import _parse_absolute, _parse_nospaces

15from dateparser.timezone_parser import pop_tz_offset_from_string

16from dateparser.utils import (

17 apply_timezone_from_settings,

18 get_timezone_from_tz_string,

19 set_correct_day_from_settings,

20 set_correct_month_from_settings,

21)

23APOSTROPHE_LOOK_ALIKE_CHARS = [

24 "\N{RIGHT SINGLE QUOTATION MARK}", # '\u2019'

25 "\N{MODIFIER LETTER APOSTROPHE}", # '\u02bc'

26 "\N{MODIFIER LETTER TURNED COMMA}", # '\u02bb'

27 "\N{ARMENIAN APOSTROPHE}", # '\u055a'

28 "\N{LATIN SMALL LETTER SALTILLO}", # '\ua78c'

29 "\N{PRIME}", # '\u2032'

30 "\N{REVERSED PRIME}", # '\u2035'

31 "\N{MODIFIER LETTER PRIME}", # '\u02b9'

32 "\N{FULLWIDTH APOSTROPHE}", # '\uff07'

33]

35RE_NBSP = re.compile("\xa0", flags=re.UNICODE)

36RE_SPACES = re.compile(r"\s+")

37RE_TRIM_SPACES = re.compile(r"^\s+(\S.*?)\s+$")

38RE_TRIM_COLONS = re.compile(r"(\S.*?):*$")

40RE_SANITIZE_SKIP = re.compile(

41 r"\t|\n|\r|\u00bb|,\s\u0432\b|\u200e|\xb7|\u200f|\u064e|\u064f", flags=re.M

42)

43RE_SANITIZE_RUSSIAN = re.compile(r"([\W\d])\u0433\.", flags=re.I | re.U)

44RE_SANITIZE_CROATIAN = re.compile(

45 r"(\d+)\.\s?(\d+)\.\s?(\d+)\.( u)?", flags=re.I | re.U

46)

47RE_SANITIZE_PERIOD = re.compile(r"(?<=[^0-9\s])\.", flags=re.U)

48RE_SANITIZE_ON = re.compile(r"^.*?on:\s+(.*)")

49RE_SANITIZE_APOSTROPHE = re.compile("|".join(APOSTROPHE_LOOK_ALIKE_CHARS))

51RE_SEARCH_TIMESTAMP = re.compile(r"^(\d{10})(\d{3})?(\d{3})?(?![^.])")

52RE_SEARCH_NEGATIVE_TIMESTAMP = re.compile(r"^([-]\d{10})(\d{3})?(\d{3})?(?![^.])")

55def sanitize_spaces(date_string):

56 date_string = RE_NBSP.sub(" ", date_string)

57 date_string = RE_SPACES.sub(" ", date_string)

58 date_string = RE_TRIM_SPACES.sub(r"\1", date_string)

59 return date_string

62def date_range(begin, end, **kwargs):

63 dateutil_error_prone_args = [

64 "year",

65 "month",

66 "week",

67 "day",

68 "hour",

69 "minute",

70 "second",

71 ]

72 for arg in dateutil_error_prone_args:

73 if arg in kwargs:

74 raise ValueError("Invalid argument: %s" % arg)

76 step = relativedelta(**kwargs) if kwargs else relativedelta(days=1)

78 date = begin

79 while date < end:

80 yield date

81 date += step

83 # handles edge-case when iterating months and last interval is < 30 days

84 if kwargs.get("months", 0) > 0 and (date.year, date.month) == (end.year, end.month):

85 yield end

88def get_intersecting_periods(low, high, period="day"):

89 if period not in [

90 "year",

91 "month",

92 "week",

93 "day",

94 "hour",

95 "minute",

96 "second",

97 "microsecond",

98 ]:

99 raise ValueError("Invalid period: {}".format(period))

100

101 if high <= low:

102 return

103

104 step = relativedelta(**{period + "s": 1})

105

106 current_period_start = low

107 if isinstance(current_period_start, datetime):

108 reset_arguments = {}

109 for test_period in ["microsecond", "second", "minute", "hour"]:

110 if test_period == period:

111 break

112 else:

113 reset_arguments[test_period] = 0

114 current_period_start = current_period_start.replace(**reset_arguments)

115

116 if period == "week":

117 current_period_start = current_period_start - timedelta(

118 days=current_period_start.weekday()

119 )

120 elif period == "month":

121 current_period_start = current_period_start.replace(day=1)

122 elif period == "year":

123 current_period_start = current_period_start.replace(month=1, day=1)

124

125 while current_period_start < high:

126 yield current_period_start

127 current_period_start += step

128

129

130def sanitize_date(date_string):

131 date_string = RE_SANITIZE_SKIP.sub(" ", date_string)

132 date_string = RE_SANITIZE_RUSSIAN.sub(

133 r"\1 ", date_string

134 ) # remove 'г.' (Russian for year) but not in words

135 date_string = RE_SANITIZE_CROATIAN.sub(

136 r"\1.\2.\3 ", date_string

137 ) # extra '.' and 'u' interferes with parsing relative fractional dates

138 date_string = sanitize_spaces(date_string)

139 date_string = RE_SANITIZE_PERIOD.sub("", date_string)

140 date_string = RE_SANITIZE_ON.sub(r"\1", date_string)

141 date_string = RE_TRIM_COLONS.sub(r"\1", date_string)

142 date_string = RE_SANITIZE_APOSTROPHE.sub("'", date_string)

143 date_string = date_string.strip()

144 return date_string

145

146

147def get_date_from_timestamp(date_string, settings, negative=False):

148 if negative:

149 match = RE_SEARCH_NEGATIVE_TIMESTAMP.search(date_string)

150 else:

151 match = RE_SEARCH_TIMESTAMP.search(date_string)

152

153 if match:

154 if (

155 settings is None

156 or settings.TIMEZONE is None

157 or "local" in settings.TIMEZONE.lower()

158 ):

159 # If the timezone in settings is unset, or it's 'local', use the

160 # local timezone

161 timezone = get_localzone()

162 else:

163 # Otherwise, use the timezone given in settings

164 timezone = get_timezone_from_tz_string(settings.TIMEZONE)

165

166 seconds = int(match.group(1))

167 millis = int(match.group(2) or 0)

168 micros = int(match.group(3) or 0)

169 date_obj = datetime.fromtimestamp(seconds, timezone).replace(

170 microsecond=millis * 1000 + micros, tzinfo=None

171 )

172 date_obj = apply_timezone_from_settings(date_obj, settings)

173 return date_obj

174

175

176def parse_with_formats(date_string, date_formats, settings):

177 """Parse with formats and return a dictionary with 'period' and 'obj_date'.

178

179 :returns: :class:`datetime.datetime`, dict or None

180

181 """

182 period = "day"

183 for date_format in date_formats:

184 try:

185 date_obj = datetime.strptime(date_string, date_format)

186 except ValueError:

187 continue

188 else:

189 missing_month = not any(m in date_format for m in ["%m", "%b", "%B"])

190 missing_day = "%d" not in date_format

191 if missing_month and missing_day:

192 period = "year"

193 date_obj = set_correct_month_from_settings(date_obj, settings)

194 date_obj = set_correct_day_from_settings(date_obj, settings)

195

196 elif missing_month:

197 period = "year"

198 date_obj = set_correct_month_from_settings(date_obj, settings)

199

200 elif missing_day:

201 period = "month"

202 date_obj = set_correct_day_from_settings(date_obj, settings)

203

204 if not ("%y" in date_format or "%Y" in date_format):

205 today = datetime.today()

206 date_obj = date_obj.replace(year=today.year)

207

208 date_obj = apply_timezone_from_settings(date_obj, settings)

209

210 return DateData(date_obj=date_obj, period=period)

211 else:

212 return DateData(date_obj=None, period=period)

213

214

215class _DateLocaleParser:

216 def __init__(self, locale, date_string, date_formats, settings=None):

217 self._settings = settings

218 if not (date_formats is None or isinstance(date_formats, (list, tuple, Set))):

219 raise TypeError("Date formats should be list, tuple or set of strings")

220

221 self.locale = locale

222 self.date_string = date_string

223 self.date_formats = date_formats

224 self._translated_date = None

225 self._translated_date_with_formatting = None

226 self._parsers = {

227 "timestamp": self._try_timestamp,

228 "negative-timestamp": self._try_negative_timestamp,

229 "relative-time": self._try_freshness_parser,

230 "custom-formats": self._try_given_formats,

231 "absolute-time": self._try_absolute_parser,

232 "no-spaces-time": self._try_nospaces_parser,

233 }

234

235 @classmethod

236 def parse(cls, locale, date_string, date_formats=None, settings=None):

237 instance = cls(locale, date_string, date_formats, settings)

238 return instance._parse()

239

240 def _parse(self):

241 for parser_name in self._settings.PARSERS:

242 date_data = self._parsers[parser_name]()

243 if self._is_valid_date_data(date_data):

244 return date_data

245 else:

246 return None

247

248 def _try_timestamp_parser(self, negative=False):

249 return DateData(

250 date_obj=get_date_from_timestamp(

251 self.date_string, self._settings, negative=negative

252 ),

253 period="time" if self._settings.RETURN_TIME_AS_PERIOD else "day",

254 )

255

256 def _try_timestamp(self):

257 return self._try_timestamp_parser()

258

259 def _try_negative_timestamp(self):

260 return self._try_timestamp_parser(negative=True)

261

262 def _try_freshness_parser(self):

263 try:

264 return freshness_date_parser.get_date_data(

265 self._get_translated_date(), self._settings

266 )

267 except (OverflowError, ValueError):

268 return None

269

270 def _try_absolute_parser(self):

271 return self._try_parser(parse_method=_parse_absolute)

272

273 def _try_nospaces_parser(self):

274 return self._try_parser(parse_method=_parse_nospaces)

275

276 def _try_parser(self, parse_method):

277 _order = self._settings.DATE_ORDER

278 try:

279 if self._settings.PREFER_LOCALE_DATE_ORDER:

280 if "DATE_ORDER" not in self._settings._mod_settings:

281 self._settings.DATE_ORDER = self.locale.info.get(

282 "date_order", _order

283 )

284 date_obj, period = date_parser.parse(

285 self._get_translated_date(),

286 parse_method=parse_method,

287 settings=self._settings,

288 )

289 self._settings.DATE_ORDER = _order

290 return DateData(

291 date_obj=date_obj,

292 period=period,

293 )

294 except ValueError:

295 self._settings.DATE_ORDER = _order

296 return None

297

298 def _try_given_formats(self):

299 if not self.date_formats:

300 return

301

302 return parse_with_formats(

303 self._get_translated_date_with_formatting(),

304 self.date_formats,

305 settings=self._settings,

306 )

307

308 def _get_translated_date(self):

309 if self._translated_date is None:

310 self._translated_date = self.locale.translate(

311 self.date_string, keep_formatting=False, settings=self._settings

312 )

313 return self._translated_date

314

315 def _get_translated_date_with_formatting(self):

316 if self._translated_date_with_formatting is None:

317 self._translated_date_with_formatting = self.locale.translate(

318 self.date_string, keep_formatting=True, settings=self._settings

319 )

320 return self._translated_date_with_formatting

321

322 def _is_valid_date_data(self, date_data):

323 if not isinstance(date_data, DateData):

324 return False

325 if not date_data["date_obj"] or not date_data["period"]:

326 return False

327 if date_data["date_obj"] and not isinstance(date_data["date_obj"], datetime):

328 return False

329 if date_data["period"] not in ("time", "day", "week", "month", "year"):

330 return False

331 return True

332

333

334class DateData:

335 """

336 Class that represents the parsed data with useful information.

337 It can be accessed with square brackets like a dict object.

338 """

339

340 def __init__(self, *, date_obj=None, period=None, locale=None):

341 self.date_obj = date_obj

342 self.period = period

343 self.locale = locale

344

345 def __getitem__(self, k):

346 if not hasattr(self, k):

347 raise KeyError(k)

348 return getattr(self, k)

349

350 def __setitem__(self, k, v):

351 if not hasattr(self, k):

352 raise KeyError(k)

353 setattr(self, k, v)

354

355 def __repr__(self):

356 properties_text = ", ".join(

357 "{}={}".format(prop, val.__repr__()) for prop, val in self.__dict__.items()

358 )

359

360 return "{}({})".format(self.__class__.__name__, properties_text)

361

362

363class DateDataParser:

364 """

365 Class which handles language detection, translation and subsequent generic parsing of

366 string representing date and/or time.

367

368 :param languages:

369 A list of language codes, e.g. ['en', 'es', 'zh-Hant'].

370 If locales are not given, languages and region are

371 used to construct locales for translation.

372 :type languages: list

373

374 :param locales:

375 A list of locale codes, e.g. ['fr-PF', 'qu-EC', 'af-NA'].

376 The parser uses only these locales to translate date string.

377 :type locales: list

378

379 :param region:

380 A region code, e.g. 'IN', '001', 'NE'.

381 If locales are not given, languages and region are

382 used to construct locales for translation.

383 :type region: str

384

385 :param try_previous_locales:

386 If True, locales previously used to translate date are tried first.

387 :type try_previous_locales: bool

388

389 :param use_given_order:

390 If True, locales are tried for translation of date string

391 in the order in which they are given.

392 :type use_given_order: bool

393

394 :param settings:

395 Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.

396 :type settings: dict

397

398 :param detect_languages_function:

399 A function for language detection that takes as input a `text` and a `confidence_threshold`,

400 and returns a list of detected language codes.

401 Note: this function is only used if ``languages`` and ``locales`` are not provided.

402 :type detect_languages_function: function

403

404 :return: A parser instance

405

406 :raises:

407 ``ValueError``: Unknown Language, ``TypeError``: Languages argument must be a list,

408 ``SettingValidationError``: A provided setting is not valid.

409 """

410

411 locale_loader = None

412

413 @apply_settings

414 def __init__(

415 self,

416 languages=None,

417 locales=None,

418 region=None,

419 try_previous_locales=False,

420 use_given_order=False,

421 settings=None,

422 detect_languages_function=None,

423 ):

424 if languages is not None and not isinstance(languages, (list, tuple, Set)):

425 raise TypeError(

426 "languages argument must be a list (%r given)" % type(languages)

427 )

428

429 if locales is not None and not isinstance(locales, (list, tuple, Set)):

430 raise TypeError(

431 "locales argument must be a list (%r given)" % type(locales)

432 )

433

434 if region is not None and not isinstance(region, str):

435 raise TypeError("region argument must be str (%r given)" % type(region))

436

437 if not isinstance(try_previous_locales, bool):

438 raise TypeError(

439 "try_previous_locales argument must be a boolean (%r given)"

440 % type(try_previous_locales)

441 )

442

443 if not isinstance(use_given_order, bool):

444 raise TypeError(

445 "use_given_order argument must be a boolean (%r given)"

446 % type(use_given_order)

447 )

448

449 if not locales and not languages and use_given_order:

450 raise ValueError(

451 "locales or languages must be given if use_given_order is True"

452 )

453

454 check_settings(settings)

455

456 self._settings = settings

457 self.try_previous_locales = try_previous_locales

458 self.use_given_order = use_given_order

459 self.languages = list(languages) if languages else None

460 self.locales = locales

461 self.region = region

462 self.detect_languages_function = detect_languages_function

463 self.previous_locales = collections.OrderedDict()

464

465 def get_date_data(self, date_string, date_formats=None):

466 """

467 Parse string representing date and/or time in recognizable localized formats.

468 Supports parsing multiple languages and timezones.

469

470 :param date_string:

471 A string representing date and/or time in a recognizably valid format.

472 :type date_string: str

473 :param date_formats:

474 A list of format strings using directives as given

475 `here <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_.

476 The parser applies formats one by one, taking into account the detected languages.

477 :type date_formats: list

478

479 :return: a ``DateData`` object.

480

481 :raises: ValueError - Unknown Language

482

483 .. note:: *Period* values can be a 'day' (default), 'week', 'month', 'year', 'time'.

484

485 *Period* represents the granularity of date parsed from the given string.

486

487 In the example below, since no day information is present, the day is assumed to be current

488 day ``16`` from *current date* (which is June 16, 2015, at the moment of writing this).

489 Hence, the level of precision is ``month``:

490

491 >>> DateDataParser().get_date_data('March 2015')

492 DateData(date_obj=datetime.datetime(2015, 3, 16, 0, 0), period='month', locale='en')

493

494 Similarly, for date strings with no day and month information present, level of precision

495 is ``year`` and day ``16`` and month ``6`` are from *current_date*.

496

497 >>> DateDataParser().get_date_data('2014')

498 DateData(date_obj=datetime.datetime(2014, 6, 16, 0, 0), period='year', locale='en')

499

500 Dates with time zone indications or UTC offsets are returned in UTC time unless

501 specified using `Settings <https://dateparser.readthedocs.io/en/latest/settings.html#settings>`__.

502

503 >>> DateDataParser().get_date_data('23 March 2000, 1:21 PM CET')

504 DateData(date_obj=datetime.datetime(2000, 3, 23, 13, 21, tzinfo=<StaticTzInfo 'CET'>),

505 period='day', locale='en')

506

507 """

508 if not isinstance(date_string, str):

509 raise TypeError("Input type must be str")

510

511 res = parse_with_formats(date_string, date_formats or [], self._settings)

512 if res["date_obj"]:

513 return res

514

515 date_string = sanitize_date(date_string)

516

517 for locale in self._get_applicable_locales(date_string):

518 parsed_date = _DateLocaleParser.parse(

519 locale, date_string, date_formats, settings=self._settings

520 )

521 if parsed_date:

522 parsed_date["locale"] = locale.shortname

523 if self.try_previous_locales:

524 self.previous_locales[locale] = None

525 return parsed_date

526 else:

527 return DateData(date_obj=None, period="day", locale=None)

528

529 def get_date_tuple(self, *args, **kwargs):

530 date_data = self.get_date_data(*args, **kwargs)

531 fields = date_data.__dict__.keys()

532 date_tuple = collections.namedtuple("DateData", fields)

533 return date_tuple(**date_data.__dict__)

534

535 def _get_applicable_locales(self, date_string):

536 pop_tz_cache = []

537

538 def date_strings():

539 """A generator instead of a static list to avoid calling

540 pop_tz_offset_from_string if the first locale matches on unmodified

541 date_string.

542 """

543 yield date_string

544 if not pop_tz_cache:

545 stripped_date_string, _ = pop_tz_offset_from_string(

546 date_string, as_offset=False

547 )

548 if stripped_date_string == date_string:

549 stripped_date_string = None

550 pop_tz_cache[:] = [stripped_date_string]

551 (stripped_date_string,) = pop_tz_cache

552 if stripped_date_string is not None:

553 yield stripped_date_string

554

555 if self.try_previous_locales:

556 for locale in self.previous_locales.keys():

557 for s in date_strings():

558 if self._is_applicable_locale(locale, s):

559 yield locale

560

561 if self.detect_languages_function and not self.languages and not self.locales:

562 detected_languages = self.detect_languages_function(

563 text=date_string,

564 confidence_threshold=self._settings.LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD,

565 )

566

567 self.languages = map_languages(detected_languages)

568

569 for locale in self._get_locale_loader().get_locales(

570 languages=self.languages,

571 locales=self.locales,

572 region=self.region,

573 use_given_order=self.use_given_order,

574 ):

575 for s in date_strings():

576 if self._is_applicable_locale(locale, s):

577 yield locale

578

579 if self._settings.DEFAULT_LANGUAGES:

580 for locale in self._get_locale_loader().get_locales(

581 languages=self._settings.DEFAULT_LANGUAGES,

582 locales=None,

583 region=self.region,

584 use_given_order=self.use_given_order,

585 ):

586 yield locale

587

588 def _is_applicable_locale(self, locale, date_string):

589 return locale.is_applicable(

590 date_string,

591 strip_timezone=False, # it is stripped outside

592 settings=self._settings,

593 )

594

595 @classmethod

596 def _get_locale_loader(cls):

597 if not cls.locale_loader:

598 cls.locale_loader = LocaleDataLoader()

599 return cls.locale_loader