Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tornado/locale.py: 20%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

220 statements  

1# Copyright 2009 Facebook 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); you may 

4# not use this file except in compliance with the License. You may obtain 

5# a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 

11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 

12# License for the specific language governing permissions and limitations 

13# under the License. 

14 

15"""Translation methods for generating localized strings. 

16 

17To load a locale and generate a translated string:: 

18 

19 user_locale = tornado.locale.get("es_LA") 

20 print(user_locale.translate("Sign out")) 

21 

22`tornado.locale.get()` returns the closest matching locale, not necessarily the 

23specific locale you requested. You can support pluralization with 

24additional arguments to `~Locale.translate()`, e.g.:: 

25 

26 people = [...] 

27 message = user_locale.translate( 

28 "%(list)s is online", "%(list)s are online", len(people)) 

29 print(message % {"list": user_locale.list(people)}) 

30 

31The first string is chosen if ``len(people) == 1``, otherwise the second 

32string is chosen. 

33 

34Applications should call one of `load_translations` (which uses a simple 

35CSV format) or `load_gettext_translations` (which uses the ``.mo`` format 

36supported by `gettext` and related tools). If neither method is called, 

37the `Locale.translate` method will simply return the original string. 

38""" 

39 

40import codecs 

41import csv 

42import datetime 

43import gettext 

44import glob 

45import os 

46import re 

47 

48from tornado import escape 

49from tornado.log import gen_log 

50 

51from tornado._locale_data import LOCALE_NAMES 

52 

53from typing import Iterable, Any, Union, Dict, Optional 

54 

55_default_locale = "en_US" 

56_translations = {} # type: Dict[str, Any] 

57_supported_locales = frozenset([_default_locale]) 

58_use_gettext = False 

59CONTEXT_SEPARATOR = "\x04" 

60 

61 

62def get(*locale_codes: str) -> "Locale": 

63 """Returns the closest match for the given locale codes. 

64 

65 We iterate over all given locale codes in order. If we have a tight 

66 or a loose match for the code (e.g., "en" for "en_US"), we return 

67 the locale. Otherwise we move to the next code in the list. 

68 

69 By default we return ``en_US`` if no translations are found for any of 

70 the specified locales. You can change the default locale with 

71 `set_default_locale()`. 

72 """ 

73 return Locale.get_closest(*locale_codes) 

74 

75 

76def set_default_locale(code: str) -> None: 

77 """Sets the default locale. 

78 

79 The default locale is assumed to be the language used for all strings 

80 in the system. The translations loaded from disk are mappings from 

81 the default locale to the destination locale. Consequently, you don't 

82 need to create a translation file for the default locale. 

83 """ 

84 global _default_locale 

85 global _supported_locales 

86 _default_locale = code 

87 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) 

88 

89 

90def load_translations(directory: str, encoding: Optional[str] = None) -> None: 

91 """Loads translations from CSV files in a directory. 

92 

93 Translations are strings with optional Python-style named placeholders 

94 (e.g., ``My name is %(name)s``) and their associated translations. 

95 

96 The directory should have translation files of the form ``LOCALE.csv``, 

97 e.g. ``es_GT.csv``. The CSV files should have two or three columns: string, 

98 translation, and an optional plural indicator. Plural indicators should 

99 be one of "plural" or "singular". A given string can have both singular 

100 and plural forms. For example ``%(name)s liked this`` may have a 

101 different verb conjugation depending on whether %(name)s is one 

102 name or a list of names. There should be two rows in the CSV file for 

103 that string, one with plural indicator "singular", and one "plural". 

104 For strings with no verbs that would change on translation, simply 

105 use "unknown" or the empty string (or don't include the column at all). 

106 

107 The file is read using the `csv` module in the default "excel" dialect. 

108 In this format there should not be spaces after the commas. 

109 

110 If no ``encoding`` parameter is given, the encoding will be 

111 detected automatically (among UTF-8 and UTF-16) if the file 

112 contains a byte-order marker (BOM), defaulting to UTF-8 if no BOM 

113 is present. 

114 

115 Example translation ``es_LA.csv``:: 

116 

117 "I love you","Te amo" 

118 "%(name)s liked this","A %(name)s les gustó esto","plural" 

119 "%(name)s liked this","A %(name)s le gustó esto","singular" 

120 

121 .. versionchanged:: 4.3 

122 Added ``encoding`` parameter. Added support for BOM-based encoding 

123 detection, UTF-16, and UTF-8-with-BOM. 

124 """ 

125 global _translations 

126 global _supported_locales 

127 _translations = {} 

128 for path in os.listdir(directory): 

129 if not path.endswith(".csv"): 

130 continue 

131 locale, extension = path.split(".") 

132 if not re.match("[a-z]+(_[A-Z]+)?$", locale): 

133 gen_log.error( 

134 "Unrecognized locale %r (path: %s)", 

135 locale, 

136 os.path.join(directory, path), 

137 ) 

138 continue 

139 full_path = os.path.join(directory, path) 

140 if encoding is None: 

141 # Try to autodetect encoding based on the BOM. 

142 with open(full_path, "rb") as bf: 

143 data = bf.read(len(codecs.BOM_UTF16_LE)) 

144 if data in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE): 

145 encoding = "utf-16" 

146 else: 

147 # utf-8-sig is "utf-8 with optional BOM". It's discouraged 

148 # in most cases but is common with CSV files because Excel 

149 # cannot read utf-8 files without a BOM. 

150 encoding = "utf-8-sig" 

151 # python 3: csv.reader requires a file open in text mode. 

152 # Specify an encoding to avoid dependence on $LANG environment variable. 

153 with open(full_path, encoding=encoding) as f: 

154 _translations[locale] = {} 

155 for i, row in enumerate(csv.reader(f)): 

156 if not row or len(row) < 2: 

157 continue 

158 row = [escape.to_unicode(c).strip() for c in row] 

159 english, translation = row[:2] 

160 if len(row) > 2: 

161 plural = row[2] or "unknown" 

162 else: 

163 plural = "unknown" 

164 if plural not in ("plural", "singular", "unknown"): 

165 gen_log.error( 

166 "Unrecognized plural indicator %r in %s line %d", 

167 plural, 

168 path, 

169 i + 1, 

170 ) 

171 continue 

172 _translations[locale].setdefault(plural, {})[english] = translation 

173 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) 

174 gen_log.debug("Supported locales: %s", sorted(_supported_locales)) 

175 

176 

177def load_gettext_translations(directory: str, domain: str) -> None: 

178 """Loads translations from `gettext`'s locale tree 

179 

180 Locale tree is similar to system's ``/usr/share/locale``, like:: 

181 

182 {directory}/{lang}/LC_MESSAGES/{domain}.mo 

183 

184 Three steps are required to have your app translated: 

185 

186 1. Generate POT translation file:: 

187 

188 xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc 

189 

190 2. Merge against existing POT file:: 

191 

192 msgmerge old.po mydomain.po > new.po 

193 

194 3. Compile:: 

195 

196 msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo 

197 """ 

198 global _translations 

199 global _supported_locales 

200 global _use_gettext 

201 _translations = {} 

202 

203 for filename in glob.glob( 

204 os.path.join(directory, "*", "LC_MESSAGES", domain + ".mo") 

205 ): 

206 lang = os.path.basename(os.path.dirname(os.path.dirname(filename))) 

207 try: 

208 _translations[lang] = gettext.translation( 

209 domain, directory, languages=[lang] 

210 ) 

211 except Exception as e: 

212 gen_log.error("Cannot load translation for '%s': %s", lang, str(e)) 

213 continue 

214 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) 

215 _use_gettext = True 

216 gen_log.debug("Supported locales: %s", sorted(_supported_locales)) 

217 

218 

219def get_supported_locales() -> Iterable[str]: 

220 """Returns a list of all the supported locale codes.""" 

221 return _supported_locales 

222 

223 

224class Locale(object): 

225 """Object representing a locale. 

226 

227 After calling one of `load_translations` or `load_gettext_translations`, 

228 call `get` or `get_closest` to get a Locale object. 

229 """ 

230 

231 _cache = {} # type: Dict[str, Locale] 

232 

233 @classmethod 

234 def get_closest(cls, *locale_codes: str) -> "Locale": 

235 """Returns the closest match for the given locale code.""" 

236 for code in locale_codes: 

237 if not code: 

238 continue 

239 code = code.replace("-", "_") 

240 parts = code.split("_") 

241 if len(parts) > 2: 

242 continue 

243 elif len(parts) == 2: 

244 code = parts[0].lower() + "_" + parts[1].upper() 

245 if code in _supported_locales: 

246 return cls.get(code) 

247 if parts[0].lower() in _supported_locales: 

248 return cls.get(parts[0].lower()) 

249 return cls.get(_default_locale) 

250 

251 @classmethod 

252 def get(cls, code: str) -> "Locale": 

253 """Returns the Locale for the given locale code. 

254 

255 If it is not supported, we raise an exception. 

256 """ 

257 if code not in cls._cache: 

258 assert code in _supported_locales 

259 translations = _translations.get(code, None) 

260 if translations is None: 

261 locale = CSVLocale(code, {}) # type: Locale 

262 elif _use_gettext: 

263 locale = GettextLocale(code, translations) 

264 else: 

265 locale = CSVLocale(code, translations) 

266 cls._cache[code] = locale 

267 return cls._cache[code] 

268 

269 def __init__(self, code: str) -> None: 

270 self.code = code 

271 self.name = LOCALE_NAMES.get(code, {}).get("name", "Unknown") 

272 self.rtl = False 

273 for prefix in ["fa", "ar", "he"]: 

274 if self.code.startswith(prefix): 

275 self.rtl = True 

276 break 

277 

278 # Initialize strings for date formatting 

279 _ = self.translate 

280 self._months = [ 

281 _("January"), 

282 _("February"), 

283 _("March"), 

284 _("April"), 

285 _("May"), 

286 _("June"), 

287 _("July"), 

288 _("August"), 

289 _("September"), 

290 _("October"), 

291 _("November"), 

292 _("December"), 

293 ] 

294 self._weekdays = [ 

295 _("Monday"), 

296 _("Tuesday"), 

297 _("Wednesday"), 

298 _("Thursday"), 

299 _("Friday"), 

300 _("Saturday"), 

301 _("Sunday"), 

302 ] 

303 

304 def translate( 

305 self, 

306 message: str, 

307 plural_message: Optional[str] = None, 

308 count: Optional[int] = None, 

309 ) -> str: 

310 """Returns the translation for the given message for this locale. 

311 

312 If ``plural_message`` is given, you must also provide 

313 ``count``. We return ``plural_message`` when ``count != 1``, 

314 and we return the singular form for the given message when 

315 ``count == 1``. 

316 """ 

317 raise NotImplementedError() 

318 

319 def pgettext( 

320 self, 

321 context: str, 

322 message: str, 

323 plural_message: Optional[str] = None, 

324 count: Optional[int] = None, 

325 ) -> str: 

326 raise NotImplementedError() 

327 

328 def format_date( 

329 self, 

330 date: Union[int, float, datetime.datetime], 

331 gmt_offset: int = 0, 

332 relative: bool = True, 

333 shorter: bool = False, 

334 full_format: bool = False, 

335 ) -> str: 

336 """Formats the given date. 

337 

338 By default, we return a relative time (e.g., "2 minutes ago"). You 

339 can return an absolute date string with ``relative=False``. 

340 

341 You can force a full format date ("July 10, 1980") with 

342 ``full_format=True``. 

343 

344 This method is primarily intended for dates in the past. 

345 For dates in the future, we fall back to full format. 

346 

347 .. versionchanged:: 6.4 

348 Aware `datetime.datetime` objects are now supported (naive 

349 datetimes are still assumed to be UTC). 

350 """ 

351 if isinstance(date, (int, float)): 

352 date = datetime.datetime.fromtimestamp(date, datetime.timezone.utc) 

353 if date.tzinfo is None: 

354 date = date.replace(tzinfo=datetime.timezone.utc) 

355 now = datetime.datetime.now(datetime.timezone.utc) 

356 if date > now: 

357 if relative and (date - now).seconds < 60: 

358 # Due to click skew, things are some things slightly 

359 # in the future. Round timestamps in the immediate 

360 # future down to now in relative mode. 

361 date = now 

362 else: 

363 # Otherwise, future dates always use the full format. 

364 full_format = True 

365 local_date = date - datetime.timedelta(minutes=gmt_offset) 

366 local_now = now - datetime.timedelta(minutes=gmt_offset) 

367 local_yesterday = local_now - datetime.timedelta(hours=24) 

368 difference = now - date 

369 seconds = difference.seconds 

370 days = difference.days 

371 

372 _ = self.translate 

373 format = None 

374 if not full_format: 

375 if relative and days == 0: 

376 if seconds < 50: 

377 return _("1 second ago", "%(seconds)d seconds ago", seconds) % { 

378 "seconds": seconds 

379 } 

380 

381 if seconds < 50 * 60: 

382 minutes = round(seconds / 60.0) 

383 return _("1 minute ago", "%(minutes)d minutes ago", minutes) % { 

384 "minutes": minutes 

385 } 

386 

387 hours = round(seconds / (60.0 * 60)) 

388 return _("1 hour ago", "%(hours)d hours ago", hours) % {"hours": hours} 

389 

390 if days == 0: 

391 format = _("%(time)s") 

392 elif days == 1 and local_date.day == local_yesterday.day and relative: 

393 format = _("yesterday") if shorter else _("yesterday at %(time)s") 

394 elif days < 5: 

395 format = _("%(weekday)s") if shorter else _("%(weekday)s at %(time)s") 

396 elif days < 334: # 11mo, since confusing for same month last year 

397 format = ( 

398 _("%(month_name)s %(day)s") 

399 if shorter 

400 else _("%(month_name)s %(day)s at %(time)s") 

401 ) 

402 

403 if format is None: 

404 format = ( 

405 _("%(month_name)s %(day)s, %(year)s") 

406 if shorter 

407 else _("%(month_name)s %(day)s, %(year)s at %(time)s") 

408 ) 

409 

410 tfhour_clock = self.code not in ("en", "en_US", "zh_CN") 

411 if tfhour_clock: 

412 str_time = "%d:%02d" % (local_date.hour, local_date.minute) 

413 elif self.code == "zh_CN": 

414 str_time = "%s%d:%02d" % ( 

415 ("\u4e0a\u5348", "\u4e0b\u5348")[local_date.hour >= 12], 

416 local_date.hour % 12 or 12, 

417 local_date.minute, 

418 ) 

419 else: 

420 str_time = "%d:%02d %s" % ( 

421 local_date.hour % 12 or 12, 

422 local_date.minute, 

423 ("am", "pm")[local_date.hour >= 12], 

424 ) 

425 

426 return format % { 

427 "month_name": self._months[local_date.month - 1], 

428 "weekday": self._weekdays[local_date.weekday()], 

429 "day": str(local_date.day), 

430 "year": str(local_date.year), 

431 "time": str_time, 

432 } 

433 

434 def format_day( 

435 self, date: datetime.datetime, gmt_offset: int = 0, dow: bool = True 

436 ) -> bool: 

437 """Formats the given date as a day of week. 

438 

439 Example: "Monday, January 22". You can remove the day of week with 

440 ``dow=False``. 

441 """ 

442 local_date = date - datetime.timedelta(minutes=gmt_offset) 

443 _ = self.translate 

444 if dow: 

445 return _("%(weekday)s, %(month_name)s %(day)s") % { 

446 "month_name": self._months[local_date.month - 1], 

447 "weekday": self._weekdays[local_date.weekday()], 

448 "day": str(local_date.day), 

449 } 

450 else: 

451 return _("%(month_name)s %(day)s") % { 

452 "month_name": self._months[local_date.month - 1], 

453 "day": str(local_date.day), 

454 } 

455 

456 def list(self, parts: Any) -> str: 

457 """Returns a comma-separated list for the given list of parts. 

458 

459 The format is, e.g., "A, B and C", "A and B" or just "A" for lists 

460 of size 1. 

461 """ 

462 _ = self.translate 

463 if len(parts) == 0: 

464 return "" 

465 if len(parts) == 1: 

466 return parts[0] 

467 comma = " \u0648 " if self.code.startswith("fa") else ", " 

468 return _("%(commas)s and %(last)s") % { 

469 "commas": comma.join(parts[:-1]), 

470 "last": parts[len(parts) - 1], 

471 } 

472 

473 def friendly_number(self, value: int) -> str: 

474 """Returns a comma-separated number for the given integer.""" 

475 if self.code not in ("en", "en_US"): 

476 return str(value) 

477 s = str(value) 

478 parts = [] 

479 while s: 

480 parts.append(s[-3:]) 

481 s = s[:-3] 

482 return ",".join(reversed(parts)) 

483 

484 

485class CSVLocale(Locale): 

486 """Locale implementation using tornado's CSV translation format.""" 

487 

488 def __init__(self, code: str, translations: Dict[str, Dict[str, str]]) -> None: 

489 self.translations = translations 

490 super().__init__(code) 

491 

492 def translate( 

493 self, 

494 message: str, 

495 plural_message: Optional[str] = None, 

496 count: Optional[int] = None, 

497 ) -> str: 

498 if plural_message is not None: 

499 assert count is not None 

500 if count != 1: 

501 message = plural_message 

502 message_dict = self.translations.get("plural", {}) 

503 else: 

504 message_dict = self.translations.get("singular", {}) 

505 else: 

506 message_dict = self.translations.get("unknown", {}) 

507 return message_dict.get(message, message) 

508 

509 def pgettext( 

510 self, 

511 context: str, 

512 message: str, 

513 plural_message: Optional[str] = None, 

514 count: Optional[int] = None, 

515 ) -> str: 

516 if self.translations: 

517 gen_log.warning("pgettext is not supported by CSVLocale") 

518 return self.translate(message, plural_message, count) 

519 

520 

521class GettextLocale(Locale): 

522 """Locale implementation using the `gettext` module.""" 

523 

524 def __init__(self, code: str, translations: gettext.NullTranslations) -> None: 

525 self.ngettext = translations.ngettext 

526 self.gettext = translations.gettext 

527 # self.gettext must exist before __init__ is called, since it 

528 # calls into self.translate 

529 super().__init__(code) 

530 

531 def translate( 

532 self, 

533 message: str, 

534 plural_message: Optional[str] = None, 

535 count: Optional[int] = None, 

536 ) -> str: 

537 if plural_message is not None: 

538 assert count is not None 

539 return self.ngettext(message, plural_message, count) 

540 else: 

541 return self.gettext(message) 

542 

543 def pgettext( 

544 self, 

545 context: str, 

546 message: str, 

547 plural_message: Optional[str] = None, 

548 count: Optional[int] = None, 

549 ) -> str: 

550 """Allows to set context for translation, accepts plural forms. 

551 

552 Usage example:: 

553 

554 pgettext("law", "right") 

555 pgettext("good", "right") 

556 

557 Plural message example:: 

558 

559 pgettext("organization", "club", "clubs", len(clubs)) 

560 pgettext("stick", "club", "clubs", len(clubs)) 

561 

562 To generate POT file with context, add following options to step 1 

563 of `load_gettext_translations` sequence:: 

564 

565 xgettext [basic options] --keyword=pgettext:1c,2 --keyword=pgettext:1c,2,3 

566 

567 .. versionadded:: 4.2 

568 """ 

569 if plural_message is not None: 

570 assert count is not None 

571 msgs_with_ctxt = ( 

572 "%s%s%s" % (context, CONTEXT_SEPARATOR, message), 

573 "%s%s%s" % (context, CONTEXT_SEPARATOR, plural_message), 

574 count, 

575 ) 

576 result = self.ngettext(*msgs_with_ctxt) 

577 if CONTEXT_SEPARATOR in result: 

578 # Translation not found 

579 result = self.ngettext(message, plural_message, count) 

580 return result 

581 else: 

582 msg_with_ctxt = "%s%s%s" % (context, CONTEXT_SEPARATOR, message) 

583 result = self.gettext(msg_with_ctxt) 

584 if CONTEXT_SEPARATOR in result: 

585 # Translation not found 

586 result = message 

587 return result