Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tornado/locale.py: 20%

3# Licensed under the Apache License, Version 2.0 (the "License"); you may

4# not use this file except in compliance with the License. You may obtain

5# a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT

11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the

12# License for the specific language governing permissions and limitations

13# under the License.

15"""Translation methods for generating localized strings.

17To load a locale and generate a translated string::

19 user_locale = tornado.locale.get("es_LA")

20 print(user_locale.translate("Sign out"))

22`tornado.locale.get()` returns the closest matching locale, not necessarily the

23specific locale you requested. You can support pluralization with

24additional arguments to `~Locale.translate()`, e.g.::

26 people = [...]

27 message = user_locale.translate(

28 "%(list)s is online", "%(list)s are online", len(people))

29 print(message % {"list": user_locale.list(people)})

31The first string is chosen if ``len(people) == 1``, otherwise the second

32string is chosen.

34Applications should call one of `load_translations` (which uses a simple

35CSV format) or `load_gettext_translations` (which uses the ``.mo`` format

36supported by `gettext` and related tools). If neither method is called,

37the `Locale.translate` method will simply return the original string.

38"""

40import codecs

41import csv

42import datetime

43import gettext

44import glob

45import os

46import re

48from tornado import escape

49from tornado.log import gen_log

51from tornado._locale_data import LOCALE_NAMES

53from typing import Iterable, Any, Union, Dict, Optional

55_default_locale = "en_US"

56_translations = {} # type: Dict[str, Any]

57_supported_locales = frozenset([_default_locale])

58_use_gettext = False

59CONTEXT_SEPARATOR = "\x04"

62def get(*locale_codes: str) -> "Locale":

63 """Returns the closest match for the given locale codes.

65 We iterate over all given locale codes in order. If we have a tight

66 or a loose match for the code (e.g., "en" for "en_US"), we return

67 the locale. Otherwise we move to the next code in the list.

69 By default we return ``en_US`` if no translations are found for any of

70 the specified locales. You can change the default locale with

71 `set_default_locale()`.

72 """

73 return Locale.get_closest(*locale_codes)

76def set_default_locale(code: str) -> None:

77 """Sets the default locale.

79 The default locale is assumed to be the language used for all strings

80 in the system. The translations loaded from disk are mappings from

81 the default locale to the destination locale. Consequently, you don't

82 need to create a translation file for the default locale.

83 """

84 global _default_locale

85 global _supported_locales

86 _default_locale = code

87 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])

90def load_translations(directory: str, encoding: Optional[str] = None) -> None:

91 """Loads translations from CSV files in a directory.

93 Translations are strings with optional Python-style named placeholders

94 (e.g., ``My name is %(name)s``) and their associated translations.

96 The directory should have translation files of the form ``LOCALE.csv``,

97 e.g. ``es_GT.csv``. The CSV files should have two or three columns: string,

98 translation, and an optional plural indicator. Plural indicators should

99 be one of "plural" or "singular". A given string can have both singular

100 and plural forms. For example ``%(name)s liked this`` may have a

101 different verb conjugation depending on whether %(name)s is one

102 name or a list of names. There should be two rows in the CSV file for

103 that string, one with plural indicator "singular", and one "plural".

104 For strings with no verbs that would change on translation, simply

105 use "unknown" or the empty string (or don't include the column at all).

106

107 The file is read using the `csv` module in the default "excel" dialect.

108 In this format there should not be spaces after the commas.

109

110 If no ``encoding`` parameter is given, the encoding will be

111 detected automatically (among UTF-8 and UTF-16) if the file

112 contains a byte-order marker (BOM), defaulting to UTF-8 if no BOM

113 is present.

114

115 Example translation ``es_LA.csv``::

116

117 "I love you","Te amo"

118 "%(name)s liked this","A %(name)s les gustó esto","plural"

119 "%(name)s liked this","A %(name)s le gustó esto","singular"

120

121 .. versionchanged:: 4.3

122 Added ``encoding`` parameter. Added support for BOM-based encoding

123 detection, UTF-16, and UTF-8-with-BOM.

124 """

125 global _translations

126 global _supported_locales

127 _translations = {}

128 for path in os.listdir(directory):

129 if not path.endswith(".csv"):

130 continue

131 locale, extension = path.split(".")

132 if not re.match("[a-z]+(_[A-Z]+)?$", locale):

133 gen_log.error(

134 "Unrecognized locale %r (path: %s)",

135 locale,

136 os.path.join(directory, path),

137 )

138 continue

139 full_path = os.path.join(directory, path)

140 if encoding is None:

141 # Try to autodetect encoding based on the BOM.

142 with open(full_path, "rb") as bf:

143 data = bf.read(len(codecs.BOM_UTF16_LE))

144 if data in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):

145 encoding = "utf-16"

146 else:

147 # utf-8-sig is "utf-8 with optional BOM". It's discouraged

148 # in most cases but is common with CSV files because Excel

149 # cannot read utf-8 files without a BOM.

150 encoding = "utf-8-sig"

151 # python 3: csv.reader requires a file open in text mode.

152 # Specify an encoding to avoid dependence on $LANG environment variable.

153 with open(full_path, encoding=encoding) as f:

154 _translations[locale] = {}

155 for i, row in enumerate(csv.reader(f)):

156 if not row or len(row) < 2:

157 continue

158 row = [escape.to_unicode(c).strip() for c in row]

159 english, translation = row[:2]

160 if len(row) > 2:

161 plural = row[2] or "unknown"

162 else:

163 plural = "unknown"

164 if plural not in ("plural", "singular", "unknown"):

165 gen_log.error(

166 "Unrecognized plural indicator %r in %s line %d",

167 plural,

168 path,

169 i + 1,

170 )

171 continue

172 _translations[locale].setdefault(plural, {})[english] = translation

173 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])

174 gen_log.debug("Supported locales: %s", sorted(_supported_locales))

175

176

177def load_gettext_translations(directory: str, domain: str) -> None:

178 """Loads translations from `gettext`'s locale tree

179

180 Locale tree is similar to system's ``/usr/share/locale``, like::

181

182 {directory}/{lang}/LC_MESSAGES/{domain}.mo

183

184 Three steps are required to have your app translated:

185

186 1. Generate POT translation file::

187

188 xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc

189

190 2. Merge against existing POT file::

191

192 msgmerge old.po mydomain.po > new.po

193

194 3. Compile::

195

196 msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo

197 """

198 global _translations

199 global _supported_locales

200 global _use_gettext

201 _translations = {}

202

203 for filename in glob.glob(

204 os.path.join(directory, "*", "LC_MESSAGES", domain + ".mo")

205 ):

206 lang = os.path.basename(os.path.dirname(os.path.dirname(filename)))

207 try:

208 _translations[lang] = gettext.translation(

209 domain, directory, languages=[lang]

210 )

211 except Exception as e:

212 gen_log.error("Cannot load translation for '%s': %s", lang, str(e))

213 continue

214 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])

215 _use_gettext = True

216 gen_log.debug("Supported locales: %s", sorted(_supported_locales))

217

218

219def get_supported_locales() -> Iterable[str]:

220 """Returns a list of all the supported locale codes."""

221 return _supported_locales

222

223

224class Locale(object):

225 """Object representing a locale.

226

227 After calling one of `load_translations` or `load_gettext_translations`,

228 call `get` or `get_closest` to get a Locale object.

229 """

230

231 _cache = {} # type: Dict[str, Locale]

232

233 @classmethod

234 def get_closest(cls, *locale_codes: str) -> "Locale":

235 """Returns the closest match for the given locale code."""

236 for code in locale_codes:

237 if not code:

238 continue

239 code = code.replace("-", "_")

240 parts = code.split("_")

241 if len(parts) > 2:

242 continue

243 elif len(parts) == 2:

244 code = parts[0].lower() + "_" + parts[1].upper()

245 if code in _supported_locales:

246 return cls.get(code)

247 if parts[0].lower() in _supported_locales:

248 return cls.get(parts[0].lower())

249 return cls.get(_default_locale)

250

251 @classmethod

252 def get(cls, code: str) -> "Locale":

253 """Returns the Locale for the given locale code.

254

255 If it is not supported, we raise an exception.

256 """

257 if code not in cls._cache:

258 assert code in _supported_locales

259 translations = _translations.get(code, None)

260 if translations is None:

261 locale = CSVLocale(code, {}) # type: Locale

262 elif _use_gettext:

263 locale = GettextLocale(code, translations)

264 else:

265 locale = CSVLocale(code, translations)

266 cls._cache[code] = locale

267 return cls._cache[code]

268

269 def __init__(self, code: str) -> None:

270 self.code = code

271 self.name = LOCALE_NAMES.get(code, {}).get("name", "Unknown")

272 self.rtl = False

273 for prefix in ["fa", "ar", "he"]:

274 if self.code.startswith(prefix):

275 self.rtl = True

276 break

277

278 # Initialize strings for date formatting

279 _ = self.translate

280 self._months = [

281 _("January"),

282 _("February"),

283 _("March"),

284 _("April"),

285 _("May"),

286 _("June"),

287 _("July"),

288 _("August"),

289 _("September"),

290 _("October"),

291 _("November"),

292 _("December"),

293 ]

294 self._weekdays = [

295 _("Monday"),

296 _("Tuesday"),

297 _("Wednesday"),

298 _("Thursday"),

299 _("Friday"),

300 _("Saturday"),

301 _("Sunday"),

302 ]

303

304 def translate(

305 self,

306 message: str,

307 plural_message: Optional[str] = None,

308 count: Optional[int] = None,

309 ) -> str:

310 """Returns the translation for the given message for this locale.

311

312 If ``plural_message`` is given, you must also provide

313 ``count``. We return ``plural_message`` when ``count != 1``,

314 and we return the singular form for the given message when

315 ``count == 1``.

316 """

317 raise NotImplementedError()

318

319 def pgettext(

320 self,

321 context: str,

322 message: str,

323 plural_message: Optional[str] = None,

324 count: Optional[int] = None,

325 ) -> str:

326 raise NotImplementedError()

327

328 def format_date(

329 self,

330 date: Union[int, float, datetime.datetime],

331 gmt_offset: int = 0,

332 relative: bool = True,

333 shorter: bool = False,

334 full_format: bool = False,

335 ) -> str:

336 """Formats the given date (which should be GMT).

337

338 By default, we return a relative time (e.g., "2 minutes ago"). You

339 can return an absolute date string with ``relative=False``.

340

341 You can force a full format date ("July 10, 1980") with

342 ``full_format=True``.

343

344 This method is primarily intended for dates in the past.

345 For dates in the future, we fall back to full format.

346 """

347 if isinstance(date, (int, float)):

348 date = datetime.datetime.utcfromtimestamp(date)

349 now = datetime.datetime.utcnow()

350 if date > now:

351 if relative and (date - now).seconds < 60:

352 # Due to click skew, things are some things slightly

353 # in the future. Round timestamps in the immediate

354 # future down to now in relative mode.

355 date = now

356 else:

357 # Otherwise, future dates always use the full format.

358 full_format = True

359 local_date = date - datetime.timedelta(minutes=gmt_offset)

360 local_now = now - datetime.timedelta(minutes=gmt_offset)

361 local_yesterday = local_now - datetime.timedelta(hours=24)

362 difference = now - date

363 seconds = difference.seconds

364 days = difference.days

365

366 _ = self.translate

367 format = None

368 if not full_format:

369 if relative and days == 0:

370 if seconds < 50:

371 return _("1 second ago", "%(seconds)d seconds ago", seconds) % {

372 "seconds": seconds

373 }

374

375 if seconds < 50 * 60:

376 minutes = round(seconds / 60.0)

377 return _("1 minute ago", "%(minutes)d minutes ago", minutes) % {

378 "minutes": minutes

379 }

380

381 hours = round(seconds / (60.0 * 60))

382 return _("1 hour ago", "%(hours)d hours ago", hours) % {"hours": hours}

383

384 if days == 0:

385 format = _("%(time)s")

386 elif days == 1 and local_date.day == local_yesterday.day and relative:

387 format = _("yesterday") if shorter else _("yesterday at %(time)s")

388 elif days < 5:

389 format = _("%(weekday)s") if shorter else _("%(weekday)s at %(time)s")

390 elif days < 334: # 11mo, since confusing for same month last year

391 format = (

392 _("%(month_name)s %(day)s")

393 if shorter

394 else _("%(month_name)s %(day)s at %(time)s")

395 )

396

397 if format is None:

398 format = (

399 _("%(month_name)s %(day)s, %(year)s")

400 if shorter

401 else _("%(month_name)s %(day)s, %(year)s at %(time)s")

402 )

403

404 tfhour_clock = self.code not in ("en", "en_US", "zh_CN")

405 if tfhour_clock:

406 str_time = "%d:%02d" % (local_date.hour, local_date.minute)

407 elif self.code == "zh_CN":

408 str_time = "%s%d:%02d" % (

409 ("\u4e0a\u5348", "\u4e0b\u5348")[local_date.hour >= 12],

410 local_date.hour % 12 or 12,

411 local_date.minute,

412 )

413 else:

414 str_time = "%d:%02d %s" % (

415 local_date.hour % 12 or 12,

416 local_date.minute,

417 ("am", "pm")[local_date.hour >= 12],

418 )

419

420 return format % {

421 "month_name": self._months[local_date.month - 1],

422 "weekday": self._weekdays[local_date.weekday()],

423 "day": str(local_date.day),

424 "year": str(local_date.year),

425 "time": str_time,

426 }

427

428 def format_day(

429 self, date: datetime.datetime, gmt_offset: int = 0, dow: bool = True

430 ) -> bool:

431 """Formats the given date as a day of week.

432

433 Example: "Monday, January 22". You can remove the day of week with

434 ``dow=False``.

435 """

436 local_date = date - datetime.timedelta(minutes=gmt_offset)

437 _ = self.translate

438 if dow:

439 return _("%(weekday)s, %(month_name)s %(day)s") % {

440 "month_name": self._months[local_date.month - 1],

441 "weekday": self._weekdays[local_date.weekday()],

442 "day": str(local_date.day),

443 }

444 else:

445 return _("%(month_name)s %(day)s") % {

446 "month_name": self._months[local_date.month - 1],

447 "day": str(local_date.day),

448 }

449

450 def list(self, parts: Any) -> str:

451 """Returns a comma-separated list for the given list of parts.

452

453 The format is, e.g., "A, B and C", "A and B" or just "A" for lists

454 of size 1.

455 """

456 _ = self.translate

457 if len(parts) == 0:

458 return ""

459 if len(parts) == 1:

460 return parts[0]

461 comma = " \u0648 " if self.code.startswith("fa") else ", "

462 return _("%(commas)s and %(last)s") % {

463 "commas": comma.join(parts[:-1]),

464 "last": parts[len(parts) - 1],

465 }

466

467 def friendly_number(self, value: int) -> str:

468 """Returns a comma-separated number for the given integer."""

469 if self.code not in ("en", "en_US"):

470 return str(value)

471 s = str(value)

472 parts = []

473 while s:

474 parts.append(s[-3:])

475 s = s[:-3]

476 return ",".join(reversed(parts))

477

478

479class CSVLocale(Locale):

480 """Locale implementation using tornado's CSV translation format."""

481

482 def __init__(self, code: str, translations: Dict[str, Dict[str, str]]) -> None:

483 self.translations = translations

484 super().__init__(code)

485

486 def translate(

487 self,

488 message: str,

489 plural_message: Optional[str] = None,

490 count: Optional[int] = None,

491 ) -> str:

492 if plural_message is not None:

493 assert count is not None

494 if count != 1:

495 message = plural_message

496 message_dict = self.translations.get("plural", {})

497 else:

498 message_dict = self.translations.get("singular", {})

499 else:

500 message_dict = self.translations.get("unknown", {})

501 return message_dict.get(message, message)

502

503 def pgettext(

504 self,

505 context: str,

506 message: str,

507 plural_message: Optional[str] = None,

508 count: Optional[int] = None,

509 ) -> str:

510 if self.translations:

511 gen_log.warning("pgettext is not supported by CSVLocale")

512 return self.translate(message, plural_message, count)

513

514

515class GettextLocale(Locale):

516 """Locale implementation using the `gettext` module."""

517

518 def __init__(self, code: str, translations: gettext.NullTranslations) -> None:

519 self.ngettext = translations.ngettext

520 self.gettext = translations.gettext

521 # self.gettext must exist before __init__ is called, since it

522 # calls into self.translate

523 super().__init__(code)

524

525 def translate(

526 self,

527 message: str,

528 plural_message: Optional[str] = None,

529 count: Optional[int] = None,

530 ) -> str:

531 if plural_message is not None:

532 assert count is not None

533 return self.ngettext(message, plural_message, count)

534 else:

535 return self.gettext(message)

536

537 def pgettext(

538 self,

539 context: str,

540 message: str,

541 plural_message: Optional[str] = None,

542 count: Optional[int] = None,

543 ) -> str:

544 """Allows to set context for translation, accepts plural forms.

545

546 Usage example::

547

548 pgettext("law", "right")

549 pgettext("good", "right")

550

551 Plural message example::

552

553 pgettext("organization", "club", "clubs", len(clubs))

554 pgettext("stick", "club", "clubs", len(clubs))

555

556 To generate POT file with context, add following options to step 1

557 of `load_gettext_translations` sequence::

558

559 xgettext [basic options] --keyword=pgettext:1c,2 --keyword=pgettext:1c,2,3

560

561 .. versionadded:: 4.2

562 """

563 if plural_message is not None:

564 assert count is not None

565 msgs_with_ctxt = (

566 "%s%s%s" % (context, CONTEXT_SEPARATOR, message),

567 "%s%s%s" % (context, CONTEXT_SEPARATOR, plural_message),

568 count,

569 )

570 result = self.ngettext(*msgs_with_ctxt)

571 if CONTEXT_SEPARATOR in result:

572 # Translation not found

573 result = self.ngettext(message, plural_message, count)

574 return result

575 else:

576 msg_with_ctxt = "%s%s%s" % (context, CONTEXT_SEPARATOR, message)

577 result = self.gettext(msg_with_ctxt)

578 if CONTEXT_SEPARATOR in result:

579 # Translation not found

580 result = message

581 return result