#!/usr/bin/env python3
"""
Builds Véto Garde's world base of veterinarians: one row per clinic, merged from three open sources.

  - Overture Maps places, the base: every place under taxonomy veterinary_care (veterinarian, animal_hospital,
    emergency_pet_hospital, veterinary_care; pet hospices, holistic care and animal physical therapy left out),
    not permanently closed, confidence ≥ 0.6, or ≥ 0.5 with a phone number (BrightQuery's records carry a flat 0.5,
    most emergency hospitals among them); head offices, vet associations, insurers and stand-alone vet pharmacies
    or labs left out (see NOT_A_CLINIC);
  - All the Places, the chains' own store locators (latest weekly run, CC0): opening hours, emergency networks;
  - OpenStreetMap amenity=veterinary, through Overpass in regional slices: opening hours, emergency=yes, 24/7,
    the OSM reference, and the clinics the other two lack.

Run by hand after an Overture release (every month or so), from the repository root:

  python3 deploy/world/build_vets.py --release 2026-10-21.0 --out build/vets [--bbox xmin,ymin,xmax,ymax] [--cache build/vets/cache]

It needs duckdb, phonenumbers and timezonefinder (pip install duckdb phonenumbers timezonefinder). The downloads
(Overture extract, All the Places spiders, Overpass slices) are cached in --cache (default <out>/cache): a rerun
reads them again instead of downloading; delete a file there to fetch it anew. The whole world takes about an hour
the first time (Overpass slices, sequential with a pause between), a few minutes from the cache. When Overpass does
not answer for 20 minutes (or takes more than 90 in all), the slices left come from QLever's OSM planet, which is a
few weeks older (cache/osm/qlever-<slice>.json: delete them to try Overpass again).

Output (then uploaded by hand: gcloud storage cp -r <out>/tiles <out>/emergency.json.gz <out>/manifest.json
gs://<data bucket>/world/):
  <out>/tiles/<lat>_<lng>.json.gz   vets of a 1° × 1° tile (south-west corner, integer degrees)
  <out>/emergency.json.gz           the vets flagged open 24/7 or emergency service, the whole world in one array
                                    (same rows), which the API loads once to search around a point
  <out>/manifest.json               {release, osm: date of the OSM extract (the oldest slice), atp: run id, builtAt,
                                     vets, tiles: {name: count}, countries: {"FR": [vets, withPhone, withHours,
                                     emergency]}, osmSlices: {overpass: n, qlever: n}} (+ osmMissing: the slices that
                                     could not be read at all, when there are any)
  <out>/open/                       the ODbL publication of the whole base: vets-<release>.json.gz, README.md, a copy
                                    of this script and index.html, its page (OpenStreetMap's share-alike: published
                                    with the app, gs://veto-garde-app-open/)

A vet: [id, lat, lng, name, phones, website, email, address, city, postcode, country, region, hours, hoursText,
flags, src, osm, tz]
  id         o:<Overture id> (kept when other sources merge into it), osm:n<id> | osm:w<id> | osm:r<id>,
             atp:<spider>:<ref> (the feature's hash id when the ref is a URL or a path)
  phones     international format ("+33 1 23 45 67 89"), 3 at most, [] when none; one that cannot be read as written
  address    street line; city, postcode; country ISO 3166-1 alpha-2; region as the source gives it
  hours      WeekHours (platform/shared stations.ts): 7 days Monday first, null unknown, [] closed, [open, close)
             in minutes, close ≤ 1440, an interval past midnight split onto the next day; null when unknown
  hoursText  the source's opening_hours when it could not be read (≤ 200 characters), else null
  flags      1 open 24/7, 2 emergency service
  src        1 Overture, 2 OpenStreetMap, 4 All the Places
  osm        node/123 | way/123 | relation/123 | null
  tz         IANA time zone of the hours, or null

Licences: Overture places are CDLA-Permissive-2.0 (Meta, Microsoft, BrightQuery, Uberall, PinMeTo, DAC…) or
Apache-2.0 (Foursquare); All the Places is CC0-1.0; OpenStreetMap is ODbL-1.0. The merged base is a derivative
database of OpenStreetMap: it is published under the ODbL (<out>/open/), and the app credits
"© OpenStreetMap contributors" (link to openstreetmap.org/copyright), the Overture Maps Foundation and All the Places.
"""
import argparse, gzip, html, json, math, os, re, shutil, string, sys, threading, time, unicodedata
import urllib.error, urllib.parse, urllib.request
from collections import Counter, defaultdict
from difflib import SequenceMatcher

try:  # the build needs it; the unit tests skip the phone cases without it
    import phonenumbers
except ImportError:  # pragma: no cover
    phonenumbers = None

# ———————————————————————————— settings

EXCLUDED_CATEGORIES = ("pet_hospice", "holistic_animal_care", "animal_physical_therapy")
MIN_CONFIDENCE = 0.6
MIN_CONFIDENCE_WITH_PHONE = 0.5

# duplicates inside Overture (metres)
DEDUPE_ANY_M = 25.0  # one place, whatever the names
DEDUPE_NAME_M = 60.0  # one place when the names are alike
DEDUPE_PHONE_M = 500.0  # the same number (chains share central numbers: never further)
# an All the Places or OSM vet is the same as a row already kept when (metres)
MATCH_PHONE_M = 1000.0
MATCH_DOMAIN_M = 300.0
MATCH_NAME_M = 100.0
MATCH_ANY_M = DEDUPE_ANY_M

MAX_PHONES = 3
MAX_HOURS_TEXT = 200

FLAG_OPEN24 = 1
FLAG_EMERGENCY = 2
SRC_OVERTURE = 1
SRC_OSM = 2
SRC_ATP = 4

COLUMNS = ["id", "lat", "lng", "name", "phones", "website", "email", "address", "city", "postcode", "country",
           "region", "hours", "hoursText", "flags", "src", "osm", "tz"]

USER_AGENT = "VetoGarde/1.0 (+https://veto-garde.pamsfactory.app)"
OVERTURE_BASE = "s3://overturemaps-us-west-2/release/{release}/theme=places/type=place/*"
ATP_INDEX = "https://data.alltheplaces.xyz/runs/latest.json"
OVERPASS_URL = "https://overpass-api.de/api/interpreter"
OVERPASS_STATUS = "https://overpass-api.de/api/status"
OVERPASS_TIMEOUT_S = 240
OVERPASS_PAUSE_S = 5.0
OVERPASS_MIN_CELL_DEG = 3.75  # a slice that keeps failing is split down to this size, then read from QLever
OVERPASS_MAX_SECONDS = 90 * 60  # past this, the slices not read yet go to QLever
OVERPASS_DOWN_S = 20 * 60  # Overpass not answering for this long: the slices left go to QLever
QLEVER_URL = "https://qlever.dev/api/osm-planet"

# All the Places spiders with vets (amenity=veterinary), from the runs of October 2026; spiders of the run whose
# names look like a vet chain (ATP_DISCOVER) are read too, and only their amenity=veterinary features are kept.
ATP_SPIDERS = (
    "anicura", "banfield_pet_hospital", "best_friends_au", "bluepearl_pet_hospital_us", "cvs_vets_gb",
    "greencross_au", "medivet_gb", "petstock_au", "petstock_nz", "spa_fr", "vca_us", "vets4pets_gb",
)
ATP_DISCOVER = re.compile(
    r"(?:^|_)(?:vets?|veterinar\w*|veterinaire\w*|tierarzt\w*|tierklinik\w*|dierenarts\w*|animal_hospital\w*|"
    r"pet_hospital\w*)(?:_|$)|anicura|evidensia|medivet|banfield|bluepearl|greencross|mon_?veto|ivc_"
)
# chains whose clinics are emergency hospitals, but for the names that match the pattern (specialty-only sites)
ATP_EMERGENCY_SPIDERS = {
    "bluepearl_pet_hospital_us": re.compile(r"rehab|fitness|neurolog|surgery|oncolog|cardiolog|dermatolog|ophthalm|dentist|imaging", re.I),
}

# hosts shared by unrelated places: never a reason to match two vets
GENERIC_HOSTS = (
    "facebook.com", "fb.com", "fb.me", "instagram.com", "google.com", "goo.gl", "g.page", "linktr.ee", "twitter.com",
    "x.com", "youtube.com", "tiktok.com", "linkedin.com", "yelp.com", "tripadvisor.com", "pagesjaunes.fr",
    "doctolib.fr", "wa.me", "whatsapp.com", "t.me", "bit.ly", "vk.com", "ok.ru", "booking.com", "jimdo.com",
    "wordpress.com", "blogspot.com", "wixsite.com", "squarespace.com", "petdesk.com", "vetstoria.com",
)

# ———————————————————————————— text

_TRANSLIT = str.maketrans({
    "ł": "l", "Ł": "l", "ø": "o", "Ø": "o", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe", "ß": "ss", "đ": "d",
    "Đ": "d", "ð": "d", "þ": "th", "ı": "i", "ħ": "h", "’": "'", "‘": "'", "´": "'", "`": "'", "–": "-", "—": "-",
})


def fold(s):
    """Lower case, accents dropped from Latin, Greek and Cyrillic letters (other scripts unchanged), spaces collapsed."""
    if not s:
        return ""
    out, base = [], 0
    for ch in unicodedata.normalize("NFKD", str(s).translate(_TRANSLIT)):
        if unicodedata.combining(ch):
            if base < 0x0530:
                continue
        else:
            base = ord(ch)
        out.append(ch)
    return " ".join(unicodedata.normalize("NFC", "".join(out)).lower().split())


def norm(s):
    return " ".join(str(s).split()) if isinstance(s, str) and s.strip() else None


# ———————————————————————————— not a clinic

NOT_A_CLINIC = re.compile(
    r"\b(?:head ?office|headquarters|support (?:office|cent(?:er|re))|corporate office|siege social|"
    r"insurance|assurance|versicherung\w*|veterinary medical association|association of veterinary|veterinary association|"
    r"tierarztekammer|ordre (?:national |regional )?des veterinaires|chambre veterinaire|colegio (?:oficial )?de veterinarios|"
    r"veterinary council|fachoberschule)\b"
)
COMPANY_ONLY = re.compile(r"\b(?:holdings?|verwaltung)\b")  # "J. Jakobs Holding B.V.", unless the name says vet
LAB_OR_SHOP = re.compile(r"^(?:laborator\w*|laboratoire|labo|farmacia|pharmacie|pharmacy|apotheke|distribuidora|drogueria|supply|supplies|wholesale)\b")
CLINIC_WORD = re.compile(r"clinic|hospital|consultorio|cabinet|klinik|praxis|surgery|centre veterinaire|centro veterinario")


def not_a_clinic(name):
    """Head offices, vet associations, insurers, stand-alone vet pharmacies and labs (Overture lists them as vets)."""
    f = fold(name)
    if not f:
        return False
    if NOT_A_CLINIC.search(f):
        return True
    if LAB_OR_SHOP.search(f) and not CLINIC_WORD.search(f):
        return True
    return bool(COMPANY_ONLY.search(f)) and not looks_like_clinic(f)


CLINIC_LIKE = re.compile(
    r"vet|clinic|klinik|kliniek|klinika|hospital|hopital|ospedale|tierarzt|tierklinik|dierenarts|dyrlaege|dyreklinik|djur|"
    r"lecznica|weteryn|veteriner|animal|animaux|tier|pet|cabinet|centre|center|centro|ambulator|praxis|"
    r"ветеринар|ветклиник|зоо|病院|医院|醫院|動物|动物|hayvan"
)


def looks_like_clinic(name):
    return bool(CLINIC_LIKE.search(fold(name)))


# ———————————————————————————— emergency

# on folded names (fold): literal pieces are folded the same way below, regular expressions are written folded
EMERGENCY_PATTERNS = [
    # French
    r"\burgences?\b", r"\bde garde\b(?!-)(?!\s*(?:d\s*'?\s*)?(?:animaux|chiens?|chats?|nac)\b)",
    r"\bgardes? (?:assurees?|veterinaires?)\b", r"\bcentre hospitalier veterinaire\b",
    # time around the clock, every language
    r"\b24 ?h ?/ ?24\b", r"\b24 ?/ ?24\b", r"\b24 ?/ ?7\b", r"\b7 ?/ ?24\b", r"\b24 ?(?:h|hs|hrs?)\b", r"\bh ?24\b",
    r"\b24[ -]?(?:hours?|horas|heures|ore|stunden|saat|jam)\b", r"\bround[ -]the[ -]clock\b",
    # English
    r"\bemergenc(?:y|ies)\b", r"\burgent\b", r"\bafter[ -]?hours?\b", r"\bout[ -]of[ -]hours\b",
    # German, Dutch
    r"notdienst", r"notfall", r"notaufnahme", r"rund um die uhr", r"spoed", r"\bwachtdienst\b", r"\bvan wacht\b",
    # Spanish, Portuguese, Italian
    r"\burgencias?\b", r"\bemergencias?\b", r"\bpronto[ -](?:socorro|atendimento)\b", r"\bplantao\b",
    r"\bpronto soccorso\b", r"\bemergenz[ae]\b", r"\burgenz[ae]\b",
    # Polish, Czech, Slovak, Hungarian, Romanian, Croatian, Serbian, Slovenian, Finnish, Scandinavian
    r"calodobow", r"pogotowi", r"\bpohotovost", r"ugyelet", r"\burgent[ae]\b", r"\bhitn[aeio]\b", r"\bdezur",
    r"paivystys", r"\bakut(?:t|en|klinik|mottak|mottagning)?\b", r"djurakut", r"dyreakut", r"\bdogn(?:apen|aben)\b",
    # Turkish, Vietnamese, Indonesian and Malay
    r"\bacil\b", r"\bnobetci\b", r"\bcap cuu\b", r"\bdarurat\b",
]
EMERGENCY_LITERALS = [
    # Russian, Ukrainian, Bulgarian, Serbian, Greek
    "круглосуточн", "неотложн", "скорая", "24 час", "цілодобов", "невідклад", "денонощ", "спешн", "дежурн", "хитн",
    "εφημερ", "επειγ", "24ωρ",
    # Japanese, Chinese, Korean
    "救急", "夜間", "24時間", "急诊", "急診", "急症", "24小时", "24小時", "응급", "24시",
    # Arabic, Persian, Hebrew, Thai, Hindi
    "طوارئ", "طوارىء", "اورژانس", "شبانه روزی", "חירום", "ฉุกเฉิน", "24 ชั่วโมง", "आपातकालीन",
]
_EMERGENCY_RE = re.compile("|".join(f"(?:{p})" for p in EMERGENCY_PATTERNS + [re.escape(fold(w)) for w in EMERGENCY_LITERALS]))
# titles that the law reserves to clinics with a 24-hour service
_EMERGENCY_BY_COUNTRY = {
    "FR": re.compile(r"\bchv\b"),  # centre hospitalier vétérinaire
    "DE": re.compile(r"tierklinik"),
    "AT": re.compile(r"tierklinik"),
}
# "non-stop" is around the clock in Central and Eastern Europe, but "no lunch break" where French is spoken
# ("ouverte NON STOP 8h-19h")
_NON_STOP = re.compile(r"\bnon ?-?stop\b")
_NON_STOP_NOT = {"FR", "BE", "LU", "MC", "CH", "CA"}


def emergency_name(name, country=None):
    """True when a vet's name says it takes emergencies or is open around the clock."""
    f = fold(name)
    if not f:
        return False
    if _EMERGENCY_RE.search(f):
        return True
    cc = (country or "").upper()
    if cc not in _NON_STOP_NOT and _NON_STOP.search(f):
        return True
    rule = _EMERGENCY_BY_COUNTRY.get(cc)
    return bool(rule and rule.search(f))


def atp_emergency(spider, name):
    rule = ATP_EMERGENCY_SPIDERS.get(spider)
    return rule is not None and not rule.search(name or "")


# ———————————————————————————— opening hours (OSM syntax → WeekHours)

_DAYS = {"mo": 0, "tu": 1, "we": 2, "th": 3, "fr": 4, "sa": 5, "su": 6}
_TOKEN = re.compile(
    r"""\s*(?:
        (?P<comment>"[^"]*")
      | (?P<always>24/7)
      | (?P<time>(?P<h1>\d{1,2})[:.hH](?P<m1>\d{2})\s*-\s*(?P<h2>\d{1,2})[:.hH](?P<m2>\d{2}))
      | (?P<days>(?P<d1>mo|tu|we|th|fr|sa|su)(?:\s*-\s*(?P<d2>mo|tu|we|th|fr|sa|su))?)(?![a-z])
      | (?P<hol>ph|sh)(?![a-z])
      | (?P<state>off|closed|open|unknown)(?![a-z])
      | (?P<comma>,)
      | (?P<colon>:)
    )""",
    re.IGNORECASE | re.VERBOSE,
)
_UNKNOWN = "unknown"


class _Unreadable(Exception):
    pass


def _tokens(rule):
    out, pos = [], 0
    rule = rule.rstrip()
    while pos < len(rule):
        m = _TOKEN.match(rule, pos)
        if not m or m.end() == pos:
            raise _Unreadable(rule[pos:])
        kind = m.lastgroup if m.lastgroup in ("comment", "always", "comma", "colon", "hol", "state") else (
            "time" if m.group("time") else "days")
        out.append((kind, m))
        pos = m.end()
    return out


def _subrules(tokens):
    """Splits a rule on its additional-rule commas (", We 14:00-18:00" after a time, a state or a comment)."""
    parts, cur = [], []
    for i, (kind, m) in enumerate(tokens):
        if kind == "comma" and cur and cur[-1][0] in ("time", "state", "comment", "always") and i + 1 < len(tokens) \
                and tokens[i + 1][0] in ("days", "hol"):
            parts.append(cur)
            cur = []
            continue
        cur.append((kind, m))
    if cur:
        parts.append(cur)
    return parts


def _minutes(h, m, close):
    h, m = int(h), int(m)
    if m > 59 or h > (48 if close else 23):
        raise _Unreadable(f"{h}:{m:02d}")
    return h * 60 + m


def add_interval(week, day, open_, close):
    """Adds [open, close) to a day, as stations.ts addInterval: past midnight, the rest goes to the next day.
    close == open is the whole day. A day still unknown (None) is left unknown."""
    def push(d, o, c):
        i = d % 7
        if week[i] is not None and c > o:
            week[i].append([o, c])
    if close == open_ or (open_ == 0 and close >= 1439):
        push(day, 0, 1440)
    elif close > open_:
        push(day, open_, 1440 if close == 1439 else close)
    else:
        push(day, open_, 1440)
        push(day + 1, 0, close)


def tidy_week(week):
    """Intervals sorted and merged; a week without a single known day is None (stations.ts tidyWeek)."""
    if all(d is None for d in week):
        return None
    out = []
    for d in week:
        if d is None:
            out.append(None)
            continue
        merged = []
        for o, c in sorted(d):
            if merged and o <= merged[-1][1]:
                merged[-1][1] = max(merged[-1][1], c)
            else:
                merged.append([o, c])
        out.append(merged)
    return out


def always_open(week):
    return bool(week) and len(week) == 7 and all(d and len(d) == 1 and d[0][0] == 0 and d[0][1] >= 1440 for d in week)


def parse_opening_hours(text):
    """OSM opening_hours → (WeekHours, None), or (None, text) when it cannot be read (months, dates, weeks, sunrise,
    "+", fallback rules, anything unknown). Handles 24/7, weekday ranges and lists, several time spans, off/closed,
    comments, ';' (a normal rule replaces the days it names) and ',' (an additional rule adds to them); PH and SH
    rules are ignored; the days no rule names are closed when at least one rule names weekdays."""
    if not isinstance(text, str) or not text.strip():
        return None, None
    src = " ".join(text.split())
    keep = src[:MAX_HOURS_TEXT]
    try:
        s = re.sub(r"[‐-―−]", "-", src)
        if s.count('"') % 2 or "||" in s:
            raise _Unreadable(s)
        s = re.sub(r'"[^"]*"', '"#"', s)  # comments out of the way of the ';' split
        own = [None] * 7  # None: no rule named the day yet
        named, weekday_rule = set(), False
        for rule in s.split(";"):
            if not rule.strip():
                continue
            for n, part in enumerate(_subrules(_tokens(rule))):
                days, hol, times, state, comment, always = [], False, [], None, False, False
                kinds = [k for k, _ in part]
                if any(a == "hol" and b == "days" for a, b in zip(kinds, kinds[1:])):
                    continue  # "SH Mo-Fr 09:00-12:00": only during the holidays, ignored ("Su,PH off" is a list)
                for kind, m in part:
                    if kind == "days":
                        a = _DAYS[m.group("d1").lower()]
                        b = _DAYS[m.group("d2").lower()] if m.group("d2") else a
                        days.extend((a + k) % 7 for k in range(((b - a) % 7) + 1))
                    elif kind == "hol":
                        hol = True
                    elif kind == "time":
                        o = _minutes(m.group("h1"), m.group("m1"), False)
                        c = _minutes(m.group("h2"), m.group("m2"), True)
                        if c > 1440:  # 18:00-26:00: past midnight written as hours after 24
                            c -= 1440
                            if c >= o:
                                raise _Unreadable(m.group(0))
                        times.append((o, c))
                    elif kind == "state":
                        state = m.group("state").lower()
                    elif kind == "comment":
                        comment = True
                    elif kind == "always":
                        always = True
                if hol and not days:
                    continue  # public or school holidays only: ignored
                if comment and not (days or times or state or always):
                    continue  # a note after the rules ('Mo-Fr 08:00-18:00; "by appointment"'); alone, it reads nothing
                if always:
                    value = [(0, 1440)]
                elif state in ("off", "closed"):
                    value = []
                elif times:
                    value = times
                elif state == "open":
                    value = [(0, 1440)]
                elif state == "unknown" or comment:
                    value = _UNKNOWN
                else:
                    raise _Unreadable(rule)  # days without times
                selected = sorted(set(days)) if days else list(range(7))
                weekday_rule = weekday_rule or bool(days)
                for d in selected:
                    if n == 0 or own[d] is None or own[d] == _UNKNOWN or value == _UNKNOWN or value == []:
                        own[d] = list(value) if value != _UNKNOWN else _UNKNOWN
                    else:
                        own[d] = own[d] + list(value)
                named.update(selected)
        if not named:
            return None, keep
        week = []
        for d in range(7):
            if d not in named:
                week.append([] if weekday_rule else None)
            else:
                week.append(None if own[d] == _UNKNOWN else [])
        for d in range(7):
            if d in named and own[d] not in (None, _UNKNOWN):
                for o, c in own[d]:
                    add_interval(week, d, o, c)
        hours = tidy_week(week)
        return (hours, None) if hours is not None else (None, keep)
    except _Unreadable:
        return None, keep


# ———————————————————————————— phones, websites, names

def format_phone(raw, country):
    """(display, key): the international format and the E.164 key, or the text as written and its digits when the
    number cannot be read."""
    s = norm(raw)
    if not s:
        return None, None
    if phonenumbers is not None:
        try:
            p = phonenumbers.parse(s, (country or "").upper() or None)
            if phonenumbers.is_possible_number(p):
                return (phonenumbers.format_number(p, phonenumbers.PhoneNumberFormat.INTERNATIONAL),
                        phonenumbers.format_number(p, phonenumbers.PhoneNumberFormat.E164))
        except phonenumbers.NumberParseException:
            pass
    digits = re.sub(r"\D", "", s)
    return s, (digits if len(digits) >= 6 else None)


def split_phones(value):
    """An OSM or All the Places phone value: several numbers separated by ';' or ','."""
    if not value:
        return []
    return [p for p in re.split(r"\s*[;,]\s*", str(value)) if p.strip()]


def site_domain(url):
    """Host of a website without www., or None for a shared host (social networks, directories)."""
    if not isinstance(url, str) or not url.strip():
        return None
    u = url.strip()
    if "://" not in u:
        u = "http://" + u
    try:
        host = urllib.parse.urlsplit(u).hostname
    except ValueError:
        return None
    if not host:
        return None
    host = host.lower().rstrip(".")
    for prefix in ("www.", "www2.", "m."):
        if host.startswith(prefix):
            host = host[len(prefix):]
    if any(host == g or host.endswith("." + g) for g in GENERIC_HOSTS):
        return None
    return host


NAME_STOPWORDS = {fold(w) for w in """
    clinique clinic clinics clinica clinicas klinik kliniek klinika veterinaire veterinaires veterinary veterinarian
    veterinarians veterinaria veterinarias veterinario veterinarios veterinari veterinarski weterynaryjna weterynaryjny
    weterynarz lecznica gabinet tierarzt tierarztin tierarzte tierarztliche tierarztpraxis tierklinik kleintierpraxis
    kleintierklinik praxis dierenarts dierenartsen dierenartspraktijk dierenkliniek dyreklinik dyrlaege djurklinik
    djursjukhus vet vets veto vetos cabinet centre center centro hospital hospitals hopital ospedale ambulatorio animal
    animals animaux animali animales tiere pet pets care the les des and und der die das von van della dei del sas sarl
    selarl selas scp ltd llc inc gmbh srl group groupe medical medicine surgery practice
""".split()}


def name_tokens(name):
    return {t for t in re.findall(r"\w+", fold(name)) if len(t) >= 3 and not t.isdigit() and t not in NAME_STOPWORDS}


def similar_names(a, b):
    """Alike once the vet words are gone; a missing or generic name ("Clinique vétérinaire") is alike any other."""
    ta, tb = name_tokens(a), name_tokens(b)
    if not ta or not tb or ta <= tb or tb <= ta:
        return True
    if len(ta & tb) / len(ta | tb) >= 0.5:
        return True
    return SequenceMatcher(None, " ".join(sorted(ta)), " ".join(sorted(tb))).ratio() >= 0.85


# ———————————————————————————— geography

def haversine_m(lat1, lng1, lat2, lng2):
    r = 6371000.0
    p1, p2 = math.radians(lat1), math.radians(lat2)
    dp, dl = p2 - p1, math.radians(lng2 - lng1)
    a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
    return 2 * r * math.asin(min(1.0, math.sqrt(a)))


class Grid:
    """Vets in cells of `cell` degrees, for the lookups within a few hundred metres."""

    def __init__(self, cell=0.01):
        self.cell = cell
        self.cells = defaultdict(list)

    def add(self, v):
        self.cells[(math.floor(v.lat / self.cell), math.floor(v.lng / self.cell))].append(v)

    def near(self, lat, lng, radius):
        dlat = radius / 111320.0
        dlng = radius / (111320.0 * max(0.02, math.cos(math.radians(lat))))
        c = self.cell
        for y in range(math.floor((lat - dlat) / c), math.floor((lat + dlat) / c) + 1):
            for x in range(math.floor((lng - dlng) / c), math.floor((lng + dlng) / c) + 1):
                for v in self.cells.get((y, x), ()):
                    d = haversine_m(lat, lng, v.lat, v.lng)
                    if d <= radius:
                        yield d, v


# ———————————————————————————— vets

class Vet:
    __slots__ = ("id", "lat", "lng", "name", "names", "raw_phones", "phones", "keys", "website", "domain", "email",
                 "address", "city", "postcode", "country", "region", "hours", "hours_text", "hours_rank", "flags",
                 "src", "osm", "tz", "rank", "spider")

    def __init__(self, id, lat, lng, src, name=None, phones=(), website=None, email=None, address=None, city=None,
                 postcode=None, country=None, region=None, flags=0, osm=None, rank=(0, 0.0, 0), spider=None):
        self.id, self.lat, self.lng, self.src = id, float(lat), float(lng), src
        self.name = norm(name)
        self.names = {self.name} if self.name else set()
        self.raw_phones = [p for p in phones if norm(p)]
        self.phones, self.keys = [], set()
        self.website = norm(website)
        self.domain = site_domain(self.website)
        self.email = norm(email)
        self.address, self.city, self.postcode = norm(address), norm(city), norm(postcode)
        c = norm(country)
        self.country = c.upper() if c and len(c) == 2 and c.isalpha() else None
        self.region = norm(region)
        self.hours, self.hours_text, self.hours_rank = None, None, 0
        self.flags, self.osm, self.tz, self.rank, self.spider = flags, osm, None, rank, spider

    def set_hours(self, text, from_atp):
        """Hours from a source: All the Places before OSM, read hours before a text that could not be read."""
        hours, unread = parse_opening_hours(text)
        if hours is None and unread is None:
            return
        rank = (4 if from_atp else 3) if hours is not None else (2 if from_atp else 1)
        if rank > self.hours_rank:
            self.hours, self.hours_text, self.hours_rank = hours, unread, rank

    def finish_phones(self):
        """Formats the phones once the country is known: international format, deduped, 3 at most."""
        for raw in self.raw_phones:
            display, key = format_phone(raw, self.country)
            if display and (key is None or key not in self.keys) and display not in self.phones and len(self.phones) < MAX_PHONES:
                self.phones.append(display)
                if key:
                    self.keys.add(key)
        self.raw_phones = []

    def row(self):
        return [self.id, round(self.lat, 6), round(self.lng, 6), self.name, self.phones, self.website, self.email,
                self.address, self.city, self.postcode, self.country, self.region, self.hours, self.hours_text,
                self.flags, self.src, self.osm, self.tz]


def merge(keep, other):
    """`keep` stays (its id, its place) and takes what it lacked from `other`."""
    if other.name and (not keep.name or (not looks_like_clinic(keep.name) and looks_like_clinic(other.name))):
        keep.name = other.name  # a clinic's name rather than one of its vets'
    keep.names |= other.names
    for display in other.phones:
        _, key = format_phone(display, keep.country or other.country)
        if (key is None or key not in keep.keys) and display not in keep.phones and len(keep.phones) < MAX_PHONES:
            keep.phones.append(display)
    keep.keys |= other.keys
    for f in ("website", "domain", "email", "address", "city", "postcode", "country", "region", "osm"):
        if not getattr(keep, f) and getattr(other, f):
            setattr(keep, f, getattr(other, f))
    if other.hours_rank > keep.hours_rank:
        keep.hours, keep.hours_text, keep.hours_rank = other.hours, other.hours_text, other.hours_rank
    keep.flags |= other.flags
    keep.src |= other.src


class Pool:
    """The vets kept so far, indexed by place, phone and website."""

    def __init__(self, vets=()):
        self.vets, self.grid, self.by_key, self.by_domain = [], Grid(), defaultdict(list), defaultdict(list)
        for v in vets:
            self.add(v)

    def add(self, v):
        self.vets.append(v)
        self.grid.add(v)
        self.index(v, v)

    def index(self, v, under):
        for k in v.keys:
            if under not in self.by_key[k]:
                self.by_key[k].append(under)
        if v.domain and under not in self.by_domain[v.domain]:
            self.by_domain[v.domain].append(under)

    def _nearest(self, v, candidates, radius):
        best = None
        for o in candidates:
            if o is v:
                continue
            d = haversine_m(v.lat, v.lng, o.lat, o.lng)
            if d <= radius and (best is None or d < best[0]):
                best = (d, o)
        return best[1] if best else None

    @staticmethod
    def _alike(v, o):
        return any(similar_names(v.name, n) for n in (o.names or {None}))

    def duplicate_of(self, v):
        """Inside Overture: ≤ 25 m whatever the names, ≤ 60 m with alike names, the same phone ≤ 500 m."""
        best = None
        for d, o in self.grid.near(v.lat, v.lng, DEDUPE_NAME_M):
            if (d <= DEDUPE_ANY_M or self._alike(v, o)) and (best is None or d < best[0]):
                best = (d, o)
        if best:
            return best[1], "place"
        o = self._nearest(v, (o for k in v.keys for o in self.by_key.get(k, ())), DEDUPE_PHONE_M)
        return (o, "phone") if o else (None, None)

    def match(self, v):
        """Another source's vet: the same phone ≤ 1 km, the same website ≤ 300 m, alike names ≤ 100 m (≤ 25 m, any)."""
        o = self._nearest(v, (o for k in v.keys for o in self.by_key.get(k, ())), MATCH_PHONE_M)
        if o:
            return o, "phone"
        if v.domain:
            o = self._nearest(v, self.by_domain.get(v.domain, ()), MATCH_DOMAIN_M)
            if o:
                return o, "website"
        best = None
        for d, o in self.grid.near(v.lat, v.lng, MATCH_NAME_M):
            if (d <= MATCH_ANY_M or self._alike(v, o)) and (best is None or d < best[0]):
                best = (d, o)
        return (best[1], "place") if best else (None, None)

    def absorb(self, keep, other):
        merge(keep, other)
        self.index(other, keep)


def dedupe(vets):
    """Overture's duplicates (several providers, a clinic and each of its vets): the most reliable row is kept."""
    pool, how = Pool(), Counter()
    for v in sorted(vets, key=lambda v: v.rank):
        dup, why = pool.duplicate_of(v)
        if dup is not None:
            pool.absorb(dup, v)
            how[why] += 1
        else:
            pool.add(v)
    return pool, how


def integrate(pool, newcomers):
    """All the Places or OSM vets: merged into the row they match, else added (later ones can match them)."""
    how = Counter()
    for v in sorted(newcomers, key=lambda v: v.rank):
        o, why = pool.match(v)
        if o is not None:
            pool.absorb(o, v)
            how[why] += 1
        else:
            pool.add(v)
            how["added"] += 1
    return how


# ———————————————————————————— logging and downloads

T0 = time.time()


def log(msg):
    print(f"[{time.time() - T0:6.0f} s] {msg}", file=sys.stderr, flush=True)


def http_get(url, data=None, timeout=120, accept=None):
    headers = {"User-Agent": USER_AGENT}
    if accept:
        headers["Accept"] = accept
    body = urllib.parse.urlencode(data).encode() if data is not None else None
    req = urllib.request.Request(url, data=body, headers=headers)
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read()


def write_atomic(path, payload):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    tmp = path + ".part"
    with open(tmp, "wb") as f:
        f.write(payload)
    os.replace(tmp, path)


def cached_get(path, url, **kw):
    if os.path.exists(path):
        with open(path, "rb") as f:
            return f.read()
    payload = http_get(url, **kw)
    write_atomic(path, payload)
    return payload


def in_bbox(lat, lng, bbox):
    return bbox is None or (bbox[0] <= lng <= bbox[2] and bbox[1] <= lat <= bbox[3])


# ———————————————————————————— Overture

def read_overture(release, bbox, cache):
    """Overture's vets (the extract is cached), then the rows kept, and every row with a country as the reference
    for the countries of the other sources."""
    import duckdb

    tag = "" if bbox is None else "-" + "_".join(f"{v:g}" for v in bbox)
    path = os.path.join(cache, "overture", f"vets-{release}{tag}.parquet")
    con = duckdb.connect()
    if not os.path.exists(path):
        log(f"Overture {release}: reading the places theme on S3 (a few minutes for the world)")
        con.execute("INSTALL httpfs; LOAD httpfs; SET s3_region='us-west-2'; INSTALL spatial; LOAD spatial; SET threads TO 32;")
        where = "list_contains(taxonomy.hierarchy, 'veterinary_care')"
        if bbox:
            where += f" AND bbox.xmin BETWEEN {bbox[0]} AND {bbox[2]} AND bbox.ymin BETWEEN {bbox[1]} AND {bbox[3]}"
        os.makedirs(os.path.dirname(path), exist_ok=True)
        con.execute(f"""COPY (
          SELECT id, ST_Y(geometry) AS lat, ST_X(geometry) AS lng, confidence, operating_status,
                 taxonomy.primary AS category, names.primary AS name, phones,
                 websites[1] AS website, emails[1] AS email,
                 addresses[1].freeform AS address, addresses[1].locality AS city, addresses[1].postcode AS postcode,
                 addresses[1].country AS country, addresses[1].region AS region
          FROM read_parquet('{OVERTURE_BASE.format(release=release)}', hive_partitioning=1)
          WHERE {where}
        ) TO '{path}.part' (FORMAT parquet, COMPRESSION zstd)""")
        os.replace(path + ".part", path)
    rows = con.execute(f"""
      SELECT id, lat, lng, confidence, category, name, phones, website, email, address, city, postcode, country, region
      FROM read_parquet('{path}')
      WHERE category NOT IN {EXCLUDED_CATEGORIES}
        AND coalesce(operating_status, 'open') <> 'permanently_closed'
        AND (confidence >= {MIN_CONFIDENCE} OR (confidence >= {MIN_CONFIDENCE_WITH_PHONE} AND len(coalesce(phones, [])) > 0))
    """).fetchall()
    reference = con.execute(f"SELECT lat, lng, upper(country) FROM read_parquet('{path}') WHERE length(country) = 2").fetchall()
    vets, left_out = [], 0
    for (pid, lat, lng, conf, cat, name, phones, website, email, address, city, postcode, country, region) in rows:
        if lat is None or lng is None or not in_bbox(lat, lng, bbox):
            continue
        if not_a_clinic(name):
            left_out += 1
            continue
        conf = conf or 0.0
        filled = sum(bool(x) for x in (phones, website, email, address))
        v = Vet(f"o:{pid}", lat, lng, SRC_OVERTURE, name=name, phones=phones or (), website=website, email=email,
                address=address, city=city, postcode=postcode, country=country, region=region,
                flags=FLAG_EMERGENCY if cat == "emergency_pet_hospital" else 0,
                rank=(0 if conf >= MIN_CONFIDENCE else 1, -conf, -filled))
        vets.append(v)
    log(f"Overture: {len(vets)} vets read ({left_out} head offices, associations, pharmacies or labs left out)")
    return vets, reference


# ———————————————————————————— All the Places

def atp_name(p):
    name, branch, brand = norm(p.get("name")), norm(p.get("branch")), norm(p.get("brand"))
    if branch:
        if name and name.lower() not in branch.lower():
            return f"{name} {branch}"
        return f"{brand} {branch}" if brand and brand.lower() not in branch.lower() and not looks_like_clinic(branch) else branch
    if name and brand and brand.lower() not in name.lower() and not looks_like_clinic(name):
        return f"{brand} {name}"  # Banfield's "Lakewood, CO"
    return name or brand


def atp_vet(spider, feature, n):
    p = feature.get("properties") or {}
    if p.get("amenity") != "veterinary":
        return None
    geom = feature.get("geometry") or {}
    if geom.get("type") != "Point":
        return None
    lng, lat = geom["coordinates"][:2]
    ref = norm(p.get("ref"))
    key = ref if ref and len(ref) <= 40 and not re.search(r"[/\s:]", ref) else (feature.get("id") or ref or str(n)).rstrip("=")
    name = atp_name(p)
    street = norm(p.get("addr:street_address")) or " ".join(x for x in (norm(p.get("addr:housenumber")), norm(p.get("addr:street"))) if x) or None
    v = Vet(f"atp:{spider}:{key}", lat, lng, SRC_ATP, name=name, phones=split_phones(p.get("phone")),
            website=p.get("website"), email=p.get("email"), address=street, city=p.get("addr:city"),
            postcode=p.get("addr:postcode"), country=p.get("addr:country"), region=p.get("addr:state"),
            flags=FLAG_EMERGENCY if atp_emergency(spider, name) else 0, rank=(0, 0.0, 0), spider=spider)
    v.set_hours(p.get("opening_hours"), True)
    return v


def fetch_atp(cache, bbox):
    """The vets of All the Places' latest run (the run's index is cached: a rerun keeps the same run)."""
    idx = json.loads(cached_get(os.path.join(cache, "atp", "latest.json"), ATP_INDEX))
    run = idx["run_id"]
    base = idx["output_url"][: -len(".zip")] if idx["output_url"].endswith(".zip") else idx["output_url"]
    rdir = os.path.join(cache, "atp", run)
    try:
        results = json.loads(cached_get(os.path.join(rdir, "_results.json"), idx["stats_url"]))
        in_run = {r["spider"] for r in results.get("results", []) if r.get("features")}
    except (urllib.error.URLError, OSError, ValueError, KeyError) as e:
        log(f"All the Places: no stats for run {run} ({e}); reading the known spiders")
        in_run = set(ATP_SPIDERS)
    spiders = sorted(s for s in in_run if s in ATP_SPIDERS or ATP_DISCOVER.search(s))
    missing = sorted(set(ATP_SPIDERS) - in_run)
    if missing:
        log(f"All the Places: not in run {run} (or empty): {', '.join(missing)}")
    vets, per_spider = [], {}
    for s in spiders:
        try:
            data = json.loads(cached_get(os.path.join(rdir, f"{s}.geojson"), f"{base}/{s}.geojson"))
        except (urllib.error.URLError, OSError, ValueError) as e:
            log(f"All the Places: {s} could not be read ({e})")
            continue
        found = [v for n, f in enumerate(data.get("features", [])) if (v := atp_vet(s, f, n)) and in_bbox(v.lat, v.lng, bbox)]
        if found:
            per_spider[s] = len(found)
        vets.extend(found)
    log(f"All the Places run {run}: {len(vets)} vets from {len(per_spider)} spiders {per_spider}")
    return run, vets


# ———————————————————————————— OpenStreetMap

NUMBER_FIRST = {"FR", "LU", "MC", "BE", "US", "CA", "GB", "IE", "AU", "NZ", "ZA", "IN", "PH", "MY", "SG", "IL", "TH"}


def osm_vet(el):
    t = el.get("tags") or {}
    if "lat" in el:
        lat, lng = el["lat"], el["lon"]
    elif el.get("center"):
        lat, lng = el["center"]["lat"], el["center"]["lon"]
    else:
        return None
    kind = {"node": "n", "way": "w", "relation": "r"}.get(el.get("type"))
    if not kind:
        return None
    country = t.get("addr:country")
    street = norm(t.get("addr:street")) or norm(t.get("addr:place"))
    number = norm(t.get("addr:housenumber"))
    if norm(t.get("addr:full")):
        address = norm(t.get("addr:full"))
    elif street and number:
        address = f"{number} {street}" if (country or "").upper() in NUMBER_FIRST else f"{street} {number}"
    else:
        address = street
    phones = []  # not emergency:phone, often a regional service's number rather than the clinic's
    for k in ("phone", "contact:phone", "contact:mobile", "mobile", "phone:mobile"):
        phones.extend(split_phones(t.get(k)))
    v = Vet(f"osm:{kind}{el['id']}", lat, lng, SRC_OSM,
            name=t.get("name") or t.get("official_name") or t.get("brand") or t.get("operator"), phones=phones,
            website=t.get("website") or t.get("contact:website") or t.get("url"),
            email=t.get("email") or t.get("contact:email"), address=address,
            city=t.get("addr:city") or t.get("addr:town") or t.get("addr:village"), postcode=t.get("addr:postcode"),
            country=country, region=t.get("addr:state") or t.get("addr:province"),
            flags=FLAG_EMERGENCY if t.get("emergency") == "yes" else 0, osm=f"{el['type']}/{el['id']}",
            rank=(0, 0.0, -sum(bool(t.get(k)) for k in ("name", "phone", "opening_hours", "website"))))
    v.set_hours(t.get("opening_hours"), False)
    return v


def world_cells(bbox):
    """Slices of 30° (15° where vets are dense), clipped to the bbox."""
    dense = {(30, -30), (30, 0), (30, 30), (30, -90), (30, -120)}
    cells = []
    for s in range(-60, 90, 30):
        for w in range(-180, 180, 30):
            parts = [(s, w, s + 30, w + 30)]
            if (s, w) in dense:
                parts = [(a, b, a + 15, b + 15) for a in (s, s + 15) for b in (w, w + 15)]
            for (cs, cw, cn, ce) in parts:
                if bbox:
                    cs, cw, cn, ce = max(cs, bbox[1]), max(cw, bbox[0]), min(cn, bbox[3]), min(ce, bbox[2])
                    if cs >= cn or cw >= ce:
                        continue
                cells.append((cs, cw, cn, ce))
    return cells


def _cell_name(c):
    return "_".join(f"{v:g}" for v in c)


def _overpass_status():
    """'free' once a slot is free (Overpass gives each address a couple of them), 'down' when its status page does
    not answer."""
    for _ in range(10):
        try:
            status = http_get(OVERPASS_STATUS, timeout=45).decode("utf-8", "replace")
        except (urllib.error.URLError, OSError):
            return "down"
        free = re.search(r"(\d+) slots? available now", status)
        if free and int(free.group(1)) > 0:
            return "free"
        waits = [int(x) for x in re.findall(r"in (\d+) seconds", status)]
        if not free and not waits:
            return "down"  # not a status page
        time.sleep(min(120, min(waits) + 2) if waits else 15)
    return "free"


def overpass_cell(cell, cache, check_status=True):
    """One slice from Overpass: (data, None, None), or from the cache (data, 'cache' | 'QLever cache', None) (a slice
    once read from QLever stays so: delete its qlever-<slice>.json to try Overpass again), or (None, None, 'cell') when
    the slice is too much for Overpass (timeout, memory), or (None, None, 'server') when Overpass does not answer."""
    path = os.path.join(cache, "osm", f"overpass-{_cell_name(cell)}.json")
    for cached, label in ((path, "cache"), (os.path.join(cache, "osm", f"qlever-{_cell_name(cell)}.json"), "QLever cache")):
        if os.path.exists(cached):
            with open(cached, "rb") as f:
                return json.loads(f.read()), label, None
    if check_status and _overpass_status() == "down":  # after a failure: is a slot free, is Overpass up?
        return None, None, "server"
    s, w, n, e = cell
    query = f'[out:json][timeout:{OVERPASS_TIMEOUT_S}];nwr["amenity"="veterinary"]({s},{w},{n},{e});out tags center;'
    try:
        payload = http_get(OVERPASS_URL, data={"data": query}, timeout=OVERPASS_TIMEOUT_S + 90)
        data = json.loads(payload)
    except urllib.error.HTTPError as err:
        log(f"Overpass {_cell_name(cell)}: HTTP {err.code}")
        if err.code == 429:
            time.sleep(60)
        return None, None, "cell" if err.code == 400 else "server"
    except (urllib.error.URLError, OSError, ValueError) as err:
        log(f"Overpass {_cell_name(cell)}: {err}")
        return None, None, "server"
    remark = data.get("remark") or ""
    if "error" in remark.lower() or "timed out" in remark.lower():
        log(f"Overpass {_cell_name(cell)}: {remark.strip()[:120]}")
        return None, None, "cell"
    write_atomic(path, payload)
    return data, None, None


def qlever_vets(cache):
    """Every OSM vet from QLever's planet (tags, then geometries), the fallback when Overpass refuses slices."""
    def literal(v):
        if v.startswith('"'):
            v = v[1:v.rfind('"')] if v.rfind('"') > 0 else v[1:]
            return v.replace('\\"', '"').replace("\\t", "\t").replace("\\n", "\n").replace("\\\\", "\\")
        return v
    prefix = "PREFIX osmkey: <https://www.openstreetmap.org/wiki/Key:>\nPREFIX geo: <http://www.opengis.net/ont/geosparql#>\n"
    tags_q = prefix + 'SELECT ?osm ?k ?v WHERE { ?osm osmkey:amenity "veterinary" . ?osm ?k ?v . FILTER(STRSTARTS(STR(?k), "https://www.openstreetmap.org/wiki/Key:")) }'
    geom_q = prefix + 'SELECT ?osm ?wkt WHERE { ?osm osmkey:amenity "veterinary" . ?osm geo:hasGeometry/geo:asWKT ?wkt }'
    tsv_tags = cached_get(os.path.join(cache, "osm", "qlever-tags.tsv"), QLEVER_URL, data={"query": tags_q},
                          timeout=900, accept="text/tab-separated-values").decode("utf-8", "replace")
    tsv_geom = cached_get(os.path.join(cache, "osm", "qlever-geometries.tsv"), QLEVER_URL, data={"query": geom_q},
                          timeout=900, accept="text/tab-separated-values").decode("utf-8", "replace")
    elements = {}
    for line in tsv_tags.splitlines()[1:]:
        parts = line.split("\t")
        if len(parts) < 3:
            continue
        m = re.search(r"/(node|way|relation)/(\d+)>", parts[0])
        if not m:
            continue
        el = elements.setdefault((m.group(1), int(m.group(2))), {"type": m.group(1), "id": int(m.group(2)), "tags": {}})
        el["tags"][parts[1].strip("<>").rsplit("Key:", 1)[-1]] = literal(parts[2])
    for line in tsv_geom.splitlines()[1:]:
        parts = line.split("\t")
        m = re.search(r"/(node|way|relation)/(\d+)>", parts[0]) if len(parts) >= 2 else None
        el = elements.get((m.group(1), int(m.group(2)))) if m else None
        if el is None:
            continue
        coords = [(float(x), float(y)) for x, y in re.findall(r"(-?\d+(?:\.\d+)?) (-?\d+(?:\.\d+)?)", parts[1])]
        if coords:
            el["center"] = {"lon": sum(c[0] for c in coords) / len(coords), "lat": sum(c[1] for c in coords) / len(coords)}
    try:
        stats = json.loads(cached_get(os.path.join(cache, "osm", "qlever-stats.json"), QLEVER_URL + "?cmd=stats", timeout=60))
        m = re.search(r"planet-(\d{2})(\d{2})(\d{2})", stats.get("name-index", ""))
        date = f"20{m.group(1)}-{m.group(2)}-{m.group(3)}" if m else "unknown"
    except (urllib.error.URLError, OSError, ValueError):
        date = "unknown"
    return [el for el in elements.values() if "center" in el], date


def fetch_osm(cache, bbox):
    """Every amenity=veterinary of OSM (or of the bbox) through Overpass, slice after slice; the date of the extract;
    the slices that could not be read at all. A slice too much for Overpass is split in four; when Overpass does not
    answer for OVERPASS_DOWN_S, the slices left are read from QLever's planet instead."""
    queue, elements, dates, missing, sources = list(world_cells(bbox)), {}, [], [], Counter()
    deadline, down_since, backoff = time.time() + OVERPASS_MAX_SECONDS, None, 30
    log(f"OSM: {len(queue)} Overpass slices to read")
    while queue:
        if time.time() > deadline:
            log("OSM: Overpass is taking too long: the slices left go to QLever")
            missing.extend(queue)
            break
        cell = queue[0]
        data, from_cache, failure = overpass_cell(cell, cache, check_status=down_since is not None or not sources)
        if failure == "server":  # at capacity or down: exponential backoff, as Overpass asks
            down_since = down_since or time.time()
            if time.time() - down_since > OVERPASS_DOWN_S:
                log(f"OSM: Overpass has not answered for {OVERPASS_DOWN_S // 60} min: the slices left go to QLever")
                missing.extend(queue)
                break
            time.sleep(backoff)
            backoff = min(backoff * 2, 300)
            continue
        down_since, backoff = None, 30
        queue.pop(0)
        if failure == "cell":
            s, w, n, e = cell
            if n - s > OVERPASS_MIN_CELL_DEG:
                ms, mw = (s + n) / 2, (w + e) / 2
                queue[:0] = [(s, w, ms, mw), (s, mw, ms, e), (ms, w, n, mw), (ms, mw, n, e)]
                log(f"OSM: slice {_cell_name(cell)} split in four")
            else:
                missing.append(cell)
            time.sleep(OVERPASS_PAUSE_S)
            continue
        sources["qlever" if from_cache == "QLever cache" else "overpass"] += 1
        stamp = (data.get("osm3s") or {}).get("timestamp_osm_base")
        if stamp:
            dates.append(stamp)
        for el in data.get("elements", []):
            elements[(el.get("type"), el.get("id"))] = el
        log(f"OSM: slice {_cell_name(cell)}: {len(data.get('elements', []))} vets{f' ({from_cache})' if from_cache else ''}, {len(queue)} slices left")
        if not from_cache and queue:
            time.sleep(OVERPASS_PAUSE_S)
    if missing:
        log(f"OSM: {len(missing)} slices refused by Overpass, read from QLever: {', '.join(map(_cell_name, missing))}")
        try:
            extra, qdate = qlever_vets(cache)
            for (s, w, n, e) in missing:
                inside = [el for el in extra if s <= el["center"]["lat"] <= n and w <= el["center"]["lon"] <= e]
                write_atomic(os.path.join(cache, "osm", f"qlever-{_cell_name((s, w, n, e))}.json"),
                             json.dumps({"generator": "QLever osm-planet (fallback of build_vets.py)",
                                         "osm3s": {"timestamp_osm_base": qdate}, "elements": inside}).encode())
                for el in inside:
                    elements.setdefault((el["type"], el["id"]), el)
                sources["qlever"] += 1
            dates.append(qdate)
            missing = []
        except (urllib.error.URLError, OSError, ValueError) as e:
            log(f"OSM: QLever failed too ({e}); these slices are missing from the base")
    osm_date = min(dates) if dates else None
    vets = [v for el in elements.values() if (v := osm_vet(el)) and in_bbox(v.lat, v.lng, bbox)]
    log(f"OSM: {len(vets)} vets, slices {dict(sources)} (extract of {osm_date})")
    return osm_date, vets, missing, dict(sources)


# ———————————————————————————— countries, time zones, flags

def assign_countries(vets, reference):
    """A vet without a country takes the nearest Overture vet's (within ~3°)."""
    grid = defaultdict(list)
    for lat, lng, c in reference:
        grid[(math.floor(lat), math.floor(lng))].append((lat, lng, c))
    done = 0
    for v in vets:
        if v.country:
            continue
        cy, cx = math.floor(v.lat), math.floor(v.lng)
        best = None
        for r in range(0, 4):
            for y in range(cy - r, cy + r + 1):
                for x in range(cx - r, cx + r + 1):
                    if max(abs(y - cy), abs(x - cx)) != r:
                        continue
                    for lat, lng, c in grid.get((y, x), ()):
                        d = haversine_m(v.lat, v.lng, lat, lng)
                        if best is None or d < best[0]:
                            best = (d, c)
            if best:
                break
        if best:
            v.country = best[1]
            done += 1
    return done


def finalize(vets):
    from timezonefinder import TimezoneFinder

    tf = TimezoneFinder(in_memory=True)
    for v in vets:
        if any(emergency_name(n, v.country) for n in v.names):
            v.flags |= FLAG_EMERGENCY
        if v.hours is not None and always_open(v.hours):
            v.flags |= FLAG_OPEN24 | FLAG_EMERGENCY
        try:
            v.tz = tf.timezone_at(lng=v.lng, lat=v.lat)
        except ValueError:
            v.tz = None


# ———————————————————————————— output

def attribution(release, osm_date, atp_run):
    return [
        f"© OpenStreetMap contributors, ODbL 1.0 (extract of {osm_date}): https://www.openstreetmap.org/copyright",
        f"Overture Maps Foundation, places theme {release}: Meta, Microsoft, BrightQuery, Uberall, PinMeTo, DAC and "
        "others (CDLA-Permissive-2.0), Foursquare (Apache-2.0), All the Places (CC0-1.0): https://docs.overturemaps.org/attribution/",
        f"All the Places, run {atp_run} (CC0-1.0): https://www.alltheplaces.xyz/",
    ]


README = """# Véto Garde: world base of veterinarians ({release})

The veterinarians of the whole world, one row per clinic, as the Véto Garde app uses them: name, phone numbers,
website, email, address, opening hours, emergency service and open 24/7 flags, time zone. Built on {built} by
`build_vets.py` (this folder) from three open sources. It contains information from OpenStreetMap, which is made
available here under the Open Database License (ODbL 1.0): this whole base is published under the same licence,
https://opendatacommons.org/licenses/odbl/1-0/.

## Sources and their licences

- **Overture Maps Foundation**, places theme release `{release}` (https://docs.overturemaps.org/attribution/):
  CDLA-Permissive-2.0 (Meta, Microsoft, BrightQuery, Uberall, PinMeTo, DAC and others), Apache-2.0 (Foursquare:
  "Copyright 2024 Foursquare Labs, Inc. All rights reserved", data transformed to the Overture schema),
  CC0-1.0 (All the Places).
- **OpenStreetMap**, `amenity=veterinary` {osm_route} (extract of {osm}):
  © OpenStreetMap contributors, ODbL 1.0, https://www.openstreetmap.org/copyright.
- **All the Places**, run `{atp}` (https://www.alltheplaces.xyz/): CC0-1.0.

## File

`vets-{release}.json.gz`: `{{licence, attribution, columns, rows}}`, {count} rows. Columns:

| column | content |
|---|---|
| id | `o:<Overture id>`, `osm:n<id>` / `osm:w<id>` / `osm:r<id>`, or `atp:<spider>:<ref>` |
| lat, lng | WGS 84, 6 decimals |
| name | the clinic's name |
| phones | international format, 3 at most (`[]` when none) |
| website, email | as the sources give them |
| address, city, postcode | street line, city, postcode |
| country, region | ISO 3166-1 alpha-2; region as the source gives it |
| hours | 7 days, Monday first: `null` unknown, `[]` closed, else `[open, close)` intervals in minutes after midnight (close ≤ 1440); `null` when unknown |
| hoursText | the source's `opening_hours` when it could not be read |
| flags | 1 open 24/7, 2 emergency service |
| src | 1 Overture, 2 OpenStreetMap, 4 All the Places (sum) |
| osm | the OpenStreetMap element (`node/123`, `way/123`, `relation/123`) or `null` |
| tz | IANA time zone |

## How it was made

1. Overture: places under taxonomy `veterinary_care` (without pet hospices, holistic care, animal physical therapy),
   not permanently closed, confidence ≥ 0.6, or ≥ 0.5 with a phone number; head offices, associations, insurers and
   stand-alone vet pharmacies or labs left out. Duplicates merged: ≤ 25 m, ≤ 60 m with alike names, the same phone
   ≤ 500 m; the most reliable row is kept and takes what it lacked.
2. All the Places: the vet chains' spiders (opening hours; BluePearl's hospitals flagged as emergency services),
   matched to a row by phone (≤ 1 km), website (≤ 300 m) or alike names (≤ 100 m, any name ≤ 25 m), else added.
3. OpenStreetMap: matched the same way; they bring opening hours (when All the Places has none), `emergency=yes`,
   24/7 and the OSM reference; the others are added.
4. Countries from the sources (else the nearest Overture vet), phones in international format (libphonenumber),
   time zones (timezonefinder), opening hours read from the OSM syntax, emergency flags from the categories, OSM
   tags, the hours (open every day around the clock) and the names in some forty languages.

`build_vets.py` reproduces it: `python3 build_vets.py --release {release} --out <dir>`.
"""


INDEX_HTML = string.Template("""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Véto Garde open data</title>
<meta name="description" content="The veterinarians of the whole world, one row per clinic, under the ODbL 1.0.">
<style>
:root { color-scheme: light dark; --bg: #fbfaf7; --fg: #1d1d1f; --muted: #5f6368; --line: #e3e1db; --card: #ffffff; --accent: #0b6e4f; }
@media (prefers-color-scheme: dark) { :root { --bg: #121212; --fg: #ececec; --muted: #a8a8a8; --line: #2e2e2e; --card: #1b1b1b; --accent: #5fd3a8; } }
* { box-sizing: border-box; }
body { margin: 0; background: var(--bg); color: var(--fg); font: 16px/1.55 system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; }
main { max-width: 760px; margin: 0 auto; padding: 32px 16px 48px; }
h1 { font-size: 1.55rem; line-height: 1.25; margin: 0 0 4px; }
h2 { font-size: 1.1rem; margin: 30px 0 10px; }
.sub, footer { color: var(--muted); }
a { color: var(--accent); }
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(130px, 1fr)); gap: 8px; margin: 20px 0 0; }
.stat, .licence { background: var(--card); border: 1px solid var(--line); border-radius: 10px; padding: 10px 12px; }
.stat b { display: block; font-size: 1.25rem; font-variant-numeric: tabular-nums; }
.stat span { color: var(--muted); font-size: .9rem; }
table { width: 100%; border-collapse: collapse; }
th, td { text-align: left; padding: 8px 6px; border-bottom: 1px solid var(--line); vertical-align: top; }
td.size { text-align: right; white-space: nowrap; font-variant-numeric: tabular-nums; }
ul { padding-left: 1.2em; }
li { margin: 6px 0; }
code { font-size: .92em; overflow-wrap: anywhere; }
footer { font-size: .9rem; margin-top: 36px; }
</style>
</head>
<body>
<main>
<h1>Véto Garde: world base of veterinarians</h1>
<p class="sub">Release $release · built $built</p>
<p>The veterinarians of the whole world, one row per clinic, as the Véto Garde app uses them: name, phone numbers,
website, email, address, opening hours, emergency service and open 24/7 flags, time zone. Merged from three open
sources by <a href="build_vets.py">build_vets.py</a>.</p>
<div class="stats">
<div class="stat"><b>$vets</b><span>veterinarians</span></div>
<div class="stat"><b>$countries</b><span>countries</span></div>
<div class="stat"><b>$phone</b><span>with a phone</span></div>
<div class="stat"><b>$hours</b><span>with opening hours</span></div>
<div class="stat"><b>$emergency</b><span>emergency or 24/7</span></div>
</div>

<h2>Licence</h2>
<div class="licence">
<p>This database is made available under the <a href="https://opendatacommons.org/licenses/odbl/1-0/">Open Database
License (ODbL 1.0)</a>. It contains information from OpenStreetMap: <strong>© OpenStreetMap contributors</strong>
(<a href="https://www.openstreetmap.org/copyright">openstreetmap.org/copyright</a>).</p>
</div>

<h2>Sources</h2>
<ul>
<li><strong>OpenStreetMap</strong>: <code>amenity=veterinary</code>, $osm_route, extract of $osm.
© OpenStreetMap contributors, ODbL 1.0.</li>
<li><strong>Overture Maps Foundation</strong>: places theme, release $release. CDLA-Permissive-2.0 (Meta, Microsoft,
BrightQuery, Uberall, PinMeTo, DAC and others), Apache-2.0 (Foursquare: © 2024 Foursquare Labs, Inc., transformed to
the Overture schema), CC0-1.0 (All the Places). <a href="https://docs.overturemaps.org/attribution/">Overture's attribution</a>.</li>
<li><strong>All the Places</strong>: the vet chains' store locators, run $atp. CC0-1.0.
<a href="https://www.alltheplaces.xyz/">alltheplaces.xyz</a>.</li>
</ul>

<h2>Files</h2>
<table>
<thead><tr><th>File</th><th>Content</th><th class="size">Size</th></tr></thead>
<tbody>
$files
</tbody>
</table>

<h2>Format</h2>
<p><code>vets-$release.json.gz</code> is gzipped JSON: <code>{licence, attribution, columns, rows}</code>, one array
per vet in the order of <code>columns</code>: id, lat, lng, name, phones, website, email, address, city, postcode,
country (ISO 3166-1 alpha-2), region, hours (7 days from Monday: null unknown, [] closed, else [open, close)
intervals in minutes after midnight), hoursText, flags (1 open 24/7, 2 emergency service), src (1 Overture,
2 OpenStreetMap, 4 All the Places), osm (the OpenStreetMap element), tz (IANA time zone).
<a href="README.md">README.md</a> details every column and how the base is made.</p>

<footer>Véto Garde, an app of Pam's Factory · <a href="https://veto-garde.pamsfactory.app">veto-garde.pamsfactory.app</a></footer>
</main>
</body>
</html>
""")


def human_size(n):
    for unit, step in (("GB", 1e9), ("MB", 1e6), ("kB", 1e3)):
        if n >= step:
            return f"{n / step:.1f} {unit}"
    return f"{n} B"


def osm_route(osm_sources):
    """How the OSM data was read, in the words of README.md and index.html."""
    if osm_sources and osm_sources.get("qlever"):
        if not osm_sources.get("overpass"):
            return "read from QLever's OpenStreetMap planet"
        return (f"read through the Overpass API ({osm_sources['overpass']} slices) and QLever's OpenStreetMap planet "
                f"for the {osm_sources['qlever']} slices Overpass could not serve")
    return "read through the Overpass API"


def write_open_index(odir, release, built, osm_date, atp_run, stats, osm_sources=None):
    """open/index.html: the publication's page (what the base is, its licence and credits, the files)."""
    e = lambda x: html.escape(str(x))
    described = [
        (f"vets-{release}.json.gz", "the whole base, every veterinarian (ODbL 1.0)"),
        ("README.md", "columns, sources and their licences, how the base is made"),
        ("build_vets.py", "the script that builds it from the sources"),
    ]
    files = "\n".join(
        f'<tr><td><a href="{e(name)}">{e(name)}</a></td><td>{e(what)}</td><td class="size">{human_size(os.path.getsize(os.path.join(odir, name)))}</td></tr>'
        for name, what in described if os.path.exists(os.path.join(odir, name)))
    when = built.replace("T", " ").replace("Z", " UTC") if built else "unknown"
    page = INDEX_HTML.substitute(
        release=e(release), built=e(when), osm=e(osm_date), atp=e(atp_run), osm_route=e(osm_route(osm_sources)),
        vets=f"{stats['vets']:,}", countries=f"{stats['countries']:,}", phone=f"{stats['phone']:,}",
        hours=f"{stats['hours']:,}", emergency=f"{stats['emergency']:,}", files=files)
    with open(os.path.join(odir, "index.html"), "w", encoding="utf-8") as f:
        f.write(page)


def write_outputs(out, vets, release, osm_date, atp_run, missing, osm_sources=None):
    tiles = defaultdict(list)
    per_country = defaultdict(lambda: [0, 0, 0, 0])
    rows = []
    for v in sorted(vets, key=lambda v: (v.lat, v.lng)):
        row = v.row()
        rows.append(row)
        tiles[(math.floor(v.lat), math.floor(v.lng))].append(row)
        c = per_country[v.country or "??"]
        c[0] += 1
        c[1] += bool(v.phones)
        c[2] += v.hours is not None
        c[3] += bool(v.flags & FLAG_EMERGENCY)
    tdir = os.path.join(out, "tiles")
    if os.path.isdir(tdir):
        for f in os.listdir(tdir):
            if f.endswith(".json.gz"):
                os.remove(os.path.join(tdir, f))
    os.makedirs(tdir, exist_ok=True)
    manifest_tiles = {}
    for (la, ln), trows in sorted(tiles.items()):
        name = f"{la}_{ln}"
        with gzip.open(os.path.join(tdir, f"{name}.json.gz"), "wt", encoding="utf-8") as f:
            json.dump(trows, f, ensure_ascii=False, separators=(",", ":"))
        manifest_tiles[name] = len(trows)
    # the emergency vets of the whole world in one file: the API loads it once to search 150 km around
    emergency = [row for row in rows if row[COLUMNS.index("flags")] & (FLAG_OPEN24 | FLAG_EMERGENCY)]
    with gzip.open(os.path.join(out, "emergency.json.gz"), "wt", encoding="utf-8") as f:
        json.dump(emergency, f, ensure_ascii=False, separators=(",", ":"))
    built = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    odir = os.path.join(out, "open")
    os.makedirs(odir, exist_ok=True)
    with gzip.open(os.path.join(odir, f"vets-{release}.json.gz"), "wt", encoding="utf-8") as f:
        json.dump({"licence": "ODbL-1.0", "attribution": attribution(release, osm_date, atp_run), "columns": COLUMNS,
                   "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
    with open(os.path.join(odir, "README.md"), "w", encoding="utf-8") as f:
        f.write(README.format(release=release, built=built, osm=osm_date, atp=atp_run, count=len(rows),
                              osm_route=osm_route(osm_sources)))
    shutil.copyfile(os.path.abspath(__file__), os.path.join(odir, "build_vets.py"))
    stats = {"vets": len(rows), "countries": len([c for c in per_country if c != "??"]),
             "phone": sum(c[1] for c in per_country.values()), "hours": sum(c[2] for c in per_country.values()),
             "emergency": len(emergency)}
    write_open_index(odir, release, built, osm_date, atp_run, stats, osm_sources)
    manifest = {
        "release": release,
        "osm": osm_date,
        "atp": atp_run,
        "builtAt": built,
        "vets": len(rows),
        "tiles": manifest_tiles,
        "countries": dict(sorted(per_country.items(), key=lambda kv: -kv[1][0])),
    }
    if osm_sources:
        manifest["osmSlices"] = osm_sources  # {"overpass": n, "qlever": n}: QLever's planet is older (its date in osm)
    if missing:
        manifest["osmMissing"] = [_cell_name(c) for c in missing]
    with open(os.path.join(out, "manifest.json"), "w", encoding="utf-8") as f:  # last: the base is complete
        json.dump(manifest, f, ensure_ascii=False, separators=(",", ":"))
    return manifest


# ———————————————————————————— main

def main():
    ap = argparse.ArgumentParser(description="Builds Véto Garde's world base of veterinarians.")
    ap.add_argument("--release", required=True, help="Overture release, e.g. 2026-10-21.0")
    ap.add_argument("--out", required=True)
    ap.add_argument("--bbox", help="xmin,ymin,xmax,ymax (tests)")
    ap.add_argument("--cache", help="downloads kept between runs (default <out>/cache)")
    args = ap.parse_args()
    if phonenumbers is None:
        sys.exit("phonenumbers is missing: pip install duckdb phonenumbers timezonefinder")
    bbox = tuple(float(v) for v in args.bbox.split(",")) if args.bbox else None
    cache = args.cache or os.path.join(args.out, "cache")

    # All the Places and Overpass (network, slow) while DuckDB reads Overture
    others = {}

    def fetch_others():
        try:
            try:
                others["atp"] = fetch_atp(cache, bbox)
            except (urllib.error.URLError, OSError, ValueError, KeyError) as e:
                log(f"All the Places could not be read ({e}): built without it")
                others["atp"] = (None, [])
            others["osm"] = fetch_osm(cache, bbox)
        except BaseException as e:  # re-raised in the main thread
            others["error"] = e

    worker = threading.Thread(target=fetch_others, daemon=True)
    worker.start()
    overture, reference = read_overture(args.release, bbox, cache)
    worker.join()
    if "error" in others:
        raise others["error"]
    atp_run, atp = others["atp"]
    osm_date, osm, missing, osm_sources = others["osm"]

    filled = assign_countries(overture + atp + osm, reference)
    for v in overture + atp + osm:
        v.finish_phones()
    log(f"countries: {filled} vets placed by their nearest Overture neighbour")
    pool, how = dedupe(overture)
    log(f"Overture: {len(pool.vets)} vets after merging {sum(how.values())} duplicates {dict(how)}")
    how = integrate(pool, atp)
    log(f"All the Places: {dict(how)}")
    how = integrate(pool, osm)
    log(f"OSM: {dict(how)}")
    finalize(pool.vets)
    manifest = write_outputs(args.out, pool.vets, args.release, osm_date, atp_run, missing, osm_sources)
    src = Counter(v.src for v in pool.vets)
    log(f"{manifest['vets']} vets in {len(manifest['tiles'])} tiles; by sources {dict(sorted(src.items()))}")


if __name__ == "__main__":
    main()
