Match every name a place has, without case or accents

Three of the misses in #12 were in the index all along, under another
spelling: Chittagong is filed as Chattogram District, Da Nang as Đà Nẵng,
Bogota as Bogotá, D.c. Search compared the query with the primary and
English names only, and Jaro-Winkler rewards a shared prefix, so a
near-homograph on another continent won.

A record is now matched under every name Overture holds for it: its own,
its translations (names.common) and its alternate, short and official
names (names.rules), each compared after lowercasing and stripping
accents. matched_name reports the spelling that hit. Natural Earth, which
has no alternates, is matched on its own and English names as before,
folded the same way.

Building and folding those names on every query would make search about
twice as slow as main, so normalize_geodata stores them in a
search_names column, and search reads it where every file has one. A raw
Overture download still works; it builds the names on the fly. On the
2026-08-19.0 release a fuzzy ids_only search now takes 0.19s against
main's 0.82s, and a full-geometry one 0.74s. The column adds 0.16 GB to
divisions_area and 0.22 GB to localities.

The natural-language pipeline sees better candidates, but still of the
subtypes its model was trained on; the name it is shown is unchanged.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

dataset/scripts/normalize_geodata.py

@@ -18,6 +18,10 @@ Cities and towns (Overture ``locality`` and ``localadmin``) go to their own
 file. Fuzzy search reads it; the training pipeline and the natural-language
 search read only ``divisions_area``, because the model was not trained on
 those subtypes.
+
+Both Overture files also store ``search_names``: every name a record
+answers to, folded for comparison, so fuzzy search does not rebuild and
+fold them on every query.
 """
 
 from pathlib import Path
@@ -25,6 +29,7 @@ from pathlib import Path
 import duckdb
 
 from gazet.config import _DATA_DIR
+from gazet.search import DIVISIONS_AREA_SEARCH_NAMES, SEARCH_NAMES_COLUMN
 
 #: The subtypes the natural-language model was trained on.
 TRAINED_SUBTYPES = ("country", "region", "county")
@@ -63,9 +68,9 @@ def normalize_geodata(output_root: Path | None = None) -> dict[str, str]:
         con.execute(
             f"""
             COPY (
-                SELECT * REPLACE (
-                    ST_GeomFromWKB(ST_AsWKB(geometry)) AS geometry
-                )
+                SELECT
+                    * REPLACE (ST_GeomFromWKB(ST_AsWKB(geometry)) AS geometry),
+                    {DIVISIONS_AREA_SEARCH_NAMES} AS {SEARCH_NAMES_COLUMN}
                 FROM read_parquet('{root / "overture/divisions_area/*.parquet"}')
                 WHERE geometry IS NOT NULL
                   AND subtype IN {subtypes}

src/gazet/search.py

@@ -1,5 +1,5 @@
 import logging
-from functools import partial
+from functools import cache, partial
 
 import duckdb
 import pandas as pd
@@ -30,6 +30,67 @@ def _readable(expr: str) -> str:
     return f"NULLIF(trim({expr}), '')"
 
 
+def _folded(expr: str) -> str:
+    """SQL that compares a name without case or accents: "Bogota" is "Bogotá"."""
+    return f"lower(strip_accents({expr}))"
+
+
+def search_names_sql(
+    name_expr: str, english_name_expr: str, alternate_names_expr: str | None = None
+) -> str:
+    """SQL for every name a record answers to, each beside its folded form.
+
+    The record's own name comes first and its English one second, so they
+    win a tie; the alternates follow with exact repeats dropped. Blank names
+    are left out.
+    """
+
+    def entry(name: str) -> str:
+        return f"{{'name': trim({name}), 'folded': {_folded(f'trim({name})')}}}"
+
+    alternates = (
+        f"list_distinct(list_transform({alternate_names_expr}, n -> {entry('n')}))"
+        if alternate_names_expr
+        else "[]"
+    )
+    return (
+        f"list_filter(list_concat([{entry(name_expr)}, {entry(english_name_expr)}], "
+        f"{alternates}), s -> NULLIF(s.name, '') IS NOT NULL)"
+    )
+
+
+#: The column normalize_geodata stores search_names_sql's result in, so a
+#: search reads it instead of rebuilding and folding every name per query.
+SEARCH_NAMES_COLUMN = "search_names"
+#: Overture's translations (``names.common``) and its alternate, short and
+#: official names (``names.rules``).
+DIVISIONS_AREA_SEARCH_NAMES = search_names_sql(
+    PRIMARY_NAME,
+    DIVISIONS_AREA_ENGLISH_NAME,
+    "list_concat(COALESCE(map_values(names.common), []), "
+    "COALESCE(list_transform(names.rules, r -> r.value), []))",
+)
+NATURAL_EARTH_SEARCH_NAMES = search_names_sql(PRIMARY_NAME, NATURAL_EARTH_ENGLISH_NAME)
+
+
+@cache
+def _stores_search_names(path: str) -> bool:
+    """Whether a parquet file (or glob) carries the stored search names."""
+    columns = duckdb.connect().execute("DESCRIBE SELECT * FROM read_parquet(?)", [path])
+    return SEARCH_NAMES_COLUMN in {row[0] for row in columns.fetchall()}
+
+
+def _search_names(path: str | list[str], computed: str) -> str:
+    """The stored search names where every file has them, else ``computed``.
+
+    Raw Overture downloads and the test fixtures have no stored column.
+    """
+    paths = [path] if isinstance(path, str) else path
+    if all(_stores_search_names(p) for p in paths):
+        return SEARCH_NAMES_COLUMN
+    return computed
+
+
 def _divisions_area_paths(include_localities: bool) -> str | list[str]:
     """The divisions_area file, and the localities beside it where asked for.
 
@@ -46,6 +107,7 @@ def simple_fuzzy_search(
     path: str | list[str],
     source: str,
     place: Place,
+    search_names_expr: str,
     name_expr: str = PRIMARY_NAME,
     english_name_expr: str | None = None,
     extra_select: str = "",
@@ -56,12 +118,14 @@ def simple_fuzzy_search(
 ) -> pd.DataFrame:
     """Jaro-Winkler similarity search using only the place name.
 
-    Each record is matched under two names: ``name_expr``, its own, and
-    ``english_name_expr``, its English one where the source carries it. A
-    record scores the better of the two, so a place is reachable by its
-    English exonym ("Copenhagen") as well as by its local name ("Københavns
-    Kommune"), and a record with no English name matches on its own name
-    alone. ``matched_name`` is whichever of the two produced the score.
+    Each record is matched under every name in ``search_names_expr`` (see
+    :func:`search_names_sql`): its own, its English one, and any others the
+    source holds. A record scores its best name, so a place is reachable by
+    its English exonym ("Copenhagen") as well as by its local name
+    ("Københavns Kommune") or an alternate one ("Chittagong" for "Chattogram
+    District"). Names are compared without case or accents, so "Bogota"
+    finds "Bogotá". ``matched_name`` is whichever name produced the score;
+    on a tie, the record's own name.
 
     ``name`` is the English name where there is one and the record's own name
     otherwise — the expression :func:`fetch_by_id` uses too, so a candidate
@@ -80,17 +144,13 @@ def simple_fuzzy_search(
     GeoJSON serialization), for lightweight context (e.g. ``ids_only``
     responses).
     """
-    params = [path, place.place, place.place, place.place, place.place, limit]
-
     english_expr = english_name_expr or "CAST(NULL AS VARCHAR)"
     extra_clause = f", {extra_select}" if extra_select else ""
-    geometry_clause = ", ST_AsGeoJSON(geometry) AS geometry" if include_geometry else ""
     bbox_clause = f", {bbox_expr} AS bbox" if include_bbox else ""
-    # Geometry and bbox are computed in the first stage, so the final select
-    # only has to name them.
-    carried_clause = (", geometry" if include_geometry else "") + (
-        ", bbox" if include_bbox else ""
-    )
+    # Geometry is carried raw and only serialised for the rows returned.
+    carried_geometry = ", geometry" if include_geometry else ""
+    geometry_clause = ", ST_AsGeoJSON(geometry) AS geometry" if include_geometry else ""
+    query = _folded("$1")
     rel = con.execute(
         f"""
         WITH named AS (
@@ -102,26 +162,35 @@ def simple_fuzzy_search(
                 region,
                 admin_level,
                 is_land,
-                is_territorial{extra_clause}{geometry_clause}{bbox_clause},
+                is_territorial{extra_clause}{bbox_clause}{carried_geometry},
                 {_readable(name_expr)} AS primary_name,
-                {_readable(english_expr)} AS english_name
-            FROM read_parquet(?)
+                {_readable(english_expr)} AS english_name,
+                {search_names_expr} AS search_names
+            FROM read_parquet($2)
         ),
         scored AS (
             SELECT
                 *,
-                COALESCE(
-                    jaro_winkler_similarity(lower(primary_name), lower(?)), 0.0
-                ) AS primary_similarity,
-                COALESCE(
-                    jaro_winkler_similarity(lower(english_name), lower(?)), 0.0
-                ) AS english_similarity,
-                COALESCE(contains(lower(primary_name), lower(?)), false)
-                    AS primary_substring,
-                COALESCE(contains(lower(english_name), lower(?)), false)
-                    AS english_substring
+                -- Struct order is the ranking: a substring hit beats a
+                -- higher raw score, and an earlier name breaks a tie.
+                list_max(
+                    list_transform(
+                        search_names,
+                        (s, i) -> {{
+                            'substring': contains(s.folded, {query}),
+                            'similarity': jaro_winkler_similarity(s.folded, {query}),
+                            'rank': -i,
+                            'name': s.name
+                        }}
+                    )
+                ) AS best
             FROM named
-            WHERE primary_name IS NOT NULL OR english_name IS NOT NULL
+            WHERE len(search_names) > 0
+        ),
+        ranked AS (
+            SELECT * FROM scored
+            ORDER BY best.substring DESC, best.similarity DESC, admin_level ASC
+            LIMIT $3
         )
         SELECT
             id,
@@ -132,22 +201,14 @@ def simple_fuzzy_search(
             region,
             admin_level,
             is_land,
-            is_territorial{extra_clause}{carried_clause},
-            -- Compared as a pair so the winning name is the one the ranking
-            -- below would pick: a substring hit beats a higher raw score.
-            CASE
-                WHEN (english_substring, english_similarity)
-                     > (primary_substring, primary_similarity)
-                THEN english_name
-                ELSE COALESCE(primary_name, english_name)
-            END AS matched_name,
-            GREATEST(primary_similarity, english_similarity) AS similarity,
-            primary_substring OR english_substring AS is_substring_match
-        FROM scored
+            is_territorial{extra_clause}{", bbox" if include_bbox else ""}{geometry_clause},
+            best.name AS matched_name,
+            best.similarity AS similarity,
+            best.substring AS is_substring_match
+        FROM ranked
         ORDER BY is_substring_match DESC, similarity DESC, admin_level ASC
-        LIMIT ?
         """,
-        params,
+        [place.place, path, limit],
     )
     df = rel.fetchdf()
     df.insert(0, "source", source)
@@ -168,20 +229,22 @@ def search_divisions_area(
 ) -> pd.DataFrame:
     """Fuzzy-match a place against divisions_area (Overture admin boundaries).
 
-    Matched on ``names.primary`` and on ``names.common.en`` together: 77% of
-    divisions_area rows have no English common name and would drop out of
-    every search if English were the only match field, while ``names.primary``
-    on its own leaves a place unreachable by the English name it is usually
-    asked for.
+    Matched on ``names.primary``, ``names.common`` and ``names.rules``
+    together: 77% of divisions_area rows have no English common name and
+    would drop out of every search if English were the only match field,
+    while ``names.primary`` on its own leaves a place unreachable by the
+    English name it is usually asked for.
 
     ``include_localities`` searches cities and towns as well. Only the fuzzy
     endpoint passes it; see :func:`_divisions_area_paths`.
     """
+    path = _divisions_area_paths(include_localities)
     return simple_fuzzy_search(
         con,
-        _divisions_area_paths(include_localities),
+        path,
         "divisions_area",
         place,
+        _search_names(path, DIVISIONS_AREA_SEARCH_NAMES),
         english_name_expr=DIVISIONS_AREA_ENGLISH_NAME,
         extra_select="division_id",
         limit=limit,
@@ -204,6 +267,7 @@ def search_natural_earth(
         NATURAL_EARTH_PATH,
         "natural_earth",
         place,
+        NATURAL_EARTH_SEARCH_NAMES,
         english_name_expr=NATURAL_EARTH_ENGLISH_NAME,
         limit=limit,
         include_geometry=include_geometry,

tests/conftest.py

@@ -6,6 +6,8 @@ from pathlib import Path
 import duckdb
 import pytest
 
+from gazet.search import DIVISIONS_AREA_SEARCH_NAMES, SEARCH_NAMES_COLUMN
+
 # Force data dir to project root for tests
 os.environ["GAZET_DATA_DIR"] = str(Path(__file__).resolve().parent.parent / "data")
 # Prefer original (non-normalized) paths — test suite ships without normalized copies
@@ -66,8 +68,17 @@ _EXONYM_ROWS = [
     ("ratnapura", None, "Ratnapura District", "LK", "county", 2, 80.4, 6.7),
     ("gva", "Genève", "", "CH", "county", 2, 6.1, 46.2),
     ("manas", "Manas", None, "KG", "county", 2, 72.9, 42.5),
+    # Reachable only by an alternate name, or only once accents are folded;
+    # the near-homographs beside them are what those queries used to hit.
+    ("ctg", "Chattogram District", None, "BD", "county", 2, 91.8, 22.3),
+    ("chitato", "Chitato", None, "AO", "county", 2, 20.8, -7.4),
+    ("bog", "Bogotá, D.c.", None, "CO", "county", 2, -74.1, 4.6),
+    ("bobota", "Bobota", None, "RO", "county", 2, 22.8, 47.4),
 ]
 
+#: ``names.rules`` values: alternate, short and official names.
+_EXONYM_RULES = {"ctg": ["Chittagong"]}
+
 #: Cities, shaped like the localities file: never trained on, so only fuzzy
 #: search may return them.
 _LOCALITY_ROWS = [
@@ -85,13 +96,26 @@ def _name_map(english):
     return "MAP([], [])" if english is None else f"MAP(['en'], ['{english}'])"
 
 
+def _rules(id):
+    """``names.rules``: the record's alternate names, as Overture shapes them."""
+    values = _EXONYM_RULES.get(id, [])
+    if not values:
+        return "CAST([] AS STRUCT(variant VARCHAR, language VARCHAR, value VARCHAR)[])"
+    items = ", ".join(
+        f"{{'variant': 'alternate', 'language': NULL, 'value': '{value}'}}"
+        for value in values
+    )
+    return f"[{items}]"
+
+
 def _write_divisions(path, rows):
-    """Write rows to a parquet with the divisions_area schema."""
+    """Write rows to a parquet with the normalised divisions_area schema,
+    search names stored as normalize_geodata stores them."""
     c = duckdb.connect()
     c.execute("INSTALL spatial")
     c.execute("LOAD spatial")
     values = ",\n".join(
-        f"({_literal(id)}, {_literal(primary)}, {_name_map(english)}, "
+        f"({_literal(id)}, {_literal(primary)}, {_name_map(english)}, {_rules(id)}, "
         f"{_literal(country)}, {_literal(subtype)}, "
         f"{'NULL' if admin_level is None else admin_level}, {lon}, {lat})"
         for id, primary, english, country, subtype, admin_level, lon, lat in rows
@@ -99,6 +123,8 @@ def _write_divisions(path, rows):
     c.execute(
         f"""
         COPY (
+            SELECT *, {DIVISIONS_AREA_SEARCH_NAMES} AS {SEARCH_NAMES_COLUMN}
+            FROM (
             SELECT
                 id,
                 ST_AsWKB(
@@ -114,7 +140,7 @@ def _write_divisions(path, rows):
                 country,
                 subtype,
                 'land' AS class,
-                {{'primary': primary_name, 'common': common}} AS names,
+                {{'primary': primary_name, 'common': common, 'rules': rules}} AS names,
                 NULL AS region,
                 CAST(admin_level AS INTEGER) AS admin_level,
                 true AS is_land,
@@ -122,7 +148,10 @@ def _write_divisions(path, rows):
                 id AS division_id
             FROM (VALUES
                 {values}
-            ) AS t(id, primary_name, common, country, subtype, admin_level, lon, lat)
+            ) AS t(
+                id, primary_name, common, rules, country, subtype, admin_level, lon, lat
+            )
+            )
         ) TO '{path}' (FORMAT PARQUET)
         """
     )

tests/test_search.py

@@ -2,6 +2,7 @@
 
 import pandas as pd
 
+from gazet import search
 from gazet.schemas import Place
 from gazet.search import (
     get_by_id,
@@ -266,6 +267,27 @@ class TestEnglishNames:
         assert bool(df.iloc[0]["is_substring_match"]) is False
 
 
+class TestOtherNames:
+    """A place is reachable by any name Overture holds for it, with or
+    without accents: "Chittagong" is an alternate name of Chattogram
+    District, and "Bogota" is how "Bogotá" is typed without the accent."""
+
+    def test_alternate_name_finds_the_place(self, con, exonym_source):
+        df = search_divisions_area(con, Place(place="Chittagong"), limit=3)
+        assert df.iloc[0]["id"] == "ctg"
+        assert df.iloc[0]["matched_name"] == "Chittagong"
+        assert df.iloc[0]["name"] == "Chattogram District"
+
+    def test_unaccented_query_finds_the_accented_name(self, con, exonym_source):
+        df = search_divisions_area(con, Place(place="Bogota"), limit=3)
+        assert df.iloc[0]["id"] == "bog"
+        assert df.iloc[0]["matched_name"] == "Bogotá, D.c."
+
+    def test_accented_query_finds_the_unaccented_name(self, con, exonym_source):
+        df = search_divisions_area(con, Place(place="Chattógram"), limit=1)
+        assert df.iloc[0]["id"] == "ctg"
+
+
 class TestLocalities:
     """Cities are searchable by fuzzy name, and kept from the
     natural-language pipeline, whose model was never trained on them."""
@@ -292,3 +314,22 @@ class TestLocalities:
     def test_a_locality_id_resolves(self, con, localities_source):
         df = get_by_id(con, "manaus", include_geometry=False)
         assert df.iloc[0]["name"] == "Manaus"
+
+
+class TestStoredSearchNames:
+    """normalize_geodata stores each record's folded names; a file without
+    them, such as a raw Overture download, is searched all the same."""
+
+    def test_the_stored_names_are_read(self, exonym_source):
+        assert search._search_names(exonym_source, "computed") == "search_names"
+
+    def test_a_raw_file_falls_back_to_computing_them(self):
+        assert (
+            search._search_names(search.DIVISIONS_AREA_PATH, "computed") == "computed"
+        )
+
+    def test_both_find_the_same_record(self, con, exonym_source, monkeypatch):
+        stored = search_divisions_area(con, Place(place="Chittagong"), limit=1)
+        monkeypatch.setattr(search, "_search_names", lambda path, computed: computed)
+        computed = search_divisions_area(con, Place(place="Chittagong"), limit=1)
+        assert stored.iloc[0]["id"] == computed.iloc[0]["id"] == "ctg"