"""Read scoped PDP search history and reproduce the signed-off search model.""" import calendar import contextlib import datetime as dt from copy import deepcopy import json import math import re from decimal import Decimal, InvalidOperation TABLE = "amzn_search_query_performance" ORDERS = "amzn_order_items" TERM = "search_query_data_search_query" # Array positions are the prototype's wire contract. FIELDS = ("vol", "tc", "mc", "tp", "mp", "mkt", "our", "ti", "mi", "tca", "mca", "pm", "py") COLUMNS = { "vol": "search_query_data_search_query_volume", "tc": "click_data_total_click_count", "mc": "click_data_asin_click_count", "tp": "purchase_data_total_purchase_count", "mp": "purchase_data_asin_purchase_count", "ti": "impression_data_total_query_impression_count", "mi": "impression_data_asin_impression_count", "tca": "cart_add_data_total_cart_add_count", "mca": "cart_add_data_asin_cart_add_count", "pm": "purchase_data_total_median_purchase_price_amount", "py": "purchase_data_asin_median_purchase_price_amount", } STEPS = ( ("ctr", "Seen to clicked", "Title and main image", "mc", "mi", "tc", "ti", 100), ("atc", "Clicked to cart", "Price, images and bullets", "mca", "mc", "tca", "tc", 10), ("a2p", "Cart to bought", "Price and delivery", "mp", "mca", "tp", "tca", 5), ("cv", "Clicked to bought", "Bullets, images and reviews", "mp", "mc", "tp", "tc", 10), ) DROP = { "ctr": ["See you, rarely click", "Title start or main image"], "atc": ["Click, rarely add to cart", "Price, images or bullets"], "a2p": ["Add to cart, rarely buy", "Price or delivery"], "cv": ["Click, rarely buy", "Bullets, images or reviews"], } FACT_SOURCES = {"amzn_listings": "asin1", "amzn_catalog_items": "asin", "amzn_listing_items": "lookup_asin"} FACT_FIELDS = {"title": ("item_name", "item_description"), "brand": ("brand",), "bullets": ("bullet_point",), "desc": ("product_description",), "terms": ("generic_keyword", "backend_keywords"), "highlights": ("title_differentiation",)} # SQP value columns: your sales and all sales on the searches. _SALES_COL = 6 _MARKET_SALES_COL = 5 @contextlib.contextmanager def _scoped_reader(cur, source_scope=None, *, tables=(TABLE, ORDERS)): from harness.runtime import mirror_cursor from core.perm_scope import app_source_cursor owned = not hasattr(cur, "execute") raw = mirror_cursor(cur) if owned else cur try: with app_source_cursor(raw, tables, source_scope) as scoped: yield scoped finally: if owned: raw.close() def _array(row): values = {k: float(row.get(column) or 0) for k, column in COLUMNS.items()} values["py"] = values["py"] or values["pm"] values["mkt"] = values["tp"] * values["pm"] values["our"] = values["mp"] * values["py"] return [values[k] for k in FIELDS] def _aggregate(arrays): values = [sum(x[i] or 0 for x in arrays) for i in range(11)] for i, wi in ((11, 3), (12, 4)): held = [x for x in arrays if x[i]] weight = sum(x[wi] or 0 for x in held) values.append(sum(x[i] * (x[wi] or 0) for x in held) / weight if weight else held[0][i] if held else 0) return values def _read_searches(cur, asin): columns = ",".join(("asin", TERM, "start_date", "_report_period", *COLUMNS.values())) result = cur.execute(f"SELECT {columns} FROM {TABLE} WHERE UPPER(_report_period) IN ('MONTH', 'WEEK') " f"AND (asin=? OR {TERM} IN (SELECT {TERM} FROM {TABLE} WHERE asin=?)) " f"ORDER BY asin, {TERM}, start_date", [asin, asin]) names = [x[0] for x in result.description] return [dict(zip(names, row)) for row in result.fetchall()] def _load(cur, asin): records = _read_searches(cur, asin) mine = [r for r in records if r["asin"] == asin] history = sorted({str(r["start_date"])[:7] for r in mine if r["_report_period"].upper() == "MONTH"}) months = history[-12:] weeks = sorted({str(r["start_date"])[:10] for r in mine if r["_report_period"].upper() == "WEEK"}) grouped = {} for row in records: key = (row["asin"], row[TERM]) search = grouped.setdefault(key, {"q": row[TERM], "m": {}, "w": {}}) period = row["_report_period"].upper() bucket = search["m" if period == "MONTH" else "w"] start = str(row["start_date"])[:7 if period == "MONTH" else 10] bucket.setdefault(start, []).append(_array(row)) rows = {} for (owner, _), search in grouped.items(): for kind in ("m", "w"): search[kind] = {start: _aggregate(xs) for start, xs in search[kind].items()} search.update(v55(search, "all", {"months": months}) or dict.fromkeys(FIELDS, 0)) rows.setdefault(owner, []).append(search) orders = cur.execute(f"SELECT SUBSTR(CAST(purchase_date AS VARCHAR),1,7), " f"SUM(COALESCE(CAST(quantity AS DOUBLE),0)), SUM(COALESCE(CAST(item_price AS DOUBLE),0)) " f"FROM {ORDERS} WHERE asin=? GROUP BY 1 ORDER BY 1", [asin]).fetchall() return {"asin": asin, "rows": rows, "months": months, "history": history, "weeks": weeks, "period": period_label("all", {"months": months}), "fields": list(FIELDS), "orders": {str(m): [float(u), float(s)] for m, u, s in orders}, "row_counts": {p: sum(r["_report_period"].upper() == p for r in mine) for p in ("MONTH", "WEEK")}} def load(cur, asin, *, source_scope=None): """Read every permitted monthly and weekly row, plus extended order-line sales.""" with _scoped_reader(cur, source_scope) as scoped: return _load(scoped, asin) def _source_listing(cur, table, asin, skus): columns = {r[0] for r in cur.execute("SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()} identity = FACT_SOURCES[table] if identity in columns: predicate, params = f"{identity}=?", [asin] elif table == "amzn_listing_items" and "sku" in columns and skus: predicate, params = "sku IN (" + ",".join("?" for _ in skus) + ")", skus else: return {} stamps = [key for key in ("_snapshot_date", "last_updated_date", "_report_row_number") if key in columns] order = " ORDER BY " + ",".join(key + " DESC NULLS LAST" for key in stamps) if stamps else "" result = cur.execute(f"SELECT * FROM {table} WHERE {predicate}{order} LIMIT 1", params) names = [r[0] for r in result.description] row = result.fetchone() return dict(zip(names, row)) if row is not None else {} def _fact_values(row): from modules.ranking_diagnostic import _content_text return {key: _content_text(value) for key, value, _ in _fact_entries(row)} def _fact_entries(row): for field, value in row.items(): if value is None or field.startswith("_"): continue if field == "attributes": attributes = json.loads(value) if isinstance(value, str) else value if isinstance(attributes, dict): yield from ((key, raw, "attributes." + key) for key, raw in attributes.items() if raw is not None) else: yield field.removeprefix("attributes_"), value, field def _product_count(facts): value = facts.get("number_of_items") if value is None: value = facts.get("item_package_quantity") try: count = Decimal(str(value)) except InvalidOperation: return "" # A nonnumeric source fact cannot establish the product count. return str(int(count)) if count.is_finite() and count > 0 and count == count.to_integral_value() else "" def _listing_context(asin, facts, stamp): parts = {part: next((facts[k] for k in keys if facts.get(k)), "") for part, keys in FACT_FIELDS.items()} title = parts["title"] count = _product_count(facts) listing = {"asin": asin, "brand": parts["brand"], "sku": {"full": f"{count} count" if count else ""}} meta = {**facts, **listing, "title": title, "bullets": parts["bullets"].splitlines()} if "price" in meta: try: price = float(meta.pop("price")) if math.isfinite(price): meta["price"] = price except ValueError: pass # An unreported numeric price cannot support a projection. return {"listing": listing, "meta": {asin: meta}, "live": {asin: {"title": title, "bullets": parts["bullets"].splitlines(), "highlights": parts["highlights"]}}, "desc": {asin: parts["desc"]}, "terms": {asin: {"terms": parts["terms"]}}, "pd": {asin: facts}, "last_updated": stamp, "part_states": {}} def listing_facts(cur, asin, *, source_scope=None): """Build live listing facts only from permitted listing and catalog fields and rows.""" facts, sources, stamps, skus = {}, {}, [], [] with _scoped_reader(cur, source_scope, tables=tuple(FACT_SOURCES)) as scoped: for table in FACT_SOURCES: row = _source_listing(scoped, table, asin, skus) if row.get("seller_sku"): skus.append(row["seller_sku"]) values = _fact_values(row) facts.update({key: value for key, value in values.items() if value}) sources.update({key: {"table": "ut_" + table, "field": field, "value": deepcopy(raw)} for key, raw, field in _fact_entries(row) if values[key]}) stamps.extend(str(row[key]) for key in ("_snapshot_date", "last_updated_date") if row.get(key)) return {**_listing_context(asin, facts, max(stamps, default=None)), "fact_sources": sources} def live_part(context, asin, part): """Return source live text without inventing a workflow state or draft authority.""" live = context["live"].get(asin, {}) values = {"title": live.get("title", ""), "bullets": live.get("bullets", []), "highlights": live.get("highlights", ""), "desc": context["desc"].get(asin, ""), "terms": context["terms"].get(asin, {}).get("terms", "")} return {"live": values[part], "draft": None} def validate(cur, asin, *, source_scope=None, per="all"): """Reconcile the loaded window against independent SQL sums within 0.5 percent.""" with _scoped_reader(cur, source_scope) as scoped: model = load(scoped, asin) history = scoped.execute(f"SELECT DISTINCT SUBSTR(CAST(start_date AS VARCHAR),1,7) " f"FROM {TABLE} WHERE asin=? AND UPPER(_report_period)='MONTH' ORDER BY 1", [asin]).fetchall() window = [r[0] for r in history][-12:] period, width = ("WEEK", 10) if per.startswith("week:") else ("MONTH", 7) selected = window if per == "all" else [per[5:] if period == "WEEK" else per] metrics = ("vol", "tp", "mp", "mc") sums = [f"SUM(COALESCE(CAST({COLUMNS[k]} AS DOUBLE),0))" for k in metrics] sums.append(f"SUM(COALESCE(CAST({COLUMNS['tp']} AS DOUBLE),0)*COALESCE(CAST({COLUMNS['pm']} AS DOUBLE),0))") sums.append(f"SUM(COALESCE(CAST({COLUMNS['mp']} AS DOUBLE),0)*COALESCE(NULLIF(CAST({COLUMNS['py']} AS DOUBLE),0)," f"CAST({COLUMNS['pm']} AS DOUBLE),0))") placeholders = ",".join("?" for _ in selected) or "NULL" sql = (f"SELECT {','.join(sums)} FROM {TABLE} WHERE asin=? AND UPPER(_report_period)=? " f"AND SUBSTR(CAST(start_date AS VARCHAR),1,{width}) IN ({placeholders})") expected = scoped.execute(sql, [asin, period, *selected]).fetchone() rows = kwModel55({"asin": asin}, per, model) actual = sumRows57(rows) checks = {k: _ties(actual.get(k, 0), v or 0) for k, v in zip((*metrics, "mkt", "our"), expected)} volume_rows = scoped.execute(f"SELECT {TERM}, SUM(COALESCE(CAST({COLUMNS['vol']} AS DOUBLE),0)) " f"FROM {TABLE} WHERE asin=? AND UPPER(_report_period)=? AND " f"SUBSTR(CAST(start_date AS VARCHAR),1,{width}) IN ({placeholders}) GROUP BY 1", [asin, period, *selected]).fetchall() volumes = dict(volume_rows) total, top = sum(volumes.values()), max([1, *volumes.values()]) checks["share denominators"] = _ties(actual.get("vol", 0), total) and _ties(max([1, *(r["vol"] for r in rows)]), top) checks["shares"] = all(part57(r["vol"], total, top) == {"p": volumes[r["q"]] / total if total else 0, "w": volumes[r["q"]] / top} for r in rows) # Orders are modeled monthly, so WEEK supplies no order validation evidence. if period == "MONTH": orders = scoped.execute(f"SELECT SUM(COALESCE(CAST(quantity AS DOUBLE),0)), " f"SUM(COALESCE(CAST(item_price AS DOUBLE),0)) " f"FROM {ORDERS} WHERE asin=? AND " f"SUBSTR(CAST(purchase_date AS VARCHAR),1,7) IN ({placeholders})", [asin, *selected]).fetchone() held = [model["orders"][m] for m in selected if m in model["orders"]] checks.update({k: _ties(sum(v[i] for v in held), orders[i] or 0) for i, k in enumerate(("units", "sales"))}) counts = dict(scoped.execute(f"SELECT UPPER(_report_period), COUNT(*) FROM {TABLE} " "WHERE asin=? AND UPPER(_report_period) IN ('MONTH','WEEK') GROUP BY 1", [asin]).fetchall()) checks["every row read"] = all(model["row_counts"][p] == counts.get(p, 0) for p in ("MONTH", "WEEK")) checks["window"] = model["months"] == window return {"ok": all(checks.values()), "checks": checks, "actual": actual, "expected": dict(zip((*metrics, "mkt", "our"), [float(v or 0) for v in expected]))} def _ties(a, b): return abs(a - float(b)) <= max(1e-9, abs(float(b)) * 0.005) def v55(q, per, context): if per.startswith("week:"): values = q.get("w", {}).get(per[5:]) return dict(zip(FIELDS, values)) if values is not None else None data = q.get("m") or {} arrays = [data[m] for m in context["months"] if data.get(m)] if per == "all" else [data[per]] if data.get(per) else [] return dict(zip(FIELDS, _aggregate(arrays))) if arrays else None def period_span(per, context): """Return the selected MONTH or WEEK window without mixing report grains.""" if per.startswith("week:"): start = dt.date.fromisoformat(per[5:]) return str(start), str(start + dt.timedelta(days=6)) months = context["months"] if per == "all" else [per] if not months: return None, None return months[0] + "-01", f"{months[-1]}-{calendar.monthrange(int(months[-1][:4]), int(months[-1][5:]))[1]}" def period_label(per, context): start, end = period_span(per, context) if start is None: return "" if per.startswith("week:"): return f"{_mon(start)} {int(start[-2:])} to {_mon(end)} {int(end[-2:])}, {end[:4]}" labels = [f"{_mon(value)} {value[:4]}" for value in (start, end)] return labels[0] if start[:7] == end[:7] else " to ".join(labels) def part57(volume, total, top): """Preserve the prototype's known-zero shares using full selected-period denominators.""" return {"p": volume / total if total else 0, "w": volume / top if top else 0} def rate55(v): divide = lambda a, b: a / b if b else None st, score = {}, {} for key, _, _, yn, yd, mn, md, minimum in STEPS: you, market = divide(v.get(yn, 0), v.get(yd, 0)), divide(v.get(mn, 0), v.get(md, 0)) st[key] = [you, market] score[key] = max(0, min(10, 5 * (you or 0) / market)) if v.get(yd, 0) >= minimum and market else None scored = [s for s in STEPS if score[s[0]] is not None] weak = list(min(scored, key=lambda s: score[s[0]])[:3]) if scored else None return {**v, **{k: divide(v.get(n, 0), v.get(d, 0)) for k, n, d in (("is", "mi", "ti"), ("cs", "mc", "tc"), ("as", "mca", "tca"), ("ps", "mp", "tp"))}, "st": st, "score": score, "total": sum(score[s[0]] for s in scored) / len(scored) if scored else None, "weak": weak} def kwModel55(listing, per, context): cached = context.get("_kw_rows", {}).get(listing["asin"], {}).get(per) if cached is not None: return cached own = context.get("rows", {}) rows = [] for src in own.get(listing["asin"], []): value = v55(src, per, context) if value is not None: owners = ([a for a in context["_owners"].get(src["q"], []) if a != listing["asin"]] if "_owners" in context else [a for a, searches in own.items() if a != listing["asin"] and any(r["q"] == src["q"] for r in searches)]) rows.append({"q": src["q"], "src": src, "asin0": listing["asin"], **rate55(value), "owners": owners}) return sorted(rows, key=lambda r: -r["mkt"]) def fix55(r): if not r["mi"] or (r["is"] or 0) < 0.002: return ["Not seen", "Title and search terms"] if not r["weak"]: return ["Too few clicks", "Wait for more data"] return ["At or above the market", "Keep"] if r["score"][r["weak"][0]] >= 5 else r["weak"][1:3] def drop56(r): if not r["mi"] or (r["is"] or 0) < 0.002: return ["Rarely see you", "Add it to the title or search terms"] if not r["weak"]: return ["Too few clicks to tell", ""] return ["Same as the market or better", "No change"] if r["score"][r["weak"][0]] >= 4.5 else DROP[r["weak"][0]] def inText55(words, q): keys = qWords(q) count = sum(k in words for k in keys) return "All" if keys and count == len(keys) else "Some" if count else "None" def fitState63(listing, q, context): cached = context.get("_fit_states", {}).get(listing["asin"], {}) if q in cached: return cached[q] keys = qWords(q) if set(keys) & {stemW(w) for w in ("clearance", "sale", "discount", "cheap", "cheapest", "free", "best", "bestseller", "deal", "offer", "guarantee")}: return "no" rows = context.get("rows", {}).get(listing["asin"], []) if (q in context["_fit_queries"].get(listing["asin"], set()) if "_fit_queries" in context else any(r["q"] == q and (r["mp"] > 0 or r["mc"] >= 3) for r in rows)): return "fit" return "fit" if all(k in fit_words(listing, context) for k in keys) else "check" def fit_words(listing, context): """Reuse the request's proved product words for search fit and competitor word fit.""" if "_fit_words" in context: return context["_fit_words"].get(listing["asin"], set()) rows = context.get("rows", {}).get(listing["asin"], []) live = context.get("live", {}).get(listing["asin"], {}) details = context.get("pd", {}).get(listing["asin"], {}) text = " ".join([live.get("title", ""), *live.get("bullets", []), context.get("desc", {}).get(listing["asin"], ""), *(v for v in details.values() if isinstance(v, str))]) own = wordsOf(text) for row in rows: if row["mp"] > 0 or row["mc"] >= 3: own.update(qWords(row["q"])) return own def indexed(context, *, per=None): """Index immutable request data without sharing derived state between requests or tenants.""" clean = {k: v for k, v in context.items() if k not in ("_fit_words", "_fit_queries", "_fit_states", "_owners", "_kw_rows")} owners = {} for asin, rows in clean["rows"].items(): for row in rows: owners.setdefault(row["q"], []).append(asin) clean.update(_owners=owners, _fit_words={asin: fit_words({"asin": asin}, clean) for asin in clean["rows"]}, _fit_queries={asin: {r["q"] for r in rows if r["mp"] > 0 or r["mc"] >= 3} for asin, rows in clean["rows"].items()}) clean["_fit_states"] = {asin: {r["q"]: fitState63({"asin": asin}, r["q"], clean) for r in rows} for asin, rows in clean["rows"].items()} if per is not None: clean["_kw_rows"] = {clean["asin"]: {per: kwModel55({"asin": clean["asin"]}, per, clean)}} return clean def expo59(v): if not v or not v["ti"]: return None if not v["mi"]: return {"k": "none", "label": "Not seen", "x": None} base = ("purchase", v["mp"], v["tp"]) if v["mp"] >= 2 else ("click", v["mc"], v["tc"]) if v["mc"] >= 10 else None if base is None: return {"k": "few", "label": "Few", "x": None} x = (base[1] / base[2]) / (v["mi"] / v["ti"]) if base[2] else math.inf key, label = ("under", "Under-exposed") if x >= 1.25 else ("over", "Over-exposed") if x <= 0.8 else ("match", "Matched") return {"k": key, "label": label, "x": x, "by": base[0]} def myPrice59(listing, context): rows = [r for r in context.get("rows", {}).get(listing["asin"], []) if r["py"] > 0 and r["mp"] > 0] weight = sum(r["mp"] for r in rows) return sum(r["py"] * r["mp"] for r in rows) / weight if weight else None def sumRows57(rows): return {k: sum(r.get(k, 0) or 0 for r in rows) for k in FIELDS[:11]} if rows else {} def listScore57(rows): scored = [r for r in rows if r["total"] is not None] weight, total = sum(r["vol"] for r in scored), sum(r["vol"] for r in rows) return {"score": sum(r["total"] * r["vol"] for r in scored) / weight if weight else None, "cover": weight / total if total else 0, "n": len(scored)} def trend63(src, context): pts = [(i, (src.get("m", {}).get(m) or [0])[0] or 0) for i, m in enumerate(context["months"])] pts = [(i, v) for i, v in pts if v > 0] if len(pts) < 3: return {"g": None, "last": pts[-1][1] if pts else 0} xb, yb = sum(i for i, _ in pts) / len(pts), sum(math.log(v) for _, v in pts) / len(pts) slope = sum((i - xb) * (math.log(v) - yb) for i, v in pts) / sum((i - xb) ** 2 for i, _ in pts) return {"g": max(-0.3, min(0.4, math.exp(slope) - 1)), "last": math.exp(yb + slope * (len(context["months"]) - 1 - xb))} def at80(src, key, month): x = src.get("m", {}).get(month) if not x: return None if key == "vol": return x[0] if x[0] > 0 else None n, d = {"ms": (_SALES_COL, _MARKET_SALES_COL), "cs": (2, 1)}[key] return (x[n] or 0) / x[d] if x[d] else None def _mon(month): return calendar.month_abbr[int(month[5:7])] def trend80(src, key, context): pts = [(m, at80(src, key, m)) for m in context["months"]] pts = [(m, v) for m, v in pts if v is not None] if not pts: return {"g": None, "why": "Not reported in the last 12 months"} first, last = pts[0], pts[-1] if len(pts) < 2: return {"g": None, "why": f"Reported in {_mon(first[0])} only, so there is no change to work out"} if first[1] <= 0: return {"g": None, "why": f"It starts at 0 in {_mon(first[0])}, so a percent change cannot be worked out"} why = f"{_mon(first[0])} to {_mon(last[0])}" if last[0] != context["months"][-1]: why += f". Amazon did not report it after {_mon(last[0])}" return {"g": last[1] / first[1] - 1, "why": why} def _back12(month): return str(int(month[:4]) - 1) + month[4:] def _like(src, index, months): data = src.get("m") or {} pairs = [(data[m], data[_back12(m)]) for m in months if data.get(m) and data.get(_back12(m))] return {"a": sum(x[index] or 0 for x, _ in pairs), "b": sum(y[index] or 0 for _, y in pairs), "n": len(pairs)} def yoyLL75(src, index, context): return _like(src, index, context["months"]) def trendScore75(src, context): r = yoyLL75(src, 0, context) year = r["a"] / r["b"] - 1 if r["n"] >= 3 and r["b"] > 0 else None trend = trend63(src, context)["g"] parts = [max(-1, min(1, v)) for v in (year, trend / 0.15 if trend is not None else None) if v is not None] return math.floor(50 + 50 * sum(parts) / len(parts) + 0.5) if parts else None def _year_months(context): months = context["months"] year = months[-1][:4] if months else "" ytd = [m for m in months if m.startswith(year)] return year, ytd, [m for m in ytd if _back12(m) in context.get("history", months)] def srch83(src, context): year, _, months = _year_months(context) r = _like(src, 0, months) if r["n"] and r["b"]: a, b = (format(r[k], ",.3f").rstrip("0").rstrip(".") for k in ("a", "b")) return {"g": r["a"] / r["b"] - 1, "why": f"{a} from {b} searches, {r['n']} {'month' if r['n'] == 1 else 'months'} reported in both years"} was = any(src.get("m", {}).get(_back12(m)) for m in months) span = f"{_mon(months[0])} to {_mon(months[-1])}" if months else "" why = "Not reported for you in the same months this year" if was else ( f"Amazon did not report this search for you in {span} {int(year) - 1 if year else -1}") return {"g": None, "why": why} def buy83(src, context): _, _, months = _year_months(context) data = src.get("m") or {} return {"a": sum((data.get(m) or [0] * 5)[4] or 0 for m in months), "b": sum((data.get(_back12(m)) or [0] * 5)[4] or 0 for m in months)} def ytd83(src, index, context): return sum((src.get("m", {}).get(m) or [0] * 13)[index] or 0 for m in _year_months(context)[1]) def stemW(word): word = re.sub("[^a-z0-9]", "", word.lower()) if word.endswith("ss"): return word if re.search("(ches|shes|xes|zes)$", word): return word[:-2] if word.endswith("ies"): return word[:-3] + "y" return word[:-1] if word.endswith("s") else word def foldTea(text): return re.sub(r"\btea\s*-?\s*lights?\b", "tealight", (text or "").lower(), flags=re.ASCII) def wordsOf(text): return {stemW(w) for w in re.split("[^a-z0-9]+", foldTea(text)) if len(w) > 3} def qWords(query): return list(dict.fromkeys(stemW(w) for w in re.split(r"\s+", foldTea(query)) if len(w) > 3))