import { describe, expect, test } from "bun:test" import { buildRetentionQueries, buildStatsQueries, toGeoAggregate, toModelAggregate, toProviderAggregate, toRetentionAggregate, } from "./inference" import { modelAuthor, normalizeInferenceModel, RETIRED_STAT_MODELS, statModel, statProvider, } from "./model-normalization" describe("inference stat normalization", () => { test("normalizes model suffixes used by router/provider variants", () => { expect(normalizeInferenceModel("GPT-5-Free")).toBe("gpt-5") expect(normalizeInferenceModel("deepseek-v4-flash-free")).toBe("deepseek-v4-flash") expect(normalizeInferenceModel("deepseek-v4-flash:global")).toBe("deepseek-v4-flash") expect(normalizeInferenceModel("mimo-v2.5-free")).toBe("mimo-v2.5") expect(normalizeInferenceModel("nemotron-3-super-free")).toBe("nemotron-3-super") expect(normalizeInferenceModel("mimo-v2.5-free:global")).toBe("mimo-v2.5") expect(normalizeInferenceModel("hy3-preview:free")).toBe("hy3-preview") }) test("maps normalized model ids to public authors", () => { expect(modelAuthor("big-pickle")).toBe("unknown") expect(modelAuthor("claude-sonnet-4-5")).toBe("anthropic") expect(modelAuthor("deepseek-v4-pro")).toBe("deepseek") expect(modelAuthor("gemini-3.5-flash")).toBe("google") expect(modelAuthor("glm-5.1")).toBe("zhipu") expect(modelAuthor("gpt-5.5-pro")).toBe("openai") expect(modelAuthor("grok-build-0.1")).toBe("xai") expect(modelAuthor("hy3-preview")).toBe("tencent") expect(modelAuthor("kimi-k2.6")).toBe("moonshot") expect(modelAuthor("mimo-v2-omni")).toBe("xiaomi") expect(modelAuthor("minimax-m2.7")).toBe("minimax") expect(modelAuthor("muse-spark-1.2-contributor")).toBe("meta") expect(modelAuthor("nemotron-3-super-free")).toBe("nvidia") expect(modelAuthor("qwen3.7-max")).toBe("qwen") expect(modelAuthor("alpha-gpt-next")).toBeUndefined() expect(modelAuthor("omen-alpha")).toBe("unknown") expect(modelAuthor("OMEN-ALPHA-free:global")).toBe("unknown") }) test("uses provider.model to resolve opencode route providers", () => { expect(statModel("big-pickle", "claude-sonnet-4-5")).toBe("claude-sonnet-4-5") expect(statModel("big-pickle", "gpt-5-free")).toBe("gpt-5") expect(statModel("big-pickle", "xiaomi/mimo-v2.5")).toBe("mimo-v2.5") expect(statModel("big-pickle", "")).toBe("unknown") expect(statProvider("big-pickle", "claude-sonnet-4-5", "opencode")).toBe("anthropic") expect(statProvider("big-pickle", "gpt-5", "opencode")).toBe("openai") expect(statProvider("big-pickle", "", "opencode")).toBe("unknown") expect(statProvider("unknown", "", "custom-provider")).toBe("custom-provider") }) test("keeps stealth model usage without exposing the route provider", () => { expect(statProvider("omen-alpha", "gpt-test-model", "test-provider")).toBe("unknown") expect(statProvider("OMEN-ALPHA-free:global", "gpt-test-model", "test-provider")).toBe("unknown") expect(statProvider("omen-alpha", "", "test-provider")).toBe("unknown") const row = { ...aggregate("omen-alpha", "test-provider"), provider_model: "gpt-test-model" } expect(toModelAggregate(row)).toMatchObject([{ model: "omen-alpha", provider: "unknown", requests: 1 }]) expect(toProviderAggregate(row)).toMatchObject([{ provider: "unknown", requests: 1 }]) expect(toGeoAggregate({ ...row, country: "US" })).toMatchObject([ { model: "omen-alpha", provider: "unknown", country: "US", requests: 1 }, ]) expect(toRetentionAggregate({ ...row, cohort_date: "2026-08-10", eligible_users: "12" })).toMatchObject([ { model: "omen-alpha", provider: "unknown", eligibleUsers: 12 }, ]) }) test("merges renamed models under their current name", () => { expect(statModel("deepseek-v4-flash-0731", "")).toBe("deepseek-v4-flash") expect(statModel("deepseek-v4-flash-0731-free", "")).toBe("deepseek-v4-flash") expect(statModel("deepseek-v4-flash-dsv4-flash-final-rnaovd", "")).toBe("deepseek-v4-flash") expect(statModel("deepseek-v4-flash-vision-exp", "")).toBe("deepseek-v4-flash-vision-exp") expect(statModel("x-preview-f", "")).toBe("glm-5.3-flash") expect(statModel("ox-alpha", "")).toBe("glm-5.3-flash") expect(statModel("ox-alpha-free", "")).toBe("glm-5.3-flash") expect(statModel("big-pickle", "zhipuai/ox-alpha-free")).toBe("glm-5.3-flash") expect(statModel("xiaomi/mimo-v2.5", "")).toBe("mimo-v2.5") expect(toModelAggregate(aggregate("x-preview-f", "unknown"))).toMatchObject([ { provider: "zhipu", model: "glm-5.3-flash", }, ]) expect(toProviderAggregate(aggregate("ox-alpha", "unknown"))).toMatchObject([{ provider: "zhipu" }]) }) test("renames DeepSeek Flash to V4.1 without merging V4 or vision usage", () => { ;["deepseek-flash", "DEEPSEEK-FLASH-free:global", "deepseek-v4.1-flash"].forEach((model) => { expect(statModel(model, "")).toBe("deepseek-v4.1-flash") expect(toModelAggregate(aggregate(model, "deepseek"))).toMatchObject([ { model: "deepseek-v4.1-flash", provider: "deepseek" }, ]) expect(toGeoAggregate({ ...aggregate(model, "deepseek"), country: "US" })).toMatchObject([ { model: "deepseek-v4.1-flash", provider: "deepseek", country: "US" }, ]) expect(toRetentionAggregate({ ...aggregate(model, "deepseek"), cohort_date: "2026-08-10" })).toMatchObject([ { model: "deepseek-v4.1-flash", provider: "deepseek" }, ]) }) expect(statModel("big-pickle", "deepseek/deepseek-flash")).toBe("deepseek-v4.1-flash") expect(statModel("deepseek-v4-flash", "")).toBe("deepseek-v4-flash") expect(statModel("deepseek-v4-flash-vision-exp", "")).toBe("deepseek-v4-flash-vision-exp") expect(RETIRED_STAT_MODELS).toContain("deepseek-flash") expect(RETIRED_STAT_MODELS).not.toContain("deepseek-v4.1-flash") }) test("model aggregates prefer provider.model and use normalized model", () => { expect(toModelAggregate(aggregate("alpha-gpt-next", "openai"))).toEqual([]) expect(toModelAggregate(aggregate("deepseek-v4-flash-free", "not-public-provider"))).toMatchObject([ { period_key: "2026-05-20", provider: "deepseek", model: "deepseek-v4-flash", }, ]) expect( toModelAggregate({ ...aggregate("big-pickle", "opencode"), provider_model: "claude-sonnet-4-5" }), ).toMatchObject([ { provider: "anthropic", model: "claude-sonnet-4-5", provider_model: "claude-sonnet-4-5", }, ]) }) test("provider aggregates never keep opencode as the provider", () => { expect(toProviderAggregate({ ...aggregate("big-pickle", "opencode"), provider_model: "gpt-5" })).toMatchObject([ { provider: "openai" }, ]) expect(toProviderAggregate(aggregate("big-pickle", "opencode"))).toMatchObject([{ provider: "unknown" }]) expect(toProviderAggregate(aggregate("muse-spark-1.2-contributor", "unknown"))).toMatchObject([ { provider: "meta" }, ]) }) test("geo aggregates never keep opencode or big-pickle dimensions", () => { expect(toGeoAggregate({ ...aggregate("big-pickle", "opencode"), country: "US" })).toMatchObject([ { provider: "unknown", model: "unknown", country: "US" }, ]) }) test("model aggregates use ISO week period keys", () => { expect( toModelAggregate({ ...aggregate("gpt-5.5-pro", "openai"), grain: "week", period_key: "2026-W20", }), ).toMatchObject([{ period_key: "2026-W20" }]) }) test("builds bounded R2 SQL queries for each day and week", () => { const queries = buildStatsQueries(new Date("2026-08-10T00:00:00.000Z"), new Date("2026-08-12T12:00:00.000Z"), { namespace: "inference", table: "generation", dataset: "zen", }) expect(queries).toHaveLength(8) queries.forEach((query) => { expect(query).toContain("WHERE lower(model) NOT IN ('alpha-gpt-next')") expect(query).toContain("CASE\n WHEN lower(model) IN ('omen-alpha') THEN 'unknown'\n") expect(query).toContain("= 'deepseek-flash' THEN 'deepseek-v4.1-flash'") }) expect(queries[0]).toContain("'week' AS grain") expect(queries[0]).toContain("'2026-W33' AS period_key") expect(queries[2]).toContain("'2026-08-10' AS period_key") expect(queries[6]).toContain("'2026-08-12' AS period_key") expect(queries[0]).toContain('FROM "inference"."generation"') expect(queries[0]).toContain("event_type = 'generation.completed'") expect(queries[0]).toContain("AND (product = 'go' OR (lower(COALESCE(model_tier, '')) = 'free'") expect(queries[0]).toContain("COALESCE(NULLIF(lower(model_tier), ''), '') AS raw_tier") expect(queries[0]).toContain("WHEN lower(COALESCE(raw_tier, '')) = 'free'") expect(queries[0]).toContain("regexp_replace(NULLIF(route_model, ''), '^.*/', '')") expect(queries[0]).toContain("= 'deepseek-v4-flash-0731' THEN 'deepseek-v4-flash'") expect(queries[0]).toContain("= 'deepseek-v4-flash-dsv4-flash-final-rnaovd' THEN 'deepseek-v4-flash'") expect(queries[0]).not.toContain("= 'deepseek-v4-flash-vision-exp' THEN 'deepseek-v4-flash'") expect(queries[0]).toContain("= 'ox-alpha' THEN 'glm-5.3-flash'") expect(queries[0]).toContain("= 'x-preview-f' THEN 'glm-5.3-flash'") expect(queries[0]).toContain("OR lower(raw_model) IN ('gpt-5-nano', 'grok-code', 'big-pickle')") expect(queries[0]).toContain("OR lower(raw_model) LIKE '%-free'") expect(queries[0]).toContain("THEN 'Free'") expect(queries[0]).toContain("LIMIT 10000") expect(queries[0]).toContain("approx_distinct(session) AS sessions") expect(queries[1]).toContain("'geo_model' ELSE 'geo'") expect(queries[1]).toContain("0 AS sessions") }) test("aligns periods to UTC calendar boundaries", () => { const queries = buildStatsQueries(new Date("2026-06-17T15:56:00.000Z"), new Date("2026-06-19T15:56:00.000Z"), { namespace: "inference", table: "generation", dataset: "zen", }) expect(queries).toHaveLength(8) expect(queries[0]).toContain("'2026-W25' AS period_key") expect(queries[0]).toContain("started_at >= '2026-06-15T00:00:00.000Z'") expect(queries[2]).toContain("'2026-06-17' AS period_key") expect(queries[2]).toContain("started_at >= '2026-06-17T00:00:00.000Z'") expect(queries[2]).toContain("started_at < '2026-06-18T00:00:00.000Z'") expect(queries[6]).toContain("'2026-06-19' AS period_key") expect(queries[6]).toContain("started_at < '2026-06-19T15:56:00.000Z'") }) test("uses an exclusive live and legacy source handoff", () => { const [query] = buildStatsQueries(new Date("2026-08-11T00:00:00.000Z"), new Date("2026-08-12T00:00:00.000Z"), { namespace: "inference", table: "generation", dataset: "zen", }) expect(query).toContain("(source = 'inference-legacy' AND started_at < '2026-08-11T10:57:48.186Z')") expect(query).toContain("(source = 'inference' AND started_at >= '2026-08-11T10:57:48.186Z')") }) test("builds complete week-over-week retention queries", () => { const queries = buildRetentionQueries(new Date("2026-08-10T00:00:00.000Z"), new Date("2026-08-31T00:00:00.000Z"), { namespace: "inference", table: "generation", dataset: "zen", }) expect(queries).toHaveLength(2) queries.forEach(({ query }) => { expect(query).toContain("= 'deepseek-flash' THEN 'deepseek-v4.1-flash'") }) expect(queries.map((query) => query.cohortDates)).toEqual([["2026-08-10"], ["2026-08-17"]]) expect(queries[0]?.query).toContain("AND product = 'go'") expect(queries[0]?.query).toContain("AND lower(model) NOT IN ('alpha-gpt-next')") expect(queries[0]?.query).toContain("CASE\n WHEN lower(model) IN ('omen-alpha') THEN 'unknown'\n") expect(queries[0]?.query).toContain("COUNT(*) AS model_requests") expect(queries[0]?.query).toContain("SUM(model_requests) AS total_requests") expect(queries[0]?.query).toContain("MAX(model_requests) AS max_model_requests") expect(queries[0]?.query).toContain("GROUP BY cohort_date, user_key") expect(queries[0]?.query).toContain("INNER JOIN user_totals") expect(queries[0]?.query).toContain("model_usage.model_requests = user_totals.max_model_requests") expect(queries[0]?.query).toContain("user_totals.total_requests >= 10") expect(queries[0]?.query).toContain( "CAST(model_usage.model_requests AS double) / NULLIF(user_totals.total_requests, 0) >= 0.8", ) expect(queries[0]?.query).not.toContain(" OVER (") expect(queries[0]?.query).toContain("WHEN '2026-08-17' THEN '2026-08-10'") expect(queries[0]?.query).not.toContain("WHEN '2026-08-24' THEN '2026-08-17'") expect(queries[1]?.query).toContain("WHEN '2026-08-24' THEN '2026-08-17'") expect(queries[0]?.query).toContain("started_at >= '2026-08-10T00:00:00.000Z'") expect(queries[0]?.query).toContain("started_at < '2026-08-24T00:00:00.000Z'") expect(queries[1]?.query).toContain("started_at >= '2026-08-17T00:00:00.000Z'") expect(queries[1]?.query).toContain("started_at < '2026-08-31T00:00:00.000Z'") expect(queries[0]?.query).toContain("LEFT JOIN returned ON primary_models.user_key = returned.user_key") expect(queries[0]?.query).toContain("primary_models.cohort_date = returned.cohort_date") expect(queries[0]?.query).toContain("'Go' AS tier") expect(queries[0]?.query).toContain("COUNT(*) AS eligible_users") expect(queries[0]?.query).toContain("LIMIT 10000") }) test("splits a full retention window without dropping or duplicating cohorts", () => { const source = { namespace: "inference", table: "generation", dataset: "zen" } const queries = buildRetentionQueries(new Date("2026-07-16T19:00:00Z"), new Date("2026-09-10T00:00:00Z"), source) expect(queries.flatMap((query) => query.cohortDates)).toEqual([ "2026-07-13", "2026-07-20", "2026-07-27", "2026-08-03", "2026-08-10", "2026-08-17", "2026-08-24", ]) queries.forEach((query) => { const start = new Date(`${query.cohortDates[0]}T00:00:00Z`) const end = new Date(start.getTime() + 14 * 86_400_000) expect(query).toEqual(buildRetentionQueries(start, end, source)[0]) }) expect(buildRetentionQueries(new Date("2026-08-31T00:00:00Z"), new Date("2026-09-10T00:00:00Z"), source)).toEqual( [], ) }) test("maps retention query results", () => { expect( toRetentionAggregate({ cohort_date: "2026-08-10", dataset: "zen", tier: "all", provider: "deepseek", model: "deepseek-v4-flash-free", eligible_users: "125", retained_users: "74", }), ).toEqual([ { cohortDate: "2026-08-10", dataset: "zen", tier: "all", provider: "deepseek", model: "deepseek-v4-flash", eligibleUsers: 125, retainedUsers: 74, }, ]) }) }) function aggregate(model: string, provider: string) { return { grain: "day", period_key: "2026-05-20", dataset: "zen", tier: "Paid", provider, model, sessions: "1", requests: "1", sample_count: "1", } }