feat(nepse): long daily history from Merolagani under the exchange and archive

src/history.ts

@@ -148,9 +148,11 @@ export async function fetchHistory(
     }
 
     // NEPSE never goes through the seed cache below: its latest session changes
-    // with every poll, and nepse/history.ts caches its own upstream (the
-    // archive) while reading the live session from memory. Nothing older than
-    // the archive exists to backfill.
+    // with every poll, and nepse/history.ts caches its own upstreams (the
+    // archive, the exchange's index history and the long series) while reading
+    // the live session from memory. The seed is the whole series — at most
+    // ~6,700 daily bars, the index since 1997, the size of an equity's daily
+    // snapshot — so there is nothing left to backfill.
     if (klass === AssetClass.Nepse) {
         const seed = await nepseSeed(symbol, tf);
         return { initial: seed.bars, backfillFn: () => empty(), session: seed.session };

src/nepse/archive.ts

@@ -6,7 +6,8 @@
  * static JSON on GitHub Pages, regenerated every trading evening (~17:00 NPT),
  * from 2024-03-04, ~520 symbols. It has no index series (history/NEPSE.json is
  * a 404); the index's daily history comes from the exchange instead
- * (index-history.ts).
+ * (index-history.ts). Everything before 2024 comes from Merolagani's long
+ * series underneath (long-history.ts); this archive's figures win on its dates.
  *
  * Decoding is a port of `decodeNepseHistory` + `collapseCarryForward` in the
  * web app's packages/domain/src/nepse.ts — the server cannot import that

src/nepse/history.ts

@@ -6,9 +6,13 @@
  *                  it holds that session, else the history's own row). Weekly and
  *                  monthly bars come with the overlay the engine needs to keep
  *                  replacing the live session's share as it trades. A company's
- *                  sessions come from the public archive (years of them); the
- *                  index's from the exchange's own index history (the last
- *                  year — the archive has no index series).
+ *                  recent sessions come from the public archive (since March
+ *                  2024); the index's from the exchange's own index history (the
+ *                  last year — the archive has no index series). Under both lies
+ *                  Merolagani's long series (long-history.ts): the index since
+ *                  1997, a company since 2010 or its listing. The recent source
+ *                  wins on every date it covers; the long one only fills the
+ *                  years before it.
  *
  *   1m … 4H        the latest session only, in one-minute bars (the floorsheet
  *                  rebuild for what the feed did not watch, then its tape),
@@ -29,6 +33,7 @@ import type { SessionOverlay } from '../engine.js';
 import { nepseTicker } from '../symbols.js';
 import { nepseFeed } from '../feeds/nepse.js';
 import { archiveHistory, type ArchiveSession } from './archive.js';
+import { longHistory, mergeSessions } from './long-history.js';
 import { sessionBook, dayToBar, INDEX_KEY } from './book.js';
 import { nepseBucketStart, sessionTimeSec } from './time.js';
 
@@ -98,7 +103,12 @@ export async function nepseSeed(symbol: string, tf: Timeframe): Promise<NepseSee
     if (!ticker) return { bars: [] };
 
     if (tf === '1D' || tf === '1W' || tf === '1M') {
-        const sessions = (ticker === INDEX_KEY ? await nepseFeed.indexSessions() : await archiveHistory(ticker)) ?? [];
+        const index = ticker === INDEX_KEY;
+        const [recent, long] = await Promise.all([
+            index ? nepseFeed.indexSessions() : archiveHistory(ticker),
+            longHistory(ticker, index),
+        ]);
+        const sessions = mergeSessions(long, recent);
         // ---- synchronous from here (see header) ----
         return composeSessionSeed(sessions, sessionBook.day(ticker), tf);
     }

src/nepse/index-history.ts

@@ -6,7 +6,9 @@
  * daily view was a single bar. The exchange publishes the index's own daily
  * sessions (`/api/nots/index/history/58`) — open, high, low, close and
  * turnover per business date, the figures its own site charts — covering the
- * last year (230 sessions on 2026-09-25). One request, a few kilobytes.
+ * last year (230 sessions on 2026-09-25; paging and date parameters are
+ * ignored). One request, a few kilobytes. The years before come from
+ * Merolagani's long series (long-history.ts); these figures win on their dates.
  *
  * CACHED SIX HOURS, like the archive: the history gains one row per trading
  * evening, and the session still trading comes from the session book anyway

src/nepse/long-history.ts

@@ -0,0 +1,268 @@
+/**
+ * NEPSE's long daily history — Merolagani's chart feed.
+ *
+ * ===========================================================================
+ * WHY A THIRD SOURCE
+ * ===========================================================================
+ *
+ * The exchange serves exactly ONE YEAR of daily history, for the index and for
+ * every company alike, and ignores every paging and date parameter. Verified
+ * 2026-09-25 against `/api/nots/index/history/58` and
+ * `/api/nots/market/security/price/{id}`: `page`, `size`, `startDate` and
+ * `endDate` all return the same 230 sessions (2025-09-23 → 2026-09-24) or an
+ * empty page. The public archive (archive.ts) starts on 2024-03-04. So the
+ * index chart began a year ago and a company's two and a half years ago, on an
+ * exchange that has traded since 1994.
+ *
+ * Merolagani's chart feed — the one its own charts draw from, in TradingView's
+ * UDF shape `{ s, t[], o[], h[], l[], c[], v[] }` — holds the index from July
+ * 1997 and each company from 2010 or from its listing if later. Asked for
+ * UNADJUSTED figures (`isAdjust=0`), they are the exchange's own prices:
+ * compared the same day with the exchange's history, all 230 overlapping index
+ * sessions were equal in open, high, low and close, and NABIL's 229 in prices
+ * and volumes (one open differed by Rs 2, identically in every third-party
+ * source). Unadjusted, like the archive and the live feed, so a bonus-share
+ * date is the same step on every source and a chart never mixes the two.
+ *
+ * ===========================================================================
+ * HOW IT IS USED
+ * ===========================================================================
+ *
+ * As the FLOOR under the other sources, never over them (`mergeSessions`): on
+ * any date the exchange's index history or the archive also covers, theirs is
+ * the figure. What this adds is only the years they do not have — and the odd
+ * session the archive missed. If Merolagani is down, blocks the request or
+ * changes its shape, the chart is exactly what it was before this module: the
+ * other sources stand on their own. `NEPSE_LONG_HISTORY_ENABLED=false` switches
+ * it off without a deploy.
+ *
+ * ===========================================================================
+ * WHAT THE PAYLOAD SAYS, AND HOW IT IS READ
+ * ===========================================================================
+ *
+ *   - DATES. Each bar is stamped at some moment of its business date in UTC
+ *     (05:45Z in the older years, 20:45Z lately, a handful elsewhere), so the
+ *     UTC calendar date IS the business date. Checked against the exchange:
+ *     all of its 230 dates present, none added.
+ *   - THE INDEX BEFORE MID-2016 IS CLOSES ONLY (open = high = low = close). It
+ *     is drawn as that — a flat bar is what a close-only day is — rather than
+ *     given an invented range.
+ *   - The index's `v` is rupee turnover, not a quantity; index bars carry no
+ *     volume anywhere else in Strix, so it is dropped (see index-history.ts).
+ *   - A few old bars have a high below their open or close; the range is
+ *     widened to contain both, as the archive decoder does.
+ *   - Figures are rounded to the exchange's two decimals (older index values
+ *     arrive with five).
+ *   - `{ s: "no_data" }` is a real answer — a symbol the feed does not carry
+ *     (debentures with a slash in the ticker, a listing it has not picked up).
+ *     It is cached like a success, so a chart of such a security does not ask
+ *     again on every subscribe.
+ *
+ * ===========================================================================
+ * COST
+ * ===========================================================================
+ *
+ * One request per security, only when a daily/weekly/monthly chart of it is
+ * opened and the six-hour copy has expired: at most four a day per security
+ * anybody watches, a few hundred kilobytes at most (the index's 29 years). A
+ * failure is remembered for a minute, so an outage costs one timeout per
+ * security per minute, not one per chart opened.
+ */
+
+import { createLogger } from '../log.js';
+import type { ArchiveSession } from './archive.js';
+
+const log = createLogger('nepse-long-history');
+
+const BASE = process.env.NEPSE_LONG_HISTORY_BASE
+    ?? 'https://merolagani.com/handlers/TechnicalChartHandler.ashx';
+
+const TTL_MS = 6 * 60 * 60 * 1000;
+const RETRY_AFTER_FAILURE_MS = 60_000;
+/** ~0.4 MB for a company, ~0.8 MB for the index: this bounds the cache near 50 MB. */
+const MAX_ENTRIES = 100;
+const TIMEOUT_MS = 10_000;
+
+/** 1990-01-01 — before the exchange's first session (1994). */
+const RANGE_START_SEC = 631_152_000;
+
+/** The feed's tickers are letters and digits; anything else it does not carry. */
+const TICKER = /^[A-Z0-9]{1,20}$/;
+
+export function longHistoryEnabled(): boolean {
+    return process.env.NEPSE_LONG_HISTORY_ENABLED !== 'false';
+}
+
+function num(v: unknown): number | null {
+    return typeof v === 'number' && Number.isFinite(v) ? v : null;
+}
+
+const round2 = (v: number): number => Math.round(v * 100) / 100;
+
+/**
+ * Decode one chart-feed payload into daily sessions, oldest first, one per
+ * date. `index` drops the volume (see the header). Pure — exported for tests.
+ *
+ * An empty result means "nothing usable" — `no_data`, a shape change, or
+ * arrays that disagree in length (read positionally, a shorter array would
+ * shift every later field onto the wrong day, so the payload is refused whole).
+ */
+export function decodeChartHistory(payload: unknown, index: boolean, nowMs: number = Date.now()): ArchiveSession[] {
+    const p = (payload ?? {}) as Record<string, unknown>;
+    if (p.s !== 'ok') return [];
+    const cols = ['t', 'o', 'h', 'l', 'c', 'v'].map((k) => p[k]);
+    if (!cols.every(Array.isArray)) return [];
+    const [t, o, h, l, c, v] = cols as unknown[][];
+    if (!cols.every((col) => (col as unknown[]).length === t.length)) return [];
+
+    // A bar stamped more than two days ahead of now is not a session.
+    const latestSec = Math.floor(nowMs / 1000) + 2 * 86_400;
+    const byDate = new Map<string, ArchiveSession>();
+    for (let i = 0; i < t.length; i++) {
+        const time = num(t[i]);
+        const close = num(c[i]);
+        if (time === null || time < RANGE_START_SEC || time > latestSec || close === null || close <= 0) continue;
+        const date = new Date(time * 1000).toISOString().slice(0, 10);
+        // Missing open/high/low fall back to the close, never to zero: a zero
+        // low draws a spike to the axis that no session ever made.
+        const open = num(o[i]) ?? 0;
+        const high = num(h[i]) ?? 0;
+        const low = num(l[i]) ?? 0;
+        const o2 = round2(open > 0 ? open : close);
+        const c2 = round2(close);
+        const h2 = round2(high > 0 ? high : close);
+        const l2 = round2(low > 0 ? low : close);
+        const vol = index ? 0 : Math.max(0, num(v[i]) ?? 0);
+        byDate.set(date, {
+            date,
+            open: o2,
+            high: Math.max(h2, o2, c2),
+            low: Math.min(l2, o2, c2),
+            close: c2,
+            volume: vol,
+            turnover: 0,
+        });
+    }
+    return [...byDate.values()].sort((a, b) => a.date.localeCompare(b.date));
+}
+
+/**
+ * The long series with the recent sources laid over it. `recent` — the
+ * exchange's index history, or the archive for a company — is the figure on
+ * every date it covers; `long` fills only the dates it does not. Either may be
+ * missing. Oldest first. Pure — exported for tests.
+ */
+export function mergeSessions(
+    long: readonly ArchiveSession[] | null,
+    recent: readonly ArchiveSession[] | null,
+): ArchiveSession[] {
+    if (!long || long.length === 0) return recent ? [...recent] : [];
+    if (!recent || recent.length === 0) return [...long];
+    const byDate = new Map<string, ArchiveSession>();
+    for (const s of long) byDate.set(s.date, s);
+    for (const s of recent) byDate.set(s.date, s);
+    return [...byDate.values()].sort((a, b) => a.date.localeCompare(b.date));
+}
+
+/** The request for one ticker. Exported for tests. */
+export function longHistoryUrl(ticker: string, nowMs: number = Date.now()): string {
+    const params = new URLSearchParams({
+        type: 'get_advanced_chart',
+        symbol: ticker,
+        resolution: '1D',
+        rangeStartDate: String(RANGE_START_SEC),
+        rangeEndDate: String(Math.floor(nowMs / 1000) + 2 * 86_400),
+        from: '',
+        isAdjust: '0',
+        currencyCode: 'NPR',
+    });
+    return `${BASE}?${params.toString()}`;
+}
+
+interface Entry { sessions: ArchiveSession[]; fetchedAt: number; }
+
+/**
+ * A six-hour cache of the feed, one entry per ticker, with failures remembered
+ * for a minute. The fetch is injected so the cache is testable without the
+ * network.
+ */
+export class LongHistoryCache {
+    private readonly entries = new Map<string, Entry>();
+    private readonly failedAt = new Map<string, number>();
+    private readonly inflight = new Map<string, Promise<ArchiveSession[] | null>>();
+
+    constructor(
+        private readonly fetchPayload: (ticker: string) => Promise<unknown>,
+        private readonly clock: () => number = Date.now,
+    ) {}
+
+    /** Sessions oldest first, [] for a ticker the feed does not carry, null when it cannot be read. */
+    get(tickerRaw: string, index: boolean): Promise<ArchiveSession[] | null> {
+        const ticker = tickerRaw.toUpperCase();
+        if (!TICKER.test(ticker)) return Promise.resolve([]);
+        const now = this.clock();
+        const hit = this.entries.get(ticker);
+        if (hit && now - hit.fetchedAt < TTL_MS) return Promise.resolve(hit.sessions);
+        const failed = this.failedAt.get(ticker);
+        if (failed !== undefined && now - failed < RETRY_AFTER_FAILURE_MS) return Promise.resolve(hit?.sessions ?? null);
+        const running = this.inflight.get(ticker);
+        if (running) return running;
+
+        const p = (async (): Promise<ArchiveSession[] | null> => {
+            try {
+                const payload = await this.fetchPayload(ticker);
+                const noData = (payload as { s?: unknown } | null)?.s === 'no_data';
+                const sessions = noData ? [] : decodeChartHistory(payload, index, this.clock());
+                // Anything but a clean "no data" that decodes to nothing is a
+                // shape change or a bad answer far more often than a real empty
+                // series: never let it replace a good copy, never cache it.
+                if (!noData && sessions.length === 0) throw new Error('unreadable payload');
+                this.entries.delete(ticker);
+                this.entries.set(ticker, { sessions, fetchedAt: this.clock() });
+                this.failedAt.delete(ticker);
+                while (this.entries.size > MAX_ENTRIES) {
+                    const oldest = this.entries.keys().next().value;
+                    if (oldest === undefined) break;
+                    this.entries.delete(oldest);
+                }
+                return sessions;
+            } catch (e) {
+                this.failedAt.set(ticker, this.clock());
+                log.warn(`${ticker} failed:`, e instanceof Error ? e.message : e);
+                // A stale series beats none: it is history, and history does not change.
+                return hit?.sessions ?? null;
+            } finally {
+                this.inflight.delete(ticker);
+            }
+        })();
+        this.inflight.set(ticker, p);
+        return p;
+    }
+}
+
+async function fetchChartPayload(ticker: string): Promise<unknown> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
+    try {
+        const res = await fetch(longHistoryUrl(ticker), {
+            signal: controller.signal,
+            headers: { Accept: 'application/json, text/plain, */*' },
+        });
+        if (!res.ok) throw new Error(`HTTP ${res.status}`);
+        // Served as text/plain; parse the body ourselves.
+        return JSON.parse(await res.text());
+    } finally {
+        clearTimeout(timer);
+    }
+}
+
+const cache = new LongHistoryCache(fetchChartPayload);
+
+/**
+ * A ticker's long daily history (`NEPSE` for the index), oldest first; [] when
+ * the feed does not carry it, null when it cannot be read or is switched off.
+ */
+export function longHistory(ticker: string, index: boolean): Promise<ArchiveSession[] | null> {
+    if (!longHistoryEnabled()) return Promise.resolve(null);
+    return cache.get(ticker, index);
+}

test/nepse.test.ts

@@ -24,6 +24,8 @@ import {
 import { isTickAcceptable, isBucketAligned } from '../src/calendar.js';
 import { setNepseSession, noteNepsePoll } from '../src/nepse/session-state.js';
 import { indexDaysToSessions, IndexHistoryCache } from '../src/nepse/index-history.js';
+import { decodeChartHistory, mergeSessions, longHistoryUrl, LongHistoryCache } from '../src/nepse/long-history.js';
+import type { ArchiveSession } from '../src/nepse/archive.js';
 import type { IndexDay } from '../src/nepse/client.js';
 import { sessionBook } from '../src/nepse/book.js';
 import { buildQuote, NEPSE_SESSION_CLOSE_MAX_AGE_MS, type QuoteRow } from '../src/quotes.js';
@@ -412,5 +414,92 @@ console.log('\n[13] NEPSE quotes are read from the session book');
     setNepseSession(false, null);
 }
 
+console.log('\n[14] the long series (Merolagani\'s chart feed) under the recent sources');
+{
+    // Stamps as the feed sends them (2026-09-25): 20:45Z lately, 05:45Z in
+    // older years — both on the business date in UTC.
+    const stamp = (date: string, hhmm = '20:45') => Date.parse(`${date}T${hhmm}:00Z`) / 1000;
+    const now = Date.parse('2026-09-25T12:00:00Z');
+    const payload = {
+        s: 'ok',
+        t: [stamp('2010-01-04', '05:45'), stamp('2016-03-02', '05:45'), stamp('2026-09-23'), stamp('2026-09-24'), stamp('2026-09-24'), stamp('2026-10-10'), stamp('2026-09-22')],
+        o: [543.2, 1400, 2670.28, 2611.03, 2611.03, 2700, 2650],
+        h: [543.2, 1398.5, 2671.04, 2629.94, 2629.94, 2700, 2662.71],
+        l: [543.2, 1390, 2613.13, 2605.32, 2605.32, 2700, 2637.57],
+        c: [543.20417, 1396.12345, 2618.03, 2629.81, 2629.81, 2700, 0],
+        v: [0, 5e8, 6356611395.73, 5447313168.2, 5447313168.2, 1, 7e9],
+    };
+    const idx = decodeChartHistory(payload, true, now);
+    check('one session per business date, oldest first', JSON.stringify(idx.map((s) => s.date)) === '["2010-01-04","2016-03-02","2026-09-23","2026-09-24"]', JSON.stringify(idx.map((s) => s.date)));
+    check('a 20:45Z stamp is its own UTC date (the exchange\'s 2026-09-24 session)',
+        idx[3]?.close === 2629.81 && idx[3]?.open === 2611.03 && idx[3]?.high === 2629.94 && idx[3]?.low === 2605.32);
+    check('a close-only day stays flat, rounded to two decimals', idx[0]?.open === 543.2 && idx[0]?.close === 543.2 && idx[0]?.high === 543.2);
+    check('a high below the open is widened to contain it', idx[1]?.high === 1400 && idx[1]?.close === 1396.12);
+    check('the index carries no volume (its v is rupee turnover)', idx.every((s) => s.volume === 0));
+    check('a bar stamped weeks ahead is not a session', !idx.some((s) => s.date === '2026-10-10'));
+    check('a zero close is dropped, not drawn to the axis', !idx.some((s) => s.date === '2026-09-22'));
+    const company = decodeChartHistory({ ...payload, t: payload.t.slice(2, 4), o: payload.o.slice(2, 4), h: payload.h.slice(2, 4), l: payload.l.slice(2, 4), c: payload.c.slice(2, 4), v: [88903, 70749] }, false, now);
+    check('a company keeps its traded quantity', company[1]?.volume === 70749);
+    check('no_data decodes to nothing', decodeChartHistory({ s: 'no_data', nextTime: 1 }, false, now).length === 0);
+    check('arrays of different lengths are refused whole', decodeChartHistory({ ...payload, c: payload.c.slice(1) }, false, now).length === 0);
+
+    const url = new URL(longHistoryUrl('NABIL', now));
+    check('asks for unadjusted daily bars of the ticker', url.searchParams.get('isAdjust') === '0'
+        && url.searchParams.get('resolution') === '1D' && url.searchParams.get('symbol') === 'NABIL');
+    check('…from before the exchange opened', Number(url.searchParams.get('rangeStartDate')) < Date.UTC(1994, 0, 1) / 1000);
+
+    const s = (date: string, close: number): ArchiveSession => ({ date, open: close, high: close, low: close, close, volume: 1, turnover: 0 });
+    const long = [s('2010-01-04', 1), s('2024-03-04', 2), s('2024-03-05', 3), s('2024-03-06', 4)];
+    const recent = [s('2024-03-04', 20), s('2024-03-06', 40), s('2024-03-07', 50)];
+    const merged = mergeSessions(long, recent);
+    check('the recent source wins on every date it covers', merged.find((m) => m.date === '2024-03-04')?.close === 20 && merged.find((m) => m.date === '2024-03-06')?.close === 40);
+    check('the long series fills the years before, and a session the archive missed',
+        merged[0]?.date === '2010-01-04' && merged.find((m) => m.date === '2024-03-05')?.close === 3);
+    check('merged oldest first, one per date', JSON.stringify(merged.map((m) => m.date)) === '["2010-01-04","2024-03-04","2024-03-05","2024-03-06","2024-03-07"]');
+    check('either side missing is the other alone', mergeSessions(null, recent).length === 3 && mergeSessions(long, null).length === 4 && mergeSessions(null, null).length === 0);
+
+    // The seed composes the merged series exactly as before.
+    const seed = composeSessionSeed(merged, null, '1M');
+    check('a monthly seed spans the long series (January 2010, March 2024)',
+        seed.bars.length === 2 && seed.bars[0]?.time === nepseBucketStart('1M', Date.parse('2010-01-04T06:00:00Z')) && seed.bars[1]?.close === 50,
+        JSON.stringify(seed.bars));
+
+    // The cache: TTL, no_data, failures, coalescing.
+    let t = Date.parse('2026-09-25T12:00:00Z');
+    let calls = 0;
+    let mode: 'ok' | 'no_data' | 'fail' | 'garbage' = 'ok';
+    const cache = new LongHistoryCache(async () => {
+        calls++;
+        if (mode === 'fail') throw new Error('down');
+        if (mode === 'garbage') return { s: 'ok', t: [1] };
+        if (mode === 'no_data') return { s: 'no_data' };
+        return payload;
+    }, () => t);
+    const [a, b] = await Promise.all([cache.get('nepse', true), cache.get('NEPSE', true)]);
+    check('two charts opening together share one request', calls === 1 && a === b && (a?.length ?? 0) === 4);
+    t += 60 * 60_000;
+    await cache.get('NEPSE', true);
+    check('served from the cache inside six hours', calls === 1);
+    t += 6 * 60 * 60_000;
+    mode = 'garbage';
+    const stale = await cache.get('NEPSE', true);
+    check('an unreadable answer never replaces a good copy', calls === 2 && stale?.length === 4);
+    mode = 'ok';
+    await cache.get('NEPSE', true);
+    check('…and is remembered for a minute rather than retried per chart', calls === 2);
+    t += 61_000;
+    await cache.get('NEPSE', true);
+    check('retried after the minute', calls === 3);
+    mode = 'no_data';
+    const none = await cache.get('GBILD', false);
+    const again = await cache.get('GBILD', false);
+    check('no_data is a real answer, cached like one', none !== null && none.length === 0 && again?.length === 0 && calls === 4);
+    mode = 'fail';
+    const down = await cache.get('NICA', false);
+    check('down with no copy → null (the other sources stand alone)', down === null && calls === 5);
+    const slash = await cache.get('GBILD86/87', false);
+    check('a ticker the feed cannot carry is not asked for', slash?.length === 0 && calls === 5);
+}
+
 console.log(`\n${failed === 0 ? '✓ PASS' : '✗ FAIL'} — ${passed} passed, ${failed} failed`);
 process.exit(failed === 0 ? 0 : 1);