Rewrite archive ZIP selection: find largest start-date <= target date

NEMWEB archive ZIPs are named PUBLIC_NEXT_DAY_FPP_YYYYMMDD.zip where the
date is the FIRST day covered by that bundle (not end-of-month). A single
archive ZIP spans multiple days (e.g. 20250228.zip covers Feb 28 - Mar 30).

Previous approach (month-string matching) was wrong in two ways:
- It matched FPPMW daily files as 'monthly archives'
- It didn't account for bundles crossing month boundaries

New approach:
- Extract the 8-digit date from every filename in the archive listing
- Select the ZIP whose date is the largest date that is still <= target date
- This correctly navigates any bundle boundary

Also:
- Current dir files (with 16-digit suffix) still matched by exact date_str
- _find_file now raises AEMOFetchError directly (cleaner call site)
- Better error message lists which dates ARE available in the archive ZIP
when the exact date CSV is missing
- Extended _is_current window to 9 days for UTC/AEST lag tolerance
- Added module docstring explaining both file naming conventions and the
UTC vs NEM-time note for timestamps inside CSVs

https://claude.ai/code/session_01Vm5dTYdRf99LZ9MfB86vjx

app/services/aemo_fetcher.py

@@ -1,11 +1,33 @@
 """
 Downloads and extracts FPPDAILY ZIP files from AEMO NEMWEB.
+
+File naming conventions on NEMWEB:
+
+Current directory (https://www.nemweb.com.au/REPORTS/Current/FPPDAILY/):
+  PUBLIC_NEXT_DAY_FPP_YYYYMMDD_<16-digit-suffix>.zip
+  One file per day, contains the single day's CSV.
+
+Archive directory (https://nemweb.com.au/Reports/Archive/FPPDAILY/):
+  PUBLIC_NEXT_DAY_FPP_YYYYMMDD.zip
+  Each ZIP bundles multiple days of CSVs. The YYYYMMDD in the filename is the
+  START date of the bundle, NOT an end-of-month date.
+  Example: PUBLIC_NEXT_DAY_FPP_20250228.zip covers Feb 28 – Mar 30 2025;
+           PUBLIC_NEXT_DAY_FPP_20250331.zip covers Mar 31 2025 onwards.
+  To find data for a given date, download the archive whose start date is the
+  largest date that is <= the target date.
+
+Timezone note:
+  INTERVAL_DATETIME / MEASUREMENT_DATETIME inside CSVs are in UTC (or NEM time
+  which is UTC+10 without DST).  An Australian market day D runs from
+  D-1 14:00 UTC to D 13:30 UTC in five-minute intervals.  The CSV file itself
+  is named with the Australian market date, so file selection by date is
+  correct without timezone conversion.
 """
 import io
 import logging
 import re
 import zipfile
-from datetime import date
+from datetime import date, datetime
 
 import httpx
 
@@ -36,12 +58,15 @@ class AEMOFetchError(Exception):
 
 
 def _is_current(target_date: date) -> bool:
-    """Files within the last ~7 days are in /Current/, older in /Archive/."""
-    return (date.today() - target_date).days <= 7
+    """
+    Files within the last ~7 days live in /Current/; older files in /Archive/.
+    We use 9 days to allow for UTC vs AEST timezone differences and publication lag.
+    """
+    return (date.today() - target_date).days <= 9
 
 
 async def _list_directory(base_url: str, client: httpx.AsyncClient) -> list[str]:
-    """Fetch the HTML directory listing and extract all ZIP filenames."""
+    """Fetch the HTML directory listing and return all ZIP filenames."""
     logger.info("Listing directory: %s", base_url)
     try:
         resp = await client.get(
@@ -68,63 +93,109 @@ async def _list_directory(base_url: str, client: httpx.AsyncClient) -> list[str]
     return filenames
 
 
+def _extract_zip_date(filename: str) -> date | None:
+    """
+    Extract the YYYYMMDD date embedded in a NEMWEB ZIP filename.
+
+    Handles both:
+      PUBLIC_NEXT_DAY_FPP_20260224_1234567890123456.zip  (Current, with suffix)
+      PUBLIC_NEXT_DAY_FPP_20260228.zip                   (Archive, no suffix)
+    """
+    # Match 8 consecutive digits that appear after an underscore and before
+    # either another underscore, a dot, or end-of-string.
+    m = re.search(r'_(\d{8})(?:[_.]|$)', filename)
+    if m:
+        try:
+            return datetime.strptime(m.group(1), "%Y%m%d").date()
+        except ValueError:
+            return None
+    return None
+
+
 def _date_str(target_date: date) -> str:
     """Return date as YYYYMMDD string."""
     return target_date.strftime("%Y%m%d")
 
 
-async def _find_file(
-    target_date: date, date_str: str, base_url: str, client: httpx.AsyncClient
-) -> tuple[list[str], str]:
+async def _find_file_in(
+    filenames: list[str],
+    target_date: date,
+    date_str: str,
+    url: str,
+) -> tuple[str, str] | None:
     """
-    Try multiple locations to find a ZIP for the target date.
-
-    NEMWEB FPPDAILY archive has two file types:
-      - Daily:   PUBLIC_NEXT_DAY_FPPMW_YYYYMMDD.zip  (individual day, MW metering data)
-      - Monthly: PUBLIC_NEXT_DAY_FPP_YYYYMMDD.zip    (month-end date, no "MW" in name,
-                                                       contains all daily CSVs for that month)
+    Given a directory listing, return (filename, url) for the best ZIP to use
+    for target_date, or None if nothing suitable is found.
 
     Strategy:
-      1. Look for an exact daily FPPMW match (date_str in filename)
-      2. Look for a monthly FPP archive (month_str in filename, "FPPMW" NOT in filename)
-      3. Also try Current URL as fallback — NEMWEB may keep files longer than 7 days
-
-    Returns (matching_filenames, base_url_where_found).
+      1. Exact date match — filename contains YYYYMMDD of the target date.
+         Covers Current files (PUBLIC_NEXT_DAY_FPP_20260224_XXXXXXXXXXXXXXXX.zip)
+         and any archive file whose start date happens to equal target_date.
+      2. Archive bundle match — find the ZIP whose embedded date is the largest
+         date that is still <= target_date.  That bundle covers target_date.
     """
-    month_str = target_date.strftime("%Y%m")
+    # 1. Exact match (e.g. Current dir or lucky archive hit)
+    exact = [f for f in filenames if date_str in f]
+    if exact:
+        logger.info("Exact match at %s: %s", url, exact[0])
+        return exact[0], url
+
+    # 2. Archive bundle: largest start-date <= target_date
+    candidates: list[tuple[date, str]] = []
+    for f in filenames:
+        zip_date = _extract_zip_date(f)
+        if zip_date is not None and zip_date <= target_date:
+            candidates.append((zip_date, f))
+
+    if candidates:
+        candidates.sort(reverse=True)  # descending by date
+        best_date, best_file = candidates[0]
+        logger.info(
+            "Archive bundle match at %s: %s (bundle start %s covers %s)",
+            url, best_file, best_date, target_date,
+        )
+        return best_file, url
+
+    return None
 
-    # Always try archive subdirectory and Current as fallbacks regardless of date age
-    urls_to_try = [base_url, AEMO_ARCHIVE_URL, AEMO_CURRENT_URL]
-    # Deduplicate while preserving order
-    seen: set[str] = set()
-    unique_urls = [u for u in urls_to_try if u not in seen and not seen.add(u)]  # type: ignore[func-returns-value]
 
-    filenames: list[str] = []
-    for url in unique_urls:
+async def _find_file(
+    target_date: date, date_str: str, base_url: str, client: httpx.AsyncClient
+) -> tuple[str, str]:
+    """
+    Search Current then Archive for a ZIP covering target_date.
+    Returns (filename, base_url_of_that_file).
+    Raises AEMOFetchError if nothing is found.
+    """
+    # Always try Current first, then Archive, regardless of age.
+    # This handles publication lag and UTC/AEST edge cases.
+    urls_to_try: list[str] = []
+    if base_url == AEMO_CURRENT_URL:
+        urls_to_try = [AEMO_CURRENT_URL, AEMO_ARCHIVE_URL]
+    else:
+        urls_to_try = [AEMO_ARCHIVE_URL, AEMO_CURRENT_URL]
+
+    last_filenames: list[str] = []
+    for url in urls_to_try:
         try:
             filenames = await _list_directory(url, client)
         except AEMOFetchError as e:
             logger.warning("Could not list %s: %s", url, e)
             continue
 
-        # 1. Exact daily match (e.g. PUBLIC_NEXT_DAY_FPPMW_20260224.zip)
-        matching = [f for f in filenames if date_str in f]
-        if matching:
-            logger.info("Found daily match at %s: %s", url, matching[0])
-            return matching, url
-
-        # 2. Monthly archive match: PUBLIC_NEXT_DAY_FPP_YYYYMMDD.zip (no "MW" in name).
-        #    Excludes daily FPPMW files which also contain the month string.
-        monthly = [f for f in filenames if month_str in f and "FPPMW" not in f]
-        if monthly:
-            logger.info("Found monthly archive at %s: %s", url, monthly[0])
-            return monthly, url
+        last_filenames = filenames
+        result = await _find_file_in(filenames, target_date, date_str, url)
+        if result:
+            return result
 
     logger.error(
-        "No file found for date %s. Searched: %s. Last listing sample: %s",
-        date_str, unique_urls, filenames[:10],
+        "No file found for %s. Last directory listing sample: %s",
+        date_str, last_filenames[:10],
+    )
+    raise AEMOFetchError(
+        f"No FPPDAILY data found for {target_date.strftime('%d %B %Y')}. "
+        "The file may not yet be published by AEMO."
     )
-    return [], base_url
 
 
 async def fetch_csv_for_date(target_date: date, duid: str) -> bytes:
@@ -135,7 +206,7 @@ async def fetch_csv_for_date(target_date: date, duid: str) -> bytes:
     if target_date < DATA_START_DATE:
         raise AEMOFetchError(
             f"Data is only available from {DATA_START_DATE.strftime('%d %B %Y')} "
-            f"when the FPP scheme commenced."
+            "when the FPP scheme commenced."
         )
     if target_date >= date.today():
         raise AEMOFetchError("Cannot request data for today or future dates.")
@@ -144,14 +215,7 @@ async def fetch_csv_for_date(target_date: date, duid: str) -> bytes:
     date_str = _date_str(target_date)
 
     async with httpx.AsyncClient() as client:
-        matching, found_url = await _find_file(target_date, date_str, base_url, client)
-        if not matching:
-            raise AEMOFetchError(
-                f"No FPPDAILY data found for {target_date.strftime('%d %B %Y')}. "
-                f"The file may not yet be published or the date may not have data."
-            )
-
-        filename = matching[0]
+        filename, found_url = await _find_file(target_date, date_str, base_url, client)
         zip_url = found_url + filename
 
         logger.info("Downloading: %s", zip_url)
@@ -166,28 +230,56 @@ async def fetch_csv_for_date(target_date: date, duid: str) -> bytes:
         except httpx.TimeoutException:
             raise AEMOFetchError(
                 "AEMO server timed out while downloading the data file. "
-                "The file may be very large. Please try again."
+                "The file may be large. Please try again."
             )
         except httpx.HTTPStatusError as e:
             raise AEMOFetchError(
                 f"Could not download data file (HTTP {e.response.status_code})."
             )
 
-        # Extract CSV from ZIP.
-        # If this is a monthly archive ZIP, find the CSV matching the specific date.
+        # Extract the correct daily CSV from the ZIP.
+        # Archive ZIPs contain multiple CSVs (one per day); match by date_str.
+        # Current ZIPs contain a single CSV.
         try:
             zip_bytes = io.BytesIO(resp.content)
             with zipfile.ZipFile(zip_bytes) as zf:
-                all_csvs = [n for n in zf.namelist() if n.lower().endswith(".csv")]
+                all_entries = zf.namelist()
+                all_csvs = [n for n in all_entries if n.lower().endswith(".csv")]
+
                 if not all_csvs:
                     raise AEMOFetchError("ZIP file contained no CSV files.")
 
-                # Prefer the CSV that matches the target date; fall back to first
+                # Prefer the CSV whose name contains the target date
                 daily_csvs = [n for n in all_csvs if date_str in n]
-                csv_name = daily_csvs[0] if daily_csvs else all_csvs[0]
+                if daily_csvs:
+                    csv_name = daily_csvs[0]
+                elif len(all_csvs) == 1:
+                    # Single-CSV ZIP (Current) — use it directly
+                    csv_name = all_csvs[0]
+                else:
+                    # Multi-CSV archive but no name match — log and use newest by name
+                    all_csvs_sorted = sorted(all_csvs, reverse=True)
+                    logger.warning(
+                        "No CSV matching %s in %s; available: %s",
+                        date_str, filename, all_csvs_sorted[:5],
+                    )
+                    raise AEMOFetchError(
+                        f"No data file found for {target_date.strftime('%d %B %Y')} "
+                        f"inside the archive ZIP. Available dates: "
+                        + ", ".join(
+                            m.group(1)
+                            for n in all_csvs_sorted[:5]
+                            for m in [re.search(r'(\d{8})', n)]
+                            if m
+                        )
+                    )
+
                 logger.info("Extracting CSV: %s", csv_name)
                 csv_bytes = zf.read(csv_name)
+
         except zipfile.BadZipFile:
-            raise AEMOFetchError("Downloaded file appears to be corrupt. Please try again.")
+            raise AEMOFetchError(
+                "Downloaded file appears to be corrupt. Please try again."
+            )
 
         return csv_bytes