"""Read-only source audit. Never modifies the supplied source directory."""
import argparse
from collections import Counter
from pathlib import Path
import pyarrow.parquet as pq
from common import dump, sha, now, MAX_SOURCE_BYTES, MAX_FILE_BYTES

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--source', default='D:/AI_DATA/wiki_clean/thinking/english')
    ap.add_argument('--out', default='artifacts/source_manifest.json')
    args = ap.parse_args()
    files, excluded, total = [], [], 0
    for p in sorted(Path(args.source).glob('*.parquet')):
        size = p.stat().st_size
        if size >= MAX_FILE_BYTES or total + size >= MAX_SOURCE_BYTES:
            excluded.append({'name': p.name, 'bytes': size, 'reason': 'strict byte limit'})
            continue
        pf = pq.ParquetFile(p)
        assert {'source','kind','prompt','think','answer'} <= set(pf.schema_arrow.names)
        row = next(pf.iter_batches(batch_size=1)).to_pylist()[0]
        owner, repo = p.name.split('__')[:2]
        files.append({'name': p.name, 'bytes': size, 'sha256': sha(p),
                      'rows': pf.metadata.num_rows, 'row_groups': pf.num_row_groups,
                      'source': row['source'], 'upstream_repo': owner + '/' + repo,
                      'source_revision': None, 'local_mtime_ns': p.stat().st_mtime_ns})
        total += size
        print('AUDIT', len(files), p.name, size, flush=True)
    manifest = {'created_utc': now(), 'source_directory': str(Path(args.source).resolve()),
                'source_revision_note': 'Historical upstream revisions were not retained in these cleaned local files. Local SHA-256 values define this exact snapshot; current upstream revisions are not claimed as historical identities.',
                'limits': {'total_bytes_strictly_less_than': MAX_SOURCE_BYTES,
                           'single_file_bytes_strictly_less_than': MAX_FILE_BYTES},
                'file_count': len(files), 'total_bytes': total,
                'total_rows': sum(x['rows'] for x in files),
                'files': files, 'excluded': excluded}
    dump(args.out, manifest)
    print('AUDIT_COMPLETE', len(files), total, manifest['total_rows'], flush=True)

if __name__ == '__main__': main()