scripts/export_research.py

"""Export allowlisted benchmark fields and aggregates; never copy execution logs."""
import argparse,collections,datetime,hashlib,json,sqlite3
from pathlib import Path
from editorial_guard.dataset import dataset_summary,validate_dataset,write_json
from editorial_guard.protocol import ANNOTATION_VERSION
from editorial_guard.core import public_input
from editorial_guard.blinding import opaque_id

def reference(value):
    return 'evidence-'+hashlib.sha256(value.encode()).hexdigest()[:24]


def quality_index(rows):
    """Keep exclusions and unresolved judgments visible without manufacturing consensus."""
    output=[]
    for row in rows:
        annotations=[v['annotation'] for v in row['annotation_records']]
        reasons=[]
        if len(annotations)<2:reasons.append('insufficient_independent_annotations')
        if annotations and all(not a['valid_task'] for a in annotations):reasons.append('all_available_reviewers_reject_task')
        elif any(not a['valid_task'] for a in annotations):reasons.append('task_validity_disputed')
        if any(a['language_scope_status']!='supported' for a in annotations):reasons.append('language_scope_unvalidated')
        if any(a['source_fidelity']=='fail' for a in annotations):reasons.append('original_source_fidelity_contested')
        if any(a['source_fidelity']=='uncertain' or a['task_satisfied']=='uncertain' for a in annotations):reasons.append('semantic_uncertainty')
        missing=[value for value in row['constraints'].get('literal_required',[]) if value not in row['input_text']]
        if missing:reasons.append('required_literal_missing')
        states={(a['style_problem'],a['process_problem']) for a in annotations}
        confirmed_state=list(next(iter(states))) if len(annotations)>=2 and len(states)==1 else None
        no_edit=len(annotations)>=2 and all(a['valid_task'] and a['no_edit_needed'] and a['source_fidelity']=='pass' and a['task_satisfied']=='pass' and not a['style_problem'] and not a['process_problem'] for a in annotations)
        output.append({'id':row['id'],'source_group_id':row['source_group_id'],'route':row['provenance']['route'],
                       'annotation_count':len(annotations),'review_flags':reasons,
                       'task_validity':'insufficient_evidence' if len(annotations)<2 else 'invalid' if all(not a['valid_task'] for a in annotations) else 'disputed' if any(not a['valid_task'] for a in annotations) else 'model_agreement_valid',
                       'agreed_state_style_process':confirmed_state,'model_confirmed_no_edit':no_edit,
                       'designed_boundary':bool(row['provenance'].get('designed_boundary')),
                       'required_literal_count':len(row['constraints'].get('literal_required',[])),
                       'missing_required_literal_count':len(missing)})
    return output

def export(work,destination,release_locked=False,release_locked_families=None):
    if release_locked_families is not None and not release_locked:raise ValueError("Family release list requires explicit locked release")
    approved_locked=None if release_locked_families is None else set(release_locked_families)
    work=Path(work);out=Path(destination)
    if out.exists() and any(out.iterdir()):raise ValueError('Data export output must be empty')
    out.mkdir(parents=True,exist_ok=True)
    schema=json.loads((Path(__file__).resolve().parents[1]/'schemas/sample-0.1.0.json').read_text(encoding='utf-8'))
    allowed=set(schema['properties']);all_rows=[];phase_counts={}
    with sqlite3.connect(work/'requests.sqlite3') as db:
        request_dates={rid:datetime.datetime.fromtimestamp(started,datetime.timezone.utc).isoformat() for rid,started in db.execute('select id,started from requests')}
    def observable(rid):
        p=work/'requests'/(rid+'.request.json')
        if not p.exists():return {'request_date':request_dates.get(rid),'transport':'unknown'}
        setting=json.loads(p.read_text(encoding='utf-8'))
        return {'request_date':request_dates.get(rid),'transport':setting.get('transport','unknown'),'provider_config_hash':setting.get('provider_config_hash','unknown'),'parameters':setting.get('parameters',{}),'endpoint':setting.get('endpoint'),'requested_max_tokens':setting.get('max_tokens'),'effective_max_tokens':setting.get('effective_max_tokens','unknown')}
    for phase,prefix in [('calibration','cal'),('main','main')]:
        directory=work/'data'/phase;rows=[]
        for path in sorted(directory.glob(prefix+'-*.json')):
            family=json.loads(path.read_text(encoding='utf-8'))
            for original in family['records']:
                row={k:v for k,v in original.items() if k in allowed}
                row=json.loads(json.dumps(row))
                rid=row['provenance'].get('request_id')
                row['generation_config'].update(observable(rid))
                row['provenance']['request_id']=reference(rid)
                votes=[]
                for p in sorted((directory/('labels-'+ANNOTATION_VERSION)).glob(row['id']+'--*.json')):
                    vote=json.loads(p.read_text(encoding='utf-8'))
                    vote={k:vote[k] for k in ('provider','requested_model','reported_model','request_id','annotation_version','annotation') if k in vote}
                    vote.update(observable(vote['request_id']))
                    vote['request_id']=reference(vote['request_id']);votes.append(vote)
                row['annotation_records']=votes
                signatures=[tuple(v['annotation'][k] for k in ('valid_task','style_problem','process_problem','source_fidelity','task_satisfied','no_edit_needed')) for v in votes]
                row['disagreement_status']='insufficient_evidence' if len(votes)<2 else 'agreement' if len(set(signatures))==1 else 'disputed'
                row['issues']=votes[0]['annotation']['issues'] if row['disagreement_status']=='agreement' else []
                rows.append(row)
        phase_counts[phase]=dataset_summary(rows);all_rows.extend(rows)
    locked_families={r["source_group_id"] for r in all_rows if r["split"]=="locked_test"}
    if approved_locked is not None and not approved_locked<=locked_families:raise ValueError("Unknown locked family in release list")
    released_locked=locked_families if release_locked and approved_locked is None else (approved_locked or set())
    validation=validate_dataset(all_rows)
    write_json(out/'validation.json',validation)
    suites={}
    for suite in ('StyleBench-ZH','DeliveryBench-ZH'):
        root=out/suite;root.mkdir(exist_ok=True)
        rows=[r for r in all_rows if r['suite']==suite]
        published=[r for r in rows if r['split']!='locked_test' or r['source_group_id'] in released_locked]
        with (root/'samples.jsonl').open('w',encoding='utf-8',newline='\n') as stream:
            for row in published:stream.write(json.dumps(row,ensure_ascii=False)+'\n')
        # Alternate task views share their existing source families; they add zero independent families.
        view_index=[]
        with (root/'task-views.jsonl').open('w',encoding='utf-8',newline='\n') as stream:
            for row in published:
                for track in ('detect','generate','workflow'):
                    view=public_input(row);view.update(id=opaque_id(row['id'],'solver-view-v1')+'-'+track,parent_id=opaque_id(row['id'],'solver-view-v1'),source_group_id=opaque_id(row['source_group_id'],'solver-family-v1'),track=track,view_status='task_view_not_independent_sample')
                    view_index.append({'view_id':view['id'],'parent_id':row['id'],'source_group_id':row['source_group_id'],'split':row['split']})
                    if track=='generate':view.pop('input_text',None)
                    stream.write(json.dumps(view,ensure_ascii=False)+'\n')
        write_json(root/'task-view-index.json',view_index)
        partition={kind:[r['id'] for r in published if r['split']==kind] for kind in ('calibration','development','validation','locked_test')}
        write_json(root/'partitions.json',partition)
        write_json(root/'disputed-index.json',[r['id'] for r in published if r['disagreement_status']=='disputed'])
        family_manifest=[]
        for group in sorted({r['source_group_id'] for r in published}):
            members=[r for r in published if r['source_group_id']==group]
            family_manifest.append({'source_group_id':group,'split':members[0]['split'],'suite':suite,'instance_ids':[r['id'] for r in members],'generator':members[0]['provenance']['generator'],'origin':'synthetic fictional task world'})
        write_json(root/'family-manifest.json',family_manifest)
        quality=quality_index(published)
        write_json(root/'quality-index.json',quality)
        summary=dataset_summary(published)
        summary['quality']={'task_validity':dict(collections.Counter(r['task_validity'] for r in quality)),'review_flags':dict(collections.Counter(flag for r in quality for flag in r['review_flags'])),'confirmed_no_edit_instances':sum(r['model_confirmed_no_edit'] for r in quality),'agreed_four_states':dict(collections.Counter(str(r['agreed_state_style_process']) for r in quality)),'required_literal_missing_instances':sum(bool(r['missing_required_literal_count']) for r in quality),'scope':'automated judgments; flags remain visible, no human validation or forced consensus'}
        summary.update(planned_main_families=250,main_families=len({r['source_group_id'] for r in rows if r['split']!='calibration'}),held_back_instances=len(rows)-len(published),generation_cohorts=dict(collections.Counter(r['provenance']['generator']+'|'+r['generation_config'].get('transport','unknown')+'|'+r['generation_config'].get('provider_config_hash','unknown') for r in published)),designed_boundary_instances=sum(bool(r['provenance'].get('designed_boundary')) for r in published),all_labels_automated=True,human_validation=False,independent_family_caveat='distinct fictional task worlds drawn from seven base templates; lexical deduplication does not prove semantic independence')
        suites[suite]=summary;write_json(root/'summary.json',summary)
    release={'version':'0.1.0','annotation_version':ANNOTATION_VERSION,'phases':phase_counts,'suites':suites,'validation_status':validation['status'],'locked_test_released':release_locked,'locked_test_release':{'scope':'explicit_family_allowlist' if approved_locked is not None else 'all_locked' if release_locked else 'none','family_ids':sorted(released_locked),'held_back_families':len(locked_families-released_locked)},'files':{str(p.relative_to(out)).replace('\\','/'):hashlib.sha256(p.read_bytes()).hexdigest() for p in out.rglob('*') if p.is_file() and p.name!='manifest.json'},'origin':'programmatic fictional facts and recorded model responses','license':'CC-BY-4.0'}
    write_json(out/'manifest.json',release)
    return release

if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('--work-dir',required=True);p.add_argument('--output',required=True);p.add_argument('--release-locked',action='store_true');p.add_argument('--release-locked-family',action='append',help='Only release evaluated locked families; repeat for each source_group_id');a=p.parse_args()
    print(json.dumps(export(a.work_dir,a.output,a.release_locked,a.release_locked_family),ensure_ascii=False,indent=2))