Buckets:
| """Offline Silver V2.1 construction. Never modifies frozen V2 artifacts.""" | |
| import argparse, csv, hashlib, json, math, shutil, difflib, io, random | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| ROOT=Path(__file__).resolve().parents[1] | |
| WORK=ROOT.parent | |
| ARCHIVE=ROOT/'experiments/clean2d/2026-09-25/payload' | |
| V2=ROOT/'outputs/gpt6_filter_v2_blind20_20260925' | |
| OUT=ROOT/'outputs/silver_clean2d_v2_1_20260925' | |
| PRIVATE=OUT/'private' | |
| POOL=ARCHIVE/'outputs/clean2d/silver_train_pool.jsonl' | |
| ANNOTATED=Path('/Users/bytedance/Downloads/human_blind20_annotated.csv') | |
| DIMS=['correctness','coherence','complexity','verbosity'] | |
| GUARD='Correctness is defined relative to the task objective and prompt constraints, not factual truth in isolation. If the task explicitly permits false statements or asks to transform/identify false content, factual truth alone does not determine correctness.' | |
| def sha(p): return hashlib.sha256(Path(p).read_bytes()).hexdigest() | |
| def hashtext(s):return hashlib.sha256(s.encode()).hexdigest() | |
| def readcsv(p): | |
| with Path(p).open(encoding='utf-8-sig',newline='') as f:return list(csv.DictReader(f)) | |
| def readjl(p): | |
| with Path(p).open() as f:return [json.loads(l) for l in f if l.strip()] | |
| def put(p,obj): | |
| data=(obj if isinstance(obj,str) else json.dumps(obj,ensure_ascii=False,indent=2)+'\n').encode() | |
| p=Path(p) | |
| if p.exists(): | |
| assert p.read_bytes()==data, f'Refusing to change frozen artifact: {p}' | |
| else: | |
| p.parent.mkdir(parents=True,exist_ok=True) | |
| with p.open('xb') as f:f.write(data) | |
| if PRIVATE in p.parents:p.chmod(0o600) | |
| def wilson(s,n): | |
| z=1.959963984540054;p=s/n;den=1+z*z/n | |
| mid=(p+z*z/(2*n))/den;half=z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/den | |
| return [mid-half,mid+half] | |
| def prepare(): | |
| PRIVATE.mkdir(parents=True,exist_ok=True,mode=0o700);PRIVATE.chmod(0o700) | |
| for src,name in [(ANNOTATED,'human_blind20_annotated.csv'),(V2/'private/private_key.json','v2_private_key.json'), | |
| (V2/'private/gpt6_reviews.json','v2_gpt6_reviews.json'),(V2/'private/protocol.json','v2_protocol.json')]: | |
| dst=PRIVATE/name | |
| if not dst.exists():shutil.copy2(src,dst) | |
| assert sha(src)==sha(dst);dst.chmod(0o400) | |
| annotations=readcsv(ANNOTATED);original={r['pair_id']:r for r in readcsv(V2/'human_blind20.csv')} | |
| oldkey=json.loads((V2/'private/private_key.json').read_text());keys={r['pair_id']:r for r in oldkey['rows']} | |
| assert len(annotations)==len(keys)==20 and len({r['pair_id'] for r in annotations})==20 | |
| groups=defaultdict(list);restored=[] | |
| for r in annotations: | |
| k=keys[r['pair_id']] | |
| assert all(r[f]==original[r['pair_id']][f] for f in ['prompt','response_A','response_B']) | |
| ds={d:int(r[d])*k['B_minus_A_to_original_to_edited_multiplier'] for d in DIMS} | |
| ok=ds[k['requested_criterion']]*k['requested_direction']>=1 and all(v==0 for d,v in ds.items() if d!=k['requested_criterion']) and r['other_meaningful_change']=='no' | |
| for g in ['overall',k['sampling_group'],k['requested_criterion']+str(k['requested_direction'])]:groups[g].append(ok) | |
| restored.append({'pair_id':r['pair_id'],'candidate_id':k['candidate_id'],'deltas_original_to_edited':ds,'strict_pass':ok, | |
| 'short_reason_as_annotated_A_to_B':r['short_reason'],'reason_orientation':'B minus A, unchanged; use key multiplier only for numeric scores.'}) | |
| stats={g:{'success':sum(v),'n':len(v),'rate':sum(v)/len(v),'wilson95':wilson(sum(v),len(v))} for g,v in groups.items()} | |
| assert [(stats[g]['success'],stats[g]['n']) for g in ['overall','high','boundary']]==[(19,20),(16,16),(3,4)] | |
| put(PRIVATE/'human_v2_frozen_results.json',{'statistics':stats,'rows':restored,'source_hashes':{str(p):sha(p) for p in [ANNOTATED,V2/'human_blind20.csv',V2/'private/private_key.json',V2/'private/gpt6_reviews.json']}}) | |
| old=readcsv(WORK/'human_calibration/private/human_calibration_80_key.csv') | |
| ids={r['candidate_id'] for r in old}|{r['candidate_id'] for r in keys.values()} | |
| families={r['family_id'] for r in old}|{r['family_id'] for r in keys.values()} | |
| prompts={str(r['prompt_id']) for r in old}|{str(r['prompt_id']) for r in keys.values()} | |
| pool=readjl(POOL);allc=readjl(ROOT/'outputs/gold_intervention/candidates.jsonl') | |
| for r in allc: | |
| if r.get('human_certified') is True: | |
| ids.add(r['candidate_id']);families.add(r['family_id']);prompts.add(str(r['prompt_id'])) | |
| excluded=json.loads((ARCHIVE/'outputs/clean2d/exclusion_manifest.json').read_text()) | |
| excluded_prompts=prompts|set(excluded['silver_excluded_prompt_ids']) | |
| source_lock=json.loads((ARCHIVE/'configs/clean2d/source_lock.json').read_text()) | |
| assert sha(POOL)==source_lock['files']['outputs/clean2d/silver_train_pool.jsonl'] | |
| for rel in ['data/processed/responses.parquet','data/processed/preferences.parquet','data/processed/attributes.parquet']: | |
| assert sha(ROOT/rel)==source_lock['files'][rel] | |
| byfamily=defaultdict(list);elig=[];removed=[] | |
| for r in pool: | |
| valid=(r['candidate_id'] not in ids and r['family_id'] not in families and str(r['prompt_id']) not in excluded_prompts | |
| and r['split']=='train' and r['automatic_pass'] is True) | |
| if valid:elig.append(r);byfamily[r['family_id']].append(r) | |
| else:removed.append({'candidate_id':r['candidate_id'],'status':'excluded_human_or_heldout_prompt','gpt6_score':None}) | |
| queue=[] | |
| for i,(fid,rs) in enumerate(sorted(byfamily.items(),key=lambda kv:(kv[1][0]['criterion'],kv[0])),1): | |
| assert len(rs)==2 and {int(r['requested_direction']) for r in rs}=={-1,1} | |
| neg=next(r for r in rs if r['requested_direction']==-1);pos=next(r for r in rs if r['requested_direction']==1) | |
| assert neg['original_response']==pos['edited_response'] and neg['edited_response']==pos['original_response'] and neg['prompt']==pos['prompt'] | |
| queue.append({'review_id':f'F{i:03}','family_id':fid,'criterion':neg['criterion'],'canonical_negative':neg,'positive_id':pos['candidate_id']}) | |
| put(PRIVATE/'family_review_queue.json',queue) | |
| put(PRIVATE/'excluded_candidates.json',removed) | |
| put(PRIVATE/'training_exclusions.json',{'human_candidate_ids':sorted(ids),'human_family_ids':sorted(families), | |
| 'human_prompt_ids':sorted(prompts),'silver_excluded_prompt_ids':sorted(excluded_prompts), | |
| 'preference_and_attribute_train_excluded_prompt_ids':sorted(prompts), | |
| 'scope':'Apply prompt exclusions identically to all three models; old 80 and new 20 never enter any main training supervision.', | |
| 'parquet_split_verification':'Original responses/preferences/attributes SHA256 match the archived source lock used to produce the non-train prompt exclusion manifest.', | |
| 'source_sha256':{str(p):sha(p) for p in [POOL,WORK/'human_calibration/private/human_calibration_80_key.csv',V2/'private/private_key.json',ARCHIVE/'outputs/clean2d/exclusion_manifest.json']}}) | |
| protocol={'version':'V2.1-task-relative','parent_version':'V2.0-direct-review','guard':GUARD, | |
| 'validation_status':'V2.0 frozen at 19/20, high 16/16, boundary 3/4; V2.1 guard is post-validation and has no independent human-validation estimate.', | |
| 'scorer':'Active GPT-6 assistant direct offline semantic review. No external API, no fresh human labels, no use of old automatic judge scores.', | |
| 'scores':{'95':'Very clear pure task-relative intervention','85':'Strong pure intervention with minor qualification','75':'Boundary interpretive ambiguity; excluded from main training','65':'Unverified or substantial ambiguity; excluded','40':'Off-target or multidimensional; excluded','0':'No justified target change or wrong direction; excluded'}, | |
| 'high_score_min':85,'boundary_max':75,'dimensions':DIMS, | |
| 'review_process':'Review compact task-and-diff evidence for every eligible family; score conservatively. Fully inspect prompt and both response contents for all selected families before release. Canonical clean-to-altered deltas are inverted only for byte-verified reverse pairs.', | |
| 'score_semantics':'Ordinal model judgments, not calibrated probabilities. Limited evidence may warrant rejection; do not invent a verified factual difference.', | |
| 'one_factor_rule':'Signed target delta >=1; zero predicted non-target deltas; no other meaningful change; task-relative guard satisfied.', | |
| 'main_budget_per_cell':50,'scaling_budget_per_cell':25,'optional_second_budget_per_cell':100, | |
| 'tie_break':'score descending, SHA256(silver-v21|family_id) ascending; select distinct prompt IDs and normalized prompt text across selected families.', | |
| 'paired_directions':'50 families per criterion generate two directions each: 200 rows represent 100 unique text pairs, not 200 independent interventions.', | |
| 'freeze_v2':'No rescoring or changing the original V2 20-example report.', | |
| 'source_pool_sha256':sha(POOL),'eligible_candidates':len(elig),'eligible_families':len(queue), | |
| 'candidate_scope':'The current clean2d 800-row pool only, not all historic gold-intervention pools.', | |
| 'fixed_training':{'backbone':'Qwen/Qwen3-32B','block':48,'pooling':'response_mean','frozen_backbone':True,'head':'linear','latent_coordinates':3,'seeds':[0,1,2,3,4], | |
| 'models':['Latent-BT-2','Clean-ILR-2','Full-Attribute-2'],'full_attribute_role':'Additional dense human supervision reference, not a fair supervision-matched baseline or guaranteed numerical upper bound.'}} | |
| put(PRIVATE/'review_protocol_before_scoring.json',protocol) | |
| print(json.dumps({'eligible_candidates':len(elig),'families':len(queue),'excluded':len(removed),'v2_frozen':True})) | |
| def selected(): | |
| queue=json.loads((PRIVATE/'family_review_queue.json').read_text()) | |
| with (PRIVATE/'gpt6_screen.tsv').open() as f:scores={r['review_id']:r for r in csv.DictReader(f,delimiter='\t')} | |
| assert len(scores)==len(queue)==380 | |
| overrides=json.loads((PRIVATE/'full_review_decisions.json').read_text()) if (PRIVATE/'full_review_decisions.json').exists() else {} | |
| for rid,r in overrides.items():scores[rid].update(r) | |
| ranked=sorted(queue,key=lambda r:(-int(scores[r['review_id']]['score']),hashtext('silver-v21|'+r['family_id']))) | |
| chosen=[];seen=set();pids=set();counts=Counter() | |
| for q in ranked: | |
| r=q['canonical_negative'];norm=' '.join(r['prompt'].split()) | |
| if int(scores[q['review_id']]['score'])>=85 and counts[q['criterion']]<50 and norm not in seen and str(r['prompt_id']) not in pids: | |
| chosen.append(q);seen.add(norm);pids.add(str(r['prompt_id']));counts[q['criterion']]+=1 | |
| return chosen,scores,ranked | |
| def show(start,count,full=False,picked=False): | |
| queue=json.loads((PRIVATE/'family_review_queue.json').read_text()) | |
| if picked:queue=selected()[0] | |
| for q in queue[start-1:start-1+count]: | |
| r=q['canonical_negative'];a=r['original_response'];b=r['edited_response'];p=r['prompt'] | |
| if full: | |
| # Lossless diff plus full original reconstructs both responses exactly. | |
| changes=[{'start':i,'end':j,'replacement':b[k:l]} for tag,i,j,k,l in difflib.SequenceMatcher(None,a,b,autojunk=False).get_opcodes() if tag!='equal'] | |
| print(json.dumps({'id':q['review_id'],'prompt':p,'A':a,'B_changes':changes},ensure_ascii=False)) | |
| else: | |
| edits=[] | |
| for tag,i,j,k,l in difflib.SequenceMatcher(None,a,b,autojunk=False).get_opcodes(): | |
| if tag!='equal':edits.append({'before':a[max(0,i-100):min(len(a),j+100)],'after':b[max(0,k-100):min(len(b),l+100)]}) | |
| task=p.rsplit('<extra_id_1>User',1)[-1] | |
| print(json.dumps({'id':q['review_id'],'task':task[:550],'task_clipped':len(task)>550,'history_present':task!=p,'diff':edits},ensure_ascii=False)) | |
| def jl(rows):return ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows) | |
| def finalize(): | |
| prepare() # Recheck source locks and immutable human-validation copies. | |
| chosen,scores,ranked=selected() | |
| reviewed=set((PRIVATE/'full_context_reviewed_ids.txt').read_text().split()) | |
| assert len(chosen)==100 and all(q['review_id'] in reviewed for q in chosen) | |
| assert Counter(q['criterion'] for q in chosen)=={'correctness':50,'coherence':50} | |
| pool={r['candidate_id']:r for r in readjl(POOL)} | |
| excludes=json.loads((PRIVATE/'training_exclusions.json').read_text()) | |
| humans=set(excludes['human_prompt_ids']) | |
| gold=readjl(ROOT/'outputs/gold_intervention/candidates.jsonl') | |
| human_texts={' '.join(r['prompt'].split()) for r in gold if str(r['prompt_id']) in humans} | |
| human_texts.update(' '.join(r['prompt'].split()) for r in readcsv(V2/'human_blind20.csv')) | |
| rows=[];small=[];manifest=[];counts=Counter();features={} | |
| for q in chosen: | |
| score=int(scores[q['review_id']]['score']);counts[q['criterion']]+=1 | |
| for cid in [q['canonical_negative']['candidate_id'],q['positive_id']]: | |
| r=pool[cid] | |
| assert cid not in excludes['human_candidate_ids'] and r['family_id'] not in excludes['human_family_ids'] | |
| assert str(r['prompt_id']) not in excludes['silver_excluded_prompt_ids'] | |
| assert ' '.join(r['prompt'].split()) not in human_texts | |
| row={k:r[k] for k in ['prompt','original_response','edited_response','criterion']} | |
| row.update(direction=int(r['requested_direction']),candidate_id=cid,gpt6_score=score) | |
| rows.append(row) | |
| if counts[q['criterion']]<=25:small.append(row) | |
| manifest.append({'candidate_id':cid,'family_id':r['family_id'],'prompt_id':r['prompt_id'], | |
| 'criterion':r['criterion'],'direction':r['requested_direction'],'gpt6_score':score, | |
| 'review_id':q['review_id'],'filter_version':'V2.1-task-relative','full_context_reviewed':True, | |
| 'rank_within_criterion':counts[q['criterion']],'in_silver100':counts[q['criterion']]<=25, | |
| 'source_pool':'experiments/clean2d/2026-09-25/payload/outputs/clean2d/silver_train_pool.jsonl', | |
| 'source_pool_sha256':sha(POOL),'prompt_sha256':hashtext(r['prompt']), | |
| 'original_response_sha256':hashtext(r['original_response']),'edited_response_sha256':hashtext(r['edited_response']), | |
| 'base_response_id':r['base_response_id'],'base_feature_index':r['base_feature_index'], | |
| 'orientation':r['orientation'],'generator':r['generator'],'split':r['split'], | |
| 'token_length_original':r['token_length_original'],'token_length_edited':r['token_length_edited']}) | |
| for role in ['original_response','edited_response']: | |
| text=r[role];key=hashtext(r['prompt']+'\0'+text) | |
| if key not in features: | |
| is_base=text==q['canonical_negative']['original_response'] | |
| features[key]={'feature_request_id':key,'prompt_id':r['prompt_id'],'prompt':r['prompt'],'response':text, | |
| 'prompt_sha256':hashtext(r['prompt']),'response_sha256':hashtext(text), | |
| 'candidate_ids':[],'base_response_id_hint':r['base_response_id'] if is_base else None, | |
| 'base_feature_index_hint':r['base_feature_index'] if is_base else None, | |
| 'cache_status':'unverified; match exact prompt, response, model and extraction settings before reuse'} | |
| features[key]['candidate_ids'].append(cid) | |
| random.Random(20260925).shuffle(rows);random.Random(20260925).shuffle(small) | |
| put(OUT/'silver_clean2d_train_200.jsonl',jl(rows));put(OUT/'silver_clean2d_train_100.jsonl',jl(small)) | |
| f=io.StringIO(newline='');w=csv.DictWriter(f,fieldnames=list(manifest[0]));w.writeheader();w.writerows(manifest) | |
| put(OUT/'silver_clean2d_manifest.csv',f.getvalue()) | |
| put(PRIVATE/'feature_requests.jsonl',jl(list(features.values()))) | |
| audit=[] | |
| for rank,q in enumerate(ranked,1): | |
| for cid in [q['canonical_negative']['candidate_id'],q['positive_id']]: | |
| audit.append({'candidate_id':cid,'family_id':q['family_id'],'review_id':q['review_id'], | |
| 'criterion':q['criterion'],'direction':pool[cid]['requested_direction'],'gpt6_score':int(scores[q['review_id']]['score']), | |
| 'evidence':scores[q['review_id']]['evidence'],'full_context_reviewed':q['review_id'] in reviewed, | |
| 'global_family_rank':rank,'selected':cid in {r['candidate_id'] for r in rows}, | |
| 'review_scope':'Full context plus lossless edits' if q['review_id'] in reviewed else 'Compact task and edit context only; screening score provisional', | |
| 'reverse_pair_rule':'Same ordinal purity score for byte-verified reverse direction; no claim of independent second judgment.'}) | |
| put(PRIVATE/'filter_v21_all_candidates_scored.jsonl',jl(audit)) | |
| protocol=json.loads((PRIVATE/'review_protocol_before_scoring.json').read_text()) | |
| config=dict(protocol) | |
| config.update({'release':'silver-clean2d-v2.1-20260925','seed_for_row_shuffle':20260925, | |
| 'selected_rows':200,'unique_families':100,'unique_prompts':100,'nested_scaling_rows':100, | |
| 'full_context_reviewed_families':len(reviewed),'selected_all_full_context_reviewed':True, | |
| 'pool_rows':800,'excluded_before_review':40,'scored_rows':760, | |
| 'high_screen_families_by_criterion':dict(Counter(q['criterion'] for q in ranked if int(scores[q['review_id']]['score'])>=85)), | |
| 'optional_400_status':'Not built: correctness has fewer than 100 high-screen families even before full-review and distinct-prompt constraints.', | |
| 'exclusions':'private/training_exclusions.json','training_fields':list(rows[0]), | |
| 'training_rule':'Only criterion and direction supervise the intervention loss; score is provenance/selection metadata, not a training target. Never train on review evidence.', | |
| 'preference_attribute_training':'NOT regenerated locally. GPU/CPU runner must apply the identical human-prompt exclusion list to every model and regenerate a common training-identity manifest before fitting.', | |
| 'feature_requests':{'file':'private/feature_requests.jsonl','unique_prompt_response_pairs':len(features), | |
| 'base_cache_hints':sum(r['base_response_id_hint'] is not None for r in features.values()),'new_features_needed':'Not assumed: verify exact-text cache hits first.'}, | |
| 'limitations':['All candidates are deterministic_v1 edits; generation diversity is narrow.', | |
| 'Correctness edits are mainly local numeric substitutions; coherence edits are sentence reorderings.', | |
| 'Scores describe differences, not globally correct base responses; shared pre-existing flaws can remain.', | |
| 'V2.1 has no independent human validation; old V2 intervals must not be relabeled as V2.1 precision.', | |
| 'Offline semantic review did not fetch external factual references.'], | |
| 'artifact_sha256':{str(p.relative_to(OUT)):sha(p) for p in [OUT/'silver_clean2d_train_200.jsonl',OUT/'silver_clean2d_train_100.jsonl',OUT/'silver_clean2d_manifest.csv',PRIVATE/'training_exclusions.json',PRIVATE/'human_v2_frozen_results.json',PRIVATE/'gpt6_screen.tsv',PRIVATE/'full_review_decisions.json',PRIVATE/'full_context_reviewed_ids.txt',PRIVATE/'filter_v21_all_candidates_scored.jsonl',PRIVATE/'feature_requests.jsonl']}}) | |
| put(OUT/'filter_v2_final_config.json',config) | |
| stats=json.loads((PRIVATE/'human_v2_frozen_results.json').read_text())['statistics'] | |
| lines=['# Human validation V2 — frozen report','', | |
| '20 条盲审标注保持原样；未根据失败案例回改 GPT 分数、标签或通过率。所有数字按私有 key 恢复到 original→edited；自由文本理由保留标注时 B 相对 A 的方向并明确命名，不伪装成已自动反转。','', | |
| '| 分组 | 通过 / 总数 | 通过率 | Wilson 95% CI |','|---|---:|---:|---:|'] | |
| for g in ['overall','high','boundary','correctness1','correctness-1','coherence1','coherence-1']: | |
| s=stats[g];lines.append(f"| {g} | {s['success']}/{s['n']} | {100*s['rate']:.1f}% | {100*s['wilson95'][0]:.1f}–{100*s['wilson95'][1]:.1f}% |") | |
| lines+=['','## 判据和失败分析','', | |
| 'Strict one-factor：目标维度与 requested direction 同向且绝对值至少 1，另外三个维度均为 0，other_meaningful_change=no。', | |
| '失败样本 HV-2875A52008E78B55 保留：任务允许真假陈述，裸事实真伪的修正不等于任务正确性提升。V2.1 增加 task-relative correctness guard，仅用于新候选。', | |
| '可引用：In a 20-example blinded human validation set, all 16 high-ranked candidates and 19/20 candidates overall satisfied the strict one-factor criterion.', | |
| '不可写“证明 100% precision”。样本按四格、高分/边界配额抽取，不是全池随机样本；Wilson 区间仅描述这批观测的不确定性，不能消除选样偏差。V2.1 尚无独立人标结果。','', | |
| '## 本次离线数据交付','', | |
| '当前归档池 800 行；排除与人标/留出集合重叠的 40 行后，直接 GPT-6 离线初筛 760 行（380 个互逆 family）。评分不是 API 批量调用，也不是校准概率。', | |
| '完整阅读拟入选 family 的 prompt、original 与可无损重建的 edited。复核 101 个 family，其中一条由 85 降至 75 并替换。最终每个 cell 50 行，总计 200 行、100 个不同 prompt、100 对独立文本；Silver100 是其中每格前 25 对方向行的嵌套子集。', | |
| '按 candidate_id、family_id、prompt_id 排除历史人工标注；额外对人标可用 prompt 文本做空白归一化重叠检查。保留 test/dev 原始排除集合；底层三张 parquet 的 SHA256 与原 source_lock 一致。这里不是重新解码 parquet 的独立 split 审计。', | |
| '这批数据适合继续训练检验假设，不等于已经证明因果可识别性。局部数字替换和句序模板可能成为捷径；正式结论需要自然人标 pairs、composition、五种子稳定性及偏好表现共同支持。即使训练失败，也不能仅凭这批小样本人标排除干预质量问题。','', | |
| '完整数据完整性检查见 integrity_checks.json；训练隔离和 feature cache 注意事项见 GPU_HANDOFF.md。原始证据和哈希保存在 private/，不进入模型输入。'] | |
| put(OUT/'human_validation_v2_report.md','\n'.join(lines)+'\n') | |
| verify() | |
| def verify(): | |
| config=json.loads((OUT/'filter_v2_final_config.json').read_text()) | |
| for rel,digest in config['artifact_sha256'].items():assert sha(OUT/rel)==digest,rel | |
| rows=readjl(OUT/'silver_clean2d_train_200.jsonl');small=readjl(OUT/'silver_clean2d_train_100.jsonl') | |
| manifest=readcsv(OUT/'silver_clean2d_manifest.csv');ex=json.loads((PRIVATE/'training_exclusions.json').read_text()) | |
| assert len(rows)==200 and len(manifest)==200 and len(small)==100 | |
| assert len({r['candidate_id'] for r in rows})==200 | |
| assert set(r['candidate_id'] for r in small)<=set(r['candidate_id'] for r in rows) | |
| for rs,n in [(rows,50),(small,25)]: | |
| assert Counter((r['criterion'],r['direction']) for r in rs)=={(c,d):n for c in DIMS[:2] for d in [-1,1]} | |
| assert all(set(r)==set(config['training_fields']) and r['gpt6_score']>=85 for r in rs) | |
| assert all(isinstance(r['direction'],int) and all(r[f] for f in ['prompt','original_response','edited_response','candidate_id']) for r in rs) | |
| assert len({r['prompt_id'] for r in manifest})==len({' '.join(r['prompt'].split()) for r in rows})==100 | |
| assert not {r['candidate_id'] for r in rows}&set(ex['human_candidate_ids']) | |
| assert not {r['family_id'] for r in manifest}&set(ex['human_family_ids']) | |
| assert not {r['prompt_id'] for r in manifest}&set(ex['silver_excluded_prompt_ids']) | |
| byid={r['candidate_id']:r for r in rows};families=defaultdict(list) | |
| for r in manifest:families[r['family_id']].append(byid[r['candidate_id']]) | |
| for rs in families.values(): | |
| assert len(rs)==2 and rs[0]['direction']==-rs[1]['direction'] | |
| assert rs[0]['prompt']==rs[1]['prompt'] and rs[0]['original_response']==rs[1]['edited_response'] and rs[0]['edited_response']==rs[1]['original_response'] | |
| frozen=json.loads((PRIVATE/'human_v2_frozen_results.json').read_text()) | |
| for path,digest in frozen['source_hashes'].items():assert sha(path)==digest | |
| result={'status':'PASS','train200_rows':200,'train100_rows':100,'rows_per_cell':50, | |
| 'unique_candidate_ids':200,'unique_families':100,'unique_prompt_ids':100,'unique_normalized_prompts':100, | |
| 'human_candidate_overlap':0,'human_family_overlap':0,'human_prompt_overlap':0, | |
| 'archived_nontrain_prompt_overlap':0,'all_selected_score_at_least_85':True, | |
| 'all_selected_full_context_reviewed':True,'schema_exact_seven_fields':True,'explanations_in_training':False, | |
| 'silver100_nested':True,'byte_exact_reverse_pairs':100,'source_and_output_hashes_match':True, | |
| 'v2_annotations_scores_and_keys_unchanged':True, | |
| 'formal_training_executed':False,'representation_extraction_executed':False} | |
| put(OUT/'integrity_checks.json',result);print(json.dumps(result,ensure_ascii=False)) | |
| if __name__=='__main__': | |
| ap=argparse.ArgumentParser();ap.add_argument('command',choices=['prepare','show','finalize','verify']);ap.add_argument('--start',type=int,default=1);ap.add_argument('--count',type=int,default=20);ap.add_argument('--full',action='store_true');ap.add_argument('--selected',action='store_true');args=ap.parse_args() | |
| if args.command=='prepare':prepare() | |
| elif args.command=='show':show(args.start,args.count,args.full,args.selected) | |
| elif args.command=='finalize':finalize() | |
| else:verify() | |
Xet Storage Details
- Size:
- 25.6 kB
- Xet hash:
- 888ba3cd7d15f8218f7fde0ac600de11dbef7ca24a6516d5312cf2347b563f11
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.