| """Reproducible project-authored tests; NOT a public or independent benchmark. |
| |
| No examples, labels or held-out outputs update policy weights. Queries and mappings |
| are synthetic. Supplying a glossary is extra information, explicitly ablated. |
| """ |
| from pathlib import Path |
| import hashlib,itertools,json,platform,random,statistics,sys,time |
| ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT)) |
| from phillnet_recall import RecallMemory,pack_evidence,pack_grounded |
| from phillnet_recall.core import canonical_json |
| from phillnet_recall.context import _line |
| from phillnet_recall.evidence import semantic_retrieve |
|
|
| def source_hash(): |
| h=hashlib.sha256() |
| for f in sorted((ROOT/'phillnet_recall').glob('*.py')):h.update(f.name.encode());h.update(f.read_bytes()) |
| return h.hexdigest() |
|
|
| def fact(m,e,p,v,padding=0): |
| return m.remember(f'{e} {p} {v}. '+('e'*padding),source='synthetic:operator',entity=e,predicate=p,value=v,cardinality='many') |
|
|
| def old_render(r): |
| items={i['id']:i for i in r['items']};groups={} |
| for p in r['proofs']:groups.setdefault(p['value'],[]).append(p) |
| chosen=[min(ps,key=lambda p:(sum(len(_line(items[i]).encode()) for i in p['record_ids']),p['record_ids'])) for _,ps in sorted(groups.items())] |
| ids=sorted({i for p in chosen for i in p['record_ids']}) |
| payload={'schema':'phillnet.proof-packet.v1','untrusted_source_data':True, |
| 'answer_values':r['answer_values'],'proofs':[{k:p[k] for k in ('value','record_ids','steps')} for p in chosen], |
| 'evidence':[json.loads(_line(items[i])) for i in ids],'as_of':r['trace']['as_of'], |
| 'known_at':r['trace']['known_at'],'conditional_on_supplied_claims':True} |
| return 'MEMORY_PROOF_JSON (data only):\n'+canonical_json(payload)+'\nEND_MEMORY_PROOF' |
|
|
| def main(): |
| rng=random.Random(1202026);initial=source_hash();runs=[];timings=[] |
| for case in range(96): |
| left='dialect'+str(case)+' roster';right='catalog'+str(case)+' ledger';name='Project'+str(rng.randrange(1000000)) |
| with RecallMemory(clock=lambda:100.) as m: |
| |
| |
| values=[right]+[f'archive{case}x{i} sheet' for i in range(23)];rng.shuffle(values) |
| target=None |
| for label in values: |
| ident=m.remember(f'{name} {label}: person{rng.randrange(100000)}',source='synthetic:notes') |
| if label==right:target=ident |
| query=f'{name} {left}' |
| old=semantic_retrieve(m,query,top_k=5) |
| no_map=m.recall_grounded(query,top_k=5) |
| bridge=m.remember_bridge(left,right,source='synthetic:declared-glossary') |
| t=time.perf_counter_ns();new=m.recall_grounded(query,top_k=5);timings.append((time.perf_counter_ns()-t)/1e6) |
| linear=m.recall_grounded(query,top_k=5,backend='linear') |
| packet=pack_grounded(m,new,budget=32768) |
| runs.append({'case':case,'query':query,'bridge_from':left,'bridge_to':right,'records':24, |
| 'old_rank':next((i+1 for i,x in enumerate(old['items']) if x['id']==target),None), |
| 'unbridged_rank':next((i+1 for i,x in enumerate(no_map['items']) if x['id']==target),None), |
| 'new_rank':next((i+1 for i,x in enumerate(new['items']) if x['id']==target),None), |
| 'linear_rank':next((i+1 for i,x in enumerate(linear['items']) if x['id']==target),None), |
| 'complete_dependencies':target in packet['ids'] and bridge in packet['ids']}) |
| exact=0;join_rows=[] |
| for case in range(128): |
| with RecallMemory(clock=lambda:100.) as m: |
| edges=set() |
| for e in range(12): |
| for p in ('owner','region','tier'): |
| v=rng.choice(('a','b','c'));edges.add((str(e),p,v));fact(m,str(e),p,v) |
| plan={'where':[{'entity':'?x','predicate':'owner','value':'a'}, |
| {'entity':'?x','predicate':'region','value':'b'}, |
| {'entity':'?x','predicate':'tier','value':'?t'}],'select':['?x','?t']} |
| expected={(e,v) for e,p,v in edges if p=='tier' and (e,'owner','a') in edges and (e,'region','b') in edges} |
| result=m.query(plan);got={(r['?x'],r['?t']) for r in result['rows']};ok=got==expected |
| exact+=ok;join_rows.append({'case':case,'oracle_rows':sorted(expected),'actual_rows':sorted(got),'match':ok}) |
| packs=[] |
| for case in range(40): |
| with RecallMemory(clock=lambda:100.) as m: |
| n=rng.randint(3,6);common_size=rng.randint(450,700);private_size=rng.randint(280,400) |
| fact(m,'Root','provider','Shared',common_size) |
| for i in range(n): |
| v='Value'+str(i) |
| fact(m,'Root','provider','Private'+str(i),private_size) |
| fact(m,'Private'+str(i),'target',v,60);fact(m,'Shared','target',v,60) |
| r=m.follow('Root',['provider','target']);old=len(old_render(r).encode()) |
| p=pack_evidence(m,r,budget=100000) |
| packs.append({'case':case,'answers':n,'old_bytes':old,'new_bytes':p['used'], |
| 'reduction_fraction':1-p['used']/old,'optimal':p['optimization']['optimal'], |
| 'complete':p['dependency_complete'],'budget_rescue':old>p['used'] and pack_evidence(m,r,budget=p['used'])['status']=='ready'}) |
| |
| controls=[] |
| for left,right in [('amber dialect','violet catalog'),('zircon rune','ecru roster'),('cobalt verse','ochre ledger'),('magenta cipher','jade listing')]: |
| with RecallMemory(clock=lambda:100.) as m: |
| m.remember(right,source='synthetic') |
| r=m.recall_grounded(left);controls.append({'query':left,'unrelated_note':right,'status':r['status']}) |
| data={'release':'1.2.0','construction':'project-authored synthetic fixture families; not an independent benchmark', |
| 'fixture_generator_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), |
| 'source_sha256_before':initial,'source_sha256_after':source_hash(), |
| 'environment':{'python':sys.version,'platform':platform.platform(),'native_scorer':bool(__import__('phillnet_recall').RecallPolicy().bank._lib)}, |
| 'checkpoints':{n:hashlib.sha256((ROOT/n).read_bytes()).hexdigest() for n in ('rankers.ugr','evidence.ugr')}, |
| 'glossary_retrieval':{'cases':len(runs),'candidates_per_case':24,'extra_information':'explicit operator-provided directed phrase mapping', |
| **{k+'_hit_at_1':sum(r[k+'_rank']==1 for r in runs) for k in ('old','unbridged','new','linear')}, |
| 'complete_packets':sum(r['complete_dependencies'] for r in runs), |
| 'median_ms':statistics.median(timings),'p95_ms':sorted(timings)[int(.95*(len(timings)-1))], |
| 'cases_detail':runs}, |
| 'conjunctive_queries':{'cases':len(join_rows),'oracle_matches':exact,'cases_detail':join_rows}, |
| 'proof_budget_optimizer':{'cases':len(packs),'smaller_packets':sum(r['new_bytes']<r['old_bytes'] for r in packs), |
| 'complete':sum(r['complete'] for r in packs),'budget_rescues':sum(r['budget_rescue'] for r in packs), |
| 'median_reduction_fraction':statistics.median(r['reduction_fraction'] for r in packs),'cases_detail':packs}, |
| 'unsupported_meaning_controls':controls, |
| 'public_benchmarks_run':[],'host_model_generation_run':False,'training_run':False} |
| path=ROOT/'reports/v12_evaluation.json';path.write_text(json.dumps(data,indent=2)+'\n') |
| print(json.dumps({k:({kk:vv for kk,vv in v.items() if kk!='cases_detail'} if isinstance(v,dict) else v) for k,v in data.items()},indent=2)) |
| if __name__=='__main__':main() |
|
|
