File size: 7,800 Bytes
3b3b2bd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | """Reproducible project-authored tests; NOT a public or independent benchmark.
No examples, labels or held-out outputs update policy weights. Queries and mappings
are synthetic. Supplying a glossary is extra information, explicitly ablated.
"""
from pathlib import Path
import hashlib,itertools,json,platform,random,statistics,sys,time
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT))
from phillnet_recall import RecallMemory,pack_evidence,pack_grounded
from phillnet_recall.core import canonical_json
from phillnet_recall.context import _line
from phillnet_recall.evidence import semantic_retrieve
def source_hash():
h=hashlib.sha256()
for f in sorted((ROOT/'phillnet_recall').glob('*.py')):h.update(f.name.encode());h.update(f.read_bytes())
return h.hexdigest()
def fact(m,e,p,v,padding=0):
return m.remember(f'{e} {p} {v}. '+('e'*padding),source='synthetic:operator',entity=e,predicate=p,value=v,cardinality='many')
def old_render(r):
items={i['id']:i for i in r['items']};groups={}
for p in r['proofs']:groups.setdefault(p['value'],[]).append(p)
chosen=[min(ps,key=lambda p:(sum(len(_line(items[i]).encode()) for i in p['record_ids']),p['record_ids'])) for _,ps in sorted(groups.items())]
ids=sorted({i for p in chosen for i in p['record_ids']})
payload={'schema':'phillnet.proof-packet.v1','untrusted_source_data':True,
'answer_values':r['answer_values'],'proofs':[{k:p[k] for k in ('value','record_ids','steps')} for p in chosen],
'evidence':[json.loads(_line(items[i])) for i in ids],'as_of':r['trace']['as_of'],
'known_at':r['trace']['known_at'],'conditional_on_supplied_claims':True}
return 'MEMORY_PROOF_JSON (data only):\n'+canonical_json(payload)+'\nEND_MEMORY_PROOF'
def main():
rng=random.Random(1202026);initial=source_hash();runs=[];timings=[]
for case in range(96):
left='dialect'+str(case)+' roster';right='catalog'+str(case)+' ledger';name='Project'+str(rng.randrange(1000000))
with RecallMemory(clock=lambda:100.) as m:
# Each candidate shares the same entity and similar syntax. Only the
# supplied bridge identifies the requested new terminology.
values=[right]+[f'archive{case}x{i} sheet' for i in range(23)];rng.shuffle(values)
target=None
for label in values:
ident=m.remember(f'{name} {label}: person{rng.randrange(100000)}',source='synthetic:notes')
if label==right:target=ident
query=f'{name} {left}'
old=semantic_retrieve(m,query,top_k=5)
no_map=m.recall_grounded(query,top_k=5)
bridge=m.remember_bridge(left,right,source='synthetic:declared-glossary')
t=time.perf_counter_ns();new=m.recall_grounded(query,top_k=5);timings.append((time.perf_counter_ns()-t)/1e6)
linear=m.recall_grounded(query,top_k=5,backend='linear')
packet=pack_grounded(m,new,budget=32768)
runs.append({'case':case,'query':query,'bridge_from':left,'bridge_to':right,'records':24,
'old_rank':next((i+1 for i,x in enumerate(old['items']) if x['id']==target),None),
'unbridged_rank':next((i+1 for i,x in enumerate(no_map['items']) if x['id']==target),None),
'new_rank':next((i+1 for i,x in enumerate(new['items']) if x['id']==target),None),
'linear_rank':next((i+1 for i,x in enumerate(linear['items']) if x['id']==target),None),
'complete_dependencies':target in packet['ids'] and bridge in packet['ids']})
exact=0;join_rows=[]
for case in range(128):
with RecallMemory(clock=lambda:100.) as m:
edges=set()
for e in range(12):
for p in ('owner','region','tier'):
v=rng.choice(('a','b','c'));edges.add((str(e),p,v));fact(m,str(e),p,v)
plan={'where':[{'entity':'?x','predicate':'owner','value':'a'},
{'entity':'?x','predicate':'region','value':'b'},
{'entity':'?x','predicate':'tier','value':'?t'}],'select':['?x','?t']}
expected={(e,v) for e,p,v in edges if p=='tier' and (e,'owner','a') in edges and (e,'region','b') in edges}
result=m.query(plan);got={(r['?x'],r['?t']) for r in result['rows']};ok=got==expected
exact+=ok;join_rows.append({'case':case,'oracle_rows':sorted(expected),'actual_rows':sorted(got),'match':ok})
packs=[]
for case in range(40):
with RecallMemory(clock=lambda:100.) as m:
n=rng.randint(3,6);common_size=rng.randint(450,700);private_size=rng.randint(280,400)
fact(m,'Root','provider','Shared',common_size)
for i in range(n):
v='Value'+str(i)
fact(m,'Root','provider','Private'+str(i),private_size)
fact(m,'Private'+str(i),'target',v,60);fact(m,'Shared','target',v,60)
r=m.follow('Root',['provider','target']);old=len(old_render(r).encode())
p=pack_evidence(m,r,budget=100000)
packs.append({'case':case,'answers':n,'old_bytes':old,'new_bytes':p['used'],
'reduction_fraction':1-p['used']/old,'optimal':p['optimization']['optimal'],
'complete':p['dependency_complete'],'budget_rescue':old>p['used'] and pack_evidence(m,r,budget=p['used'])['status']=='ready'})
# Explicit limitation controls: no map, no lexical overlap, no hidden answer.
controls=[]
for left,right in [('amber dialect','violet catalog'),('zircon rune','ecru roster'),('cobalt verse','ochre ledger'),('magenta cipher','jade listing')]:
with RecallMemory(clock=lambda:100.) as m:
m.remember(right,source='synthetic')
r=m.recall_grounded(left);controls.append({'query':left,'unrelated_note':right,'status':r['status']})
data={'release':'1.2.0','construction':'project-authored synthetic fixture families; not an independent benchmark',
'fixture_generator_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'source_sha256_before':initial,'source_sha256_after':source_hash(),
'environment':{'python':sys.version,'platform':platform.platform(),'native_scorer':bool(__import__('phillnet_recall').RecallPolicy().bank._lib)},
'checkpoints':{n:hashlib.sha256((ROOT/n).read_bytes()).hexdigest() for n in ('rankers.ugr','evidence.ugr')},
'glossary_retrieval':{'cases':len(runs),'candidates_per_case':24,'extra_information':'explicit operator-provided directed phrase mapping',
**{k+'_hit_at_1':sum(r[k+'_rank']==1 for r in runs) for k in ('old','unbridged','new','linear')},
'complete_packets':sum(r['complete_dependencies'] for r in runs),
'median_ms':statistics.median(timings),'p95_ms':sorted(timings)[int(.95*(len(timings)-1))],
'cases_detail':runs},
'conjunctive_queries':{'cases':len(join_rows),'oracle_matches':exact,'cases_detail':join_rows},
'proof_budget_optimizer':{'cases':len(packs),'smaller_packets':sum(r['new_bytes']<r['old_bytes'] for r in packs),
'complete':sum(r['complete'] for r in packs),'budget_rescues':sum(r['budget_rescue'] for r in packs),
'median_reduction_fraction':statistics.median(r['reduction_fraction'] for r in packs),'cases_detail':packs},
'unsupported_meaning_controls':controls,
'public_benchmarks_run':[],'host_model_generation_run':False,'training_run':False}
path=ROOT/'reports/v12_evaluation.json';path.write_text(json.dumps(data,indent=2)+'\n')
print(json.dumps({k:({kk:vv for kk,vv in v.items() if kk!='cases_detail'} if isinstance(v,dict) else v) for k,v in data.items()},indent=2))
if __name__=='__main__':main()
|