Buckets:
| import json | |
| import numpy as np,pandas as pd | |
| from data.splits import prompt_id,assert_disjoint | |
| from representations.cache import FeatureCache | |
| from intervention.natural_pairs import natural_pairs | |
| def test_real_prompt_split_and_preference_sign(): | |
| ex=pd.read_parquet('data/processed/responses.parquet');p=pd.read_parquet('data/processed/preferences.parquet') | |
| assert_disjoint(ex);assert len(ex.response_id.unique())==len(ex) | |
| assert ((p[p.strength<0].label)==1).all();assert ((p[p.strength>0].label)==0).all() | |
| assert (p.a_index!=p.b_index).all() | |
| assert prompt_id('a b\n')==prompt_id('a b') | |
| def test_cache_resume_and_provenance(tmp_path): | |
| manifest={'n':3,'d':2,'layers':[1],'source':'abc'};c=FeatureCache(tmp_path,manifest) | |
| vals={k:np.ones((1,2),dtype='float16') for k in c.arrays};c.write([2],vals) | |
| d=FeatureCache(tmp_path,manifest);assert d.done.tolist()==[False,False,True] | |
| np.testing.assert_array_equal(d.arrays['layer1_mean'][2],np.ones(2)) | |
| import pytest | |
| with pytest.raises(AssertionError):FeatureCache(tmp_path,{**manifest,'source':'different'}) | |
| def test_natural_pairs_must_be_same_test_prompt(): | |
| ex=pd.DataFrame({'split':['test']*4,'prompt_id':['a','a','b','b']});a=np.array([[0,0,0,0],[2,0,0,0],[0,0,0,0],[0,2,1,0]]) | |
| p=natural_pairs(ex,a);assert p.tolist()==[[0.,1.,0.,1.]] | |
Xet Storage Details
- Size:
- 1.33 kB
- Xet hash:
- 62f963b6f58e6cebad5e9b30660eba5d3edb8f73b6816a4c41c78475f2638037
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.