| """Deterministic spaced sample for literal-span and data-format diagnostics.""" |
| import ast,collections,csv,gzip,json,re,time |
| from pathlib import Path |
| from evaluate_model import ROOT,dump |
|
|
| def main(): |
| csv.field_size_limit(64*1024*1024);stats=collections.Counter();examples=[];lengths=[];start=time.time() |
| with gzip.open(ROOT/'final_gliner_file.csv.gz','rt',newline='') as stream: |
| for index,row in enumerate(csv.DictReader(stream),1): |
| if index%512!=137:continue |
| stats['sampled_rows']+=1 |
| try: |
| try:item=json.loads(row['gliner_ner']) |
| except json.JSONDecodeError:item=ast.literal_eval(row['gliner_ner']) |
| text=item['text'];entities=item['entities'];schema=item.get('entity_descriptions',{}) |
| stats['valid_sampled_rows']+=1;lengths.append(len(text.split())) |
| stats['schema_types']+=len(schema) |
| stats['schema_types_without_positive_mentions']+=sum(not entities.get(k) for k in schema) |
| stats['empty_gold_rows']+=not any(entities.values()) |
| for label,values in entities.items(): |
| if not isinstance(values,list):stats['nonlist_values']+=1;continue |
| for value in values: |
| stats['mentions']+=1 |
| if not isinstance(value,str):stats['nonstring_mentions']+=1;continue |
| if not value:stats['empty_string_mentions']+=1;continue |
| stats['mentions_over8_whitespace_words']+=len(value.split())>8 |
| stats['mentions_over8_regex_units']+=len(re.findall(r'\w+(?:[-_]\w+)*|\S',value))>8 |
| n=text.count(value) |
| if n==0: |
| stats['mentions_not_literal_substrings']+=1 |
| if value.casefold() in text.casefold():stats['case_only_mismatch']+=1 |
| if len(examples)<30:examples.append({'row':index,'label':label,'mention':value,'text':text}) |
| elif n>1:stats['mentions_with_multiple_literal_occurrences']+=1 |
| except Exception:stats['parse_errors']+=1 |
| summary={'method':'Every 512th CSV record, offset 137; spaced sample rather than a random or domain-stratified sample.', |
| 'counts':dict(stats),'seconds':time.time()-start, |
| 'text_word_lengths':{'min':min(lengths),'median':sorted(lengths)[len(lengths)//2],'max':max(lengths)}} |
| dump(ROOT/'investigation/training-span-quality.json',summary) |
| |
| dump(ROOT/'investigation/private-training-mismatch-examples.json',examples) |
| print(json.dumps(summary,indent=2),flush=True) |
|
|
| if __name__=='__main__':main() |
|
|
