File size: 1,618 Bytes
aa02c28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
"""Eight logical ranks, including ranks without observations, reduce once."""
import json
from types import SimpleNamespace
import torch
import torch.distributed as dist
import torch.multiprocessing as mp


def worker(rank, rendezvous, output):
    from psi.models.interaction_arch.control_metrics import flush
    torch.set_num_threads(1)
    dist.init_process_group('gloo', init_method='file://'+rendezvous, rank=rank, world_size=8)
    try:
        trainer=SimpleNamespace(model=SimpleNamespace(), accelerator=SimpleNamespace(
            device=torch.device('cpu'), unwrap_model=lambda model:model))
        if rank < 2:
            cm=torch.zeros(9,dtype=torch.long)
            cm[rank]=3 if rank==0 else 1
            trainer._token_control_meters={'train':dict(confusion=cm, paired=0,
                gt_switch=0,pred_switch=0,exact_switch=0,false_switch=0,triplets=0,flicker=0)}
        metrics=flush(trainer,'train')
        assert metrics['train/token_control_accuracy']==0.75
        assert metrics['train/token_control_cm_listen_listen']==3
        assert metrics['train/token_control_cm_listen_motion']==1
        assert flush(trainer,'train')=={}
        if rank==0:
            with open(output,'w') as stream:json.dump(metrics,stream)
    finally:
        dist.destroy_process_group()


def test_eight_rank_metric_reduction_with_empty_ranks(tmp_path):
    mp.start_processes(worker,args=(str(tmp_path/'init'),str(tmp_path/'metrics.json')),
                       nprocs=8,join=True,start_method='fork')
    assert json.loads((tmp_path/'metrics.json').read_text())['train/token_control_accuracy']==.75