package dibaguard import ( "container/list" "math" ) // Per-entity stream state, a faithful port of dibaguard/flows/entity.py (same counters, same decay, same // bounded memories), so the entity features a Go load balancer computes are the numbers the model was trained on. const ( entK = 10 // counters per horizon: flows, bytes_in, pkts_in, noreply, synonly, rst, newport, newhost, repeat, short entMemory = 512 // (host, port) keys remembered per source / sources per destination entGapAlpha = 0.2 ) var DefaultHorizons = []float64{10, 120, 1800} type entKey struct { host string port int } type entRec struct { t float64 c []float64 seen map[entKey]*list.Element // key -> element in order (value: seenEntry) order *list.List hosts map[string]int gapEwma float64 n int // flows ever seen (the source's true age, not a decayed count) lru *list.Element // position in the table's LRU key string } type seenEntry struct { key entKey host string } type entTable struct { m map[string]*entRec lru *list.List max int } func newTable(max int) *entTable { return &entTable{m: map[string]*entRec{}, lru: list.New(), max: max} } // EntityState keeps decayed counters for sources (full feature set) and destinations (fan-in only). type EntityState struct { tau []float64 src, dst *entTable } // NewEntityState mirrors EntityState(horizons, max_entities) in Python. func NewEntityState(horizons []float64, maxEntities int) *EntityState { if horizons == nil { horizons = DefaultHorizons } tau := make([]float64, len(horizons)) for i, h := range horizons { tau[i] = h / math.Ln2 } return &EntityState{tau: tau, src: newTable(maxEntities), dst: newTable(maxEntities)} } // NumFeatures is len(feature_names()) in Python. func (s *EntityState) NumFeatures() int { return len(s.tau)*entK + 6 } // get is EntityState._get: (record decayed to t, gap since its last event (-1 when new), lateness). A late record // never rewinds the record's clock (V-12); a full table evicts the least recently used record, passing over up to 8 // that earned state (8+ flows), so a spray of fresh sources cannot flush a scanner's record (V-13, decide.evict). func (s *EntityState) get(tb *entTable, key string, t float64) (*entRec, float64, float64) { r, ok := tb.m[key] if !ok { r = &entRec{t: t, c: make([]float64, len(s.tau)*entK), seen: map[entKey]*list.Element{}, order: list.New(), hosts: map[string]int{}, gapEwma: -1, key: key} tb.m[key] = r r.lru = tb.lru.PushBack(r) if len(tb.m) > tb.max { tb.evict() } return r, -1, 0 } tb.lru.MoveToBack(r.lru) dt := t - r.t if dt > 0 { for i, tau := range s.tau { f := math.Exp(-dt / tau) for j := i * entK; j < i*entK+entK; j++ { r.c[j] *= f } } r.t = t return r, dt, 0 } return r, 0, -dt } // evict is decide.evict(table, lambda v: v.n >= 8): least recently used first, a record with a history goes to the // back of the queue instead, up to 8 times; then the oldest goes whatever it holds. func (tb *entTable) evict() { for i := 0; i < 8; i++ { f := tb.lru.Front() r := f.Value.(*entRec) if r.n < 8 { tb.lru.Remove(f) delete(tb.m, r.key) return } tb.lru.MoveToBack(f) } old := tb.lru.Remove(tb.lru.Front()).(*entRec) delete(tb.m, old.key) } func remember(r *entRec, key entKey, host string) { if el, ok := r.seen[key]; ok { r.order.MoveToBack(el) return } r.seen[key] = r.order.PushBack(seenEntry{key, host}) r.hosts[host]++ if r.order.Len() > entMemory { old := r.order.Remove(r.order.Front()).(seenEntry) delete(r.seen, old.key) if n := r.hosts[old.host] - 1; n > 0 { r.hosts[old.host] = n } else { delete(r.hosts, old.host) } } } func b2f(b bool) float64 { if b { return 1 } return 0 } // Update appends this flow's entity features to out (NumFeatures values) and absorbs the flow. func (s *EntityState) Update(t float64, src, dst string, dport int, bytesIn, pktsIn, pktsOut float64, synOnly, rst, short bool, out []float32) []float32 { r, gap, late := s.get(s.src, src, t) d, _, dlate := s.get(s.dst, dst, t) r.n++ key := entKey{dst, dport} _, seen := r.seen[key] _, hostKnown := r.hosts[dst] newHost := !hostKnown _, srcKnown := d.seen[entKey{src, 0}] newSrc := !srcKnown noreply := pktsOut <= 0 ev := [entK]float64{1, bytesIn, pktsIn, b2f(noreply), b2f(synOnly), b2f(rst), b2f(!seen), b2f(newHost), b2f(seen), b2f(short)} // an out-of-order record (older than the record's clock) counts in each horizon at its decayed weight; in order, // every weight is 1 (EntityState.update / _update_late) w := make([]float64, len(s.tau)) dw := make([]float64, len(s.tau)) for i, tau := range s.tau { w[i], dw[i] = 1, 1 if late > 0 || dlate > 0 { w[i] = math.Max(math.Exp(-late/tau), 1e-9) dw[i] = math.Max(math.Exp(-dlate/tau), 1e-9) } } c := r.c for i := range s.tau { b, wi := i*entK, w[i] n := c[b] + wi out = append(out, float32(n), float32(c[b+1]+wi*ev[1]), float32(c[b+2]+wi*ev[2])) for j := 3; j < entK; j++ { out = append(out, float32((c[b+j]+wi*ev[j])/n)) } } out = append(out, float32(gap), float32(r.gapEwma)) dc := d.c n1 := dc[0] + dw[0] out = append(out, float32(n1), float32((dc[3]+dw[0]*b2f(noreply))/n1), float32((dc[6]+dw[0]*b2f(newSrc))/n1), float32(dc[entK]+dw[1])) for i := range s.tau { b := i * entK for j := 0; j < entK; j++ { c[b+j] += w[i] * ev[j] } dc[b] += dw[i] dc[b+3] += dw[i] * b2f(noreply) dc[b+6] += dw[i] * b2f(newSrc) } remember(r, key, dst) remember(d, entKey{src, 0}, src) if gap >= 0 { if r.gapEwma < 0 { r.gapEwma = gap } else { r.gapEwma = (1-entGapAlpha)*r.gapEwma + entGapAlpha*gap } } return out }