DibaGuard / runtime /go /entity.go
DibaAi's picture
DibaGuard: public licence-clean release (Dibachain)
5d97275 verified
Raw History Blame Contribute Delete
5.75 kB
package dibaguard
import (
"container/list"
"math"
)
// Per-entity stream state, a faithful port of dibaguard/flows/entity.py (same counters, same decay, same
// bounded memories), so the entity features a Go load balancer computes are the numbers the model was trained on.
const (
entK = 10 // counters per horizon: flows, bytes_in, pkts_in, noreply, synonly, rst, newport, newhost, repeat, short
entMemory = 512 // (host, port) keys remembered per source / sources per destination
entGapAlpha = 0.2
)
var DefaultHorizons = []float64{10, 120, 1800}
type entKey struct {
host string
port int
}
type entRec struct {
t float64
c []float64
seen map[entKey]*list.Element // key -> element in order (value: seenEntry)
order *list.List
hosts map[string]int
gapEwma float64
n int // flows ever seen (the source's true age, not a decayed count)
lru *list.Element // position in the table's LRU
key string
}
type seenEntry struct {
key entKey
host string
}
type entTable struct {
m map[string]*entRec
lru *list.List
max int
}
func newTable(max int) *entTable {
return &entTable{m: map[string]*entRec{}, lru: list.New(), max: max}
}
// EntityState keeps decayed counters for sources (full feature set) and destinations (fan-in only).
type EntityState struct {
tau []float64
src, dst *entTable
}
// NewEntityState mirrors EntityState(horizons, max_entities) in Python.
func NewEntityState(horizons []float64, maxEntities int) *EntityState {
if horizons == nil {
horizons = DefaultHorizons
}
tau := make([]float64, len(horizons))
for i, h := range horizons {
tau[i] = h / math.Ln2
}
return &EntityState{tau: tau, src: newTable(maxEntities), dst: newTable(maxEntities)}
}
// NumFeatures is len(feature_names()) in Python.
func (s *EntityState) NumFeatures() int { return len(s.tau)*entK + 6 }
// get is EntityState._get: (record decayed to t, gap since its last event (-1 when new), lateness). A late record
// never rewinds the record's clock (V-12); a full table evicts the least recently used record, passing over up to 8
// that earned state (8+ flows), so a spray of fresh sources cannot flush a scanner's record (V-13, decide.evict).
func (s *EntityState) get(tb *entTable, key string, t float64) (*entRec, float64, float64) {
r, ok := tb.m[key]
if !ok {
r = &entRec{t: t, c: make([]float64, len(s.tau)*entK), seen: map[entKey]*list.Element{}, order: list.New(),
hosts: map[string]int{}, gapEwma: -1, key: key}
tb.m[key] = r
r.lru = tb.lru.PushBack(r)
if len(tb.m) > tb.max {
tb.evict()
}
return r, -1, 0
}
tb.lru.MoveToBack(r.lru)
dt := t - r.t
if dt > 0 {
for i, tau := range s.tau {
f := math.Exp(-dt / tau)
for j := i * entK; j < i*entK+entK; j++ {
r.c[j] *= f
}
}
r.t = t
return r, dt, 0
}
return r, 0, -dt
}
// evict is decide.evict(table, lambda v: v.n >= 8): least recently used first, a record with a history goes to the
// back of the queue instead, up to 8 times; then the oldest goes whatever it holds.
func (tb *entTable) evict() {
for i := 0; i < 8; i++ {
f := tb.lru.Front()
r := f.Value.(*entRec)
if r.n < 8 {
tb.lru.Remove(f)
delete(tb.m, r.key)
return
}
tb.lru.MoveToBack(f)
}
old := tb.lru.Remove(tb.lru.Front()).(*entRec)
delete(tb.m, old.key)
}
func remember(r *entRec, key entKey, host string) {
if el, ok := r.seen[key]; ok {
r.order.MoveToBack(el)
return
}
r.seen[key] = r.order.PushBack(seenEntry{key, host})
r.hosts[host]++
if r.order.Len() > entMemory {
old := r.order.Remove(r.order.Front()).(seenEntry)
delete(r.seen, old.key)
if n := r.hosts[old.host] - 1; n > 0 {
r.hosts[old.host] = n
} else {
delete(r.hosts, old.host)
}
}
}
func b2f(b bool) float64 {
if b {
return 1
}
return 0
}
// Update appends this flow's entity features to out (NumFeatures values) and absorbs the flow.
func (s *EntityState) Update(t float64, src, dst string, dport int, bytesIn, pktsIn, pktsOut float64,
synOnly, rst, short bool, out []float32) []float32 {
r, gap, late := s.get(s.src, src, t)
d, _, dlate := s.get(s.dst, dst, t)
r.n++
key := entKey{dst, dport}
_, seen := r.seen[key]
_, hostKnown := r.hosts[dst]
newHost := !hostKnown
_, srcKnown := d.seen[entKey{src, 0}]
newSrc := !srcKnown
noreply := pktsOut <= 0
ev := [entK]float64{1, bytesIn, pktsIn, b2f(noreply), b2f(synOnly), b2f(rst), b2f(!seen), b2f(newHost), b2f(seen), b2f(short)}
// an out-of-order record (older than the record's clock) counts in each horizon at its decayed weight; in order,
// every weight is 1 (EntityState.update / _update_late)
w := make([]float64, len(s.tau))
dw := make([]float64, len(s.tau))
for i, tau := range s.tau {
w[i], dw[i] = 1, 1
if late > 0 || dlate > 0 {
w[i] = math.Max(math.Exp(-late/tau), 1e-9)
dw[i] = math.Max(math.Exp(-dlate/tau), 1e-9)
}
}
c := r.c
for i := range s.tau {
b, wi := i*entK, w[i]
n := c[b] + wi
out = append(out, float32(n), float32(c[b+1]+wi*ev[1]), float32(c[b+2]+wi*ev[2]))
for j := 3; j < entK; j++ {
out = append(out, float32((c[b+j]+wi*ev[j])/n))
}
}
out = append(out, float32(gap), float32(r.gapEwma))
dc := d.c
n1 := dc[0] + dw[0]
out = append(out, float32(n1), float32((dc[3]+dw[0]*b2f(noreply))/n1), float32((dc[6]+dw[0]*b2f(newSrc))/n1), float32(dc[entK]+dw[1]))
for i := range s.tau {
b := i * entK
for j := 0; j < entK; j++ {
c[b+j] += w[i] * ev[j]
}
dc[b] += dw[i]
dc[b+3] += dw[i] * b2f(noreply)
dc[b+6] += dw[i] * b2f(newSrc)
}
remember(r, key, dst)
remember(d, entKey{src, 0}, src)
if gap >= 0 {
if r.gapEwma < 0 {
r.gapEwma = gap
} else {
r.gapEwma = (1-entGapAlpha)*r.gapEwma + entGapAlpha*gap
}
}
return out
}