source/pkg/bundle/analyzer.go

package bundle

import (
	"regexp"
	"sort"
	"strings"
)

// Severity ranks the confidence/impact of a Finding. The numeric ordering is
// load-bearing: bundleEscalates and the heuristic bridge both key off
// severityToWeight(SevHigh) >= 0.85 as the single source of truth.
type Severity int

const (
	SevInfo Severity = iota
	SevLow
	SevMedium
	SevHigh
	SevCritical
)

// String renders a Severity for JSON/SARIF output and human logs.
func (s Severity) String() string {
	switch s {
	case SevInfo:
		return "info"
	case SevLow:
		return "low"
	case SevMedium:
		return "medium"
	case SevHigh:
		return "high"
	case SevCritical:
		return "critical"
	default:
		return "unknown"
	}
}

// Finding is a single per-sibling (or per-archive-member) detection produced by
// an Analyzer. Structural marks shape-only signals (ships-a-.so,
// delegates-to-a-file) that must NOT escalate to malicious on their own;
// Corroborated marks that a behavioral co-factor (exfil+env, source mismatch,
// padding, hidden placement) was observed. The precision-aware escalation in
// aggregate.go consumes exactly these two flags.
type Finding struct {
	Analyzer     string   `json:"analyzer"`
	File         string   `json:"file"`   // RelPath of offending file (or member path inside an archive)
	Signal       string   `json:"signal"` // stable signal name, e.g. "archive-contains-executable"
	Severity     Severity `json:"severity"`
	Detail       string   `json:"detail"`
	Line         int      `json:"line,omitempty"`
	Opaque       bool     `json:"opaque,omitempty"`       // content could not be fully analyzed
	Structural   bool     `json:"structural,omitempty"`   // signal is about shape, not behavior
	Corroborated bool     `json:"corroborated,omitempty"` // a behavioral co-factor was observed
}

// Analyzer inspects one File (by FileKind) within a Bundle and returns Findings.
// Handles reports which kinds an analyzer claims; Analyze must never panic and
// should flag opaque artifacts rather than silently ignoring them.
type Analyzer interface {
	Name() string
	Handles(kind FileKind) bool
	Analyze(f *File, b *Bundle) ([]Finding, error)
}

// severityToWeight is the SINGLE SOURCE OF TRUTH mapping severities to the
// heuristic-engine weight space. SevHigh maps to 0.85 so that the bridged
// CategoryScore clears the scorer's cat.Score>=0.85 escalation gate.
func severityToWeight(s Severity) float64 {
	switch s {
	case SevInfo:
		return 0.0
	case SevLow:
		return 0.4
	case SevMedium:
		return 0.6
	case SevHigh:
		return 0.85
	case SevCritical:
		return 0.95
	default:
		return 0.0
	}
}

// DefaultAnalyzers returns the registry of concrete analyzers. File-content
// analyzers come first; the SKILL.md-level cross-reference analyzers
// (Indirection, NLDirective) are listed last because aggregate.Analyze runs
// them after the per-file pass so they can corroborate against the full
// findings set.
func DefaultAnalyzers() []Analyzer {
	return []Analyzer{
		ShellAnalyzer{},
		PythonSourceAnalyzer{},
		ScriptOtherAnalyzer{},
		DataAnalyzer{},
		PycAnalyzer{},
		ArchiveAnalyzer{},
		BinaryAnalyzer{},
		ImageAnalyzer{},
		IndirectionAnalyzer{},
		NLDirectiveAnalyzer{},
	}
}

// AnalyzeFile dispatches a single File to every analyzer that Handles its kind
// and concatenates their Findings. Errors from individual analyzers are folded
// into an opaque Finding rather than aborting (graceful degradation); a nil
// File yields nothing.
func AnalyzeFile(f *File, b *Bundle) []Finding {
	if f == nil {
		return nil
	}
	var out []Finding
	for _, a := range DefaultAnalyzers() {
		if !a.Handles(f.Kind) {
			continue
		}
		findings, err := safeAnalyze(a, f, b)
		if err != nil {
			out = append(out, Finding{
				Analyzer: a.Name(),
				File:     f.RelPath,
				Signal:   "analyzer-error",
				Severity: SevLow,
				Opaque:   true,
				Detail:   "analyzer failed: " + err.Error(),
			})
			continue
		}
		out = append(out, findings...)
	}
	return out
}

// safeAnalyze runs an analyzer and converts a panic on malformed input into an
// error so a single corrupt artifact can never crash a scan.
func safeAnalyze(a Analyzer, f *File, b *Bundle) (findings []Finding, err error) {
	defer func() {
		if r := recover(); r != nil {
			findings = nil
			err = &analyzerPanic{name: a.Name(), v: r}
		}
	}()
	return a.Analyze(f, b)
}

type analyzerPanic struct {
	name string
	v    any
}

func (e *analyzerPanic) Error() string {
	return e.name + " panicked on malformed input"
}

// ---- Shared indicator vocabulary used across the source analyzers ----

// indicator families. A "source" read CO-OCCURRING (within a small line window)
// with a "sink" is the high-confidence exfil pattern; isolated members are
// lower confidence. Kept lowercase; callers lowercase the text.
var (
	envSourceTerms = []string{
		"os.environ", "os.getenv", "process.env", "getenv(", "$env:",
		"printenv", "/proc/self/environ", "env |", "env >",
		".aws/credentials", "~/.aws", "~/.ssh", "id_rsa", "id_ed25519",
		".npmrc", ".pypirc", ".netrc", "aws_secret_access_key",
		"aws_access_key_id", "anthropic_api_key", "openai_api_key",
		"access_token", "auth_token", "secret_key", "private key",
		"cat .env", "read .env", "${{ secrets", "secrets.",
	}
	networkSinkTerms = []string{
		"curl ", "wget ", "requests.post", "requests.get", "urllib",
		"http.client", "socket.", "net::http", "open-uri", "net/http",
		"invoke-webrequest", "invoke-restmethod", "system.net.webclient",
		"child_process", "fetch(", "axios", "nc ", "ncat ", " -d ",
		"--data", "xmlhttprequest", "webclient", "uploadstring",
	}
	rceTerms = []string{
		"curl | sh", "curl|sh", "curl | bash", "curl|bash",
		"wget | sh", "wget|sh", "| sudo bash", "|sh", "|bash",
		"base64 -d | sh", "base64 --decode | sh", "iex(", "iex (",
		"eval(atob", "eval(base64", "exec(base64", "exec(__import__",
	}
	destructiveTerms = []string{
		"rm -rf /", "rm -rf ~", "rm -rf .", ":(){ :|:& };:",
		"dd if=/dev/zero", "dd if=/dev/random", "mkfs", "mkfs.",
		"> /dev/sda", "chmod -r 777 /", "format c:",
	}
	registryRewriteTerms = []string{
		"registry=", "registry =", "set registry", "config set registry",
		"npm config set registry", "yarn config set registry",
		"--index-url", "--extra-index-url", "global.index-url",
		"pip config set global.index-url", "[global]\nindex-url",
		"publishconfig", ".npmrc", "set-pypiserver",
	}
	// reverseShellTerms are host-agnostic reverse/bind-shell idioms: the payload
	// is "give an attacker an interactive shell", independent of any host string.
	reverseShellTerms = []string{
		"/dev/tcp/", "/dev/udp/", // bash pseudo-device network shell
		"nc -e", "ncat -e", "nc -c", "ncat -c", // netcat -e/-c command execution
		"exec 5<>/dev/tcp", "0>&1", // fd-dup reverse shell plumbing
	}
	// persistenceTerms are host-agnostic persistence/scheduling mechanisms (cron,
	// init, login shells, service managers). Persistence ALONE is only INFO (many
	// installers schedule jobs); persistence CO-OCCURRING with a network sink in
	// the same file is the host-agnostic "scheduled phone-home" escalator.
	persistenceTerms = []string{
		"crontab", "/etc/cron", "cron.d", "* * * *", "*/", // cron
		"systemctl enable", "systemctl --user enable", "/etc/systemd", "/lib/systemd",
		"launchctl load", "launchagents", "launchdaemons", // macOS launchd
		"/etc/rc.local", "/etc/profile.d", "schtasks /create", "schtasks/create",
		".bashrc", ".zshrc", ".bash_profile", ".zprofile", ".profile", // login-shell rc append
	}
	base64BlobRe = regexp.MustCompile(`[A-Za-z0-9+/]{120,}={0,2}`)
	longHexRe    = regexp.MustCompile(`(?i)(?:0x)?[0-9a-f]{80,}`)
	// The defanged exfil host the corpus uses; appearance anywhere is critical.
	exfilHostRe = regexp.MustCompile(`(?i)(attacker\.example|198\.51\.100\.\d{1,3})`)
	urlRe       = regexp.MustCompile(`(?i)https?://[^\s'")>]+`)
	hostFromURL = regexp.MustCompile(`(?i)https?://([^/\s:'")>]+)`)
	// shellVarRefRe captures a shell variable reference ($MIRROR / ${MIRROR:-...})
	// so a registry rewrite that points at "$MIRROR" can be resolved to the URL in
	// the variable's assignment line elsewhere in the same script.
	shellVarRefRe = regexp.MustCompile(`\$\{?([A-Za-z_][A-Za-z0-9_]*)`)
)

// sharedIndicatorScan applies the language-agnostic source/sink vocabulary over
// a block of text and returns line-anchored Findings. SevHigh+Corroborated when
// an env/secret source co-occurs with a network sink within a 3-line window, or
// on RCE/destructive/registry-rewrite patterns; isolated indicators are
// SevMedium. The defanged exfil host alone is SevCritical. A merely-present
// script with no indicator returns nothing (presence != malice).
//
// When an embedded base64/hex/gzip blob is found, it is also DECODED and
// re-scanned (decode.go); a behavioral hit in the recovered bytes adds a
// corroborated SevHigh decoded-* finding so the precision gate escalates an
// otherwise opaque obfuscated payload.
func sharedIndicatorScan(text string, fileRel string, analyzer string) []Finding {
	return sharedIndicatorScanInner(text, fileRel, analyzer, true)
}

// sharedIndicatorScanInner is the implementation; decode controls whether
// embedded blobs are decoded-and-rescanned. decodeAndRescan re-scans recovered
// bytes with decode=false so the bounded multi-layer decode (handled inside
// decode.go) is the ONLY recursion path — this scanner never re-enters itself.
func sharedIndicatorScanInner(text string, fileRel string, analyzer string, decode bool) []Finding {
	var out []Finding
	lines := strings.Split(text, "\n")
	lower := make([]string, len(lines))
	for i, l := range lines {
		lower[i] = strings.ToLower(l)
	}

	// Exfil host (defanged) — EVIDENCE ONLY. This matches the corpus's own
	// defanged host literally (attacker.example / TEST-NET), so escalating on it
	// is self-grading: swap the host and detection vanishes. It is kept as
	// SevMedium, non-corroborated NETWORK_EGRESS supporting evidence and never
	// escalates a bundle on its own — host-agnostic signals (reverse-shell,
	// exfil-env-to-network source+sink, scheduled-network-callback, RCE, etc.)
	// carry the escalation.
	for i, l := range lines {
		if exfilHostRe.MatchString(l) {
			out = append(out, Finding{
				Analyzer: analyzer,
				File:     fileRel,
				Signal:   "exfil-host-reference",
				Severity: SevMedium,
				Detail:   "references a known exfiltration host (evidence only): " + exfilHostRe.FindString(l),
				Line:     i + 1,
			})
		}
	}

	// Reverse/bind shell — host-agnostic remote-control payload.
	for i, l := range lower {
		if matchedAny(l, reverseShellTerms) {
			out = append(out, Finding{
				Analyzer:     analyzer,
				File:         fileRel,
				Signal:       "reverse-shell",
				Severity:     SevHigh,
				Detail:       "reverse/bind shell idiom — hands an attacker an interactive shell",
				Line:         i + 1,
				Corroborated: true,
			})
		}
	}

	// Persistence: scheduling/init/login-shell install. Escalates (host-agnostic)
	// only when a network sink also appears in the file (scheduled phone-home);
	// persistence on its own is reported as a low-power PERSISTENCE capability.
	fileHasNetworkSink := matchedAny(strings.ToLower(text), networkSinkTerms)
	for i, l := range lower {
		if !matchedAny(l, persistenceTerms) {
			continue
		}
		if fileHasNetworkSink {
			out = append(out, Finding{
				Analyzer:     analyzer,
				File:         fileRel,
				Signal:       "scheduled-network-callback",
				Severity:     SevHigh,
				Detail:       "installs a scheduled/persistent job that also performs network I/O",
				Line:         i + 1,
				Corroborated: true,
			})
		} else {
			out = append(out, Finding{
				Analyzer: analyzer,
				File:     fileRel,
				Signal:   "persistence",
				Severity: SevMedium,
				Detail:   "installs a persistence/scheduling mechanism (cron/init/login-shell)",
				Line:     i + 1,
			})
		}
		break // one persistence finding per file is enough
	}

	// RCE: pipe-to-shell / decode-and-exec.
	for i, l := range lower {
		if matchedAny(l, rceTerms) {
			out = append(out, Finding{
				Analyzer:     analyzer,
				File:         fileRel,
				Signal:       "remote-code-execution",
				Severity:     SevHigh,
				Detail:       "downloads-and-executes or decodes-and-executes a payload",
				Line:         i + 1,
				Corroborated: true,
			})
		}
	}

	// Destructive commands.
	for i, l := range lower {
		if matchedAny(l, destructiveTerms) {
			out = append(out, Finding{
				Analyzer:     analyzer,
				File:         fileRel,
				Signal:       "destructive-command",
				Severity:     SevHigh,
				Detail:       "destructive filesystem/disk command",
				Line:         i + 1,
				Corroborated: true,
			})
		}
	}

	// Registry rewrite to a non-default host (dev-env-setup vector).
	for i, l := range lower {
		if !matchedAny(l, registryRewriteTerms) {
			continue
		}
		host := resolveRegistryHost(lines, i)
		sev := SevHigh
		detail := "package-manager registry/index rewrite to a non-default host"
		if host != "" && isKnownBenignHost(host) {
			sev = SevLow
			detail = "registry rewrite to known-benign corporate mirror: " + host
		} else if host != "" && isInternalRegistryHost(host) {
			sev = SevLow
			detail = "registry rewrite to internal/private mirror host: " + host
		} else if isKnownBenignScriptIdiom(lines[i]) {
			sev = SevLow
			detail = "registry rewrite matching a known-benign idiom"
		}
		out = append(out, Finding{
			Analyzer:     analyzer,
			File:         fileRel,
			Signal:       "registry-rewrite",
			Severity:     sev,
			Detail:       detail,
			Line:         i + 1,
			Corroborated: sev >= SevHigh,
		})
	}

	// Source<->sink co-occurrence within a small window.
	for i := range lower {
		window := lower[i]
		if i+1 < len(lower) {
			window += "\n" + lower[i+1]
		}
		if i+2 < len(lower) {
			window += "\n" + lower[i+2]
		}
		hasSource := matchedAny(window, envSourceTerms)
		hasSink := matchedAny(window, networkSinkTerms)
		switch {
		case hasSource && hasSink:
			out = append(out, Finding{
				Analyzer:     analyzer,
				File:         fileRel,
				Signal:       "exfil-env-to-network",
				Severity:     SevHigh,
				Detail:       "environment/credential read co-occurs with a network sink",
				Line:         i + 1,
				Corroborated: true,
			})
		case hasSource:
			out = append(out, Finding{
				Analyzer: analyzer,
				File:     fileRel,
				Signal:   "env-credential-access",
				Severity: SevMedium,
				Detail:   "reads environment variables or credential material",
				Line:     i + 1,
			})
		}
	}

	// Long base64 / hex blob: obfuscated payload carrier.
	for i, l := range lines {
		if base64BlobRe.MatchString(l) || longHexRe.MatchString(l) {
			out = append(out, Finding{
				Analyzer: analyzer,
				File:     fileRel,
				Signal:   "embedded-encoded-blob",
				Severity: SevMedium,
				Detail:   "long base64/hex blob (possible obfuscated payload)",
				Line:     i + 1,
			})
		}
	}

	// Decode-and-rescan: invert hex/base64/gzip(zlib)+base64/split-runs carriers
	// and re-scan the recovered bytes. A behavioral hit there yields a corroborated
	// SevHigh decoded-* finding so an obfuscated payload (the docx-indirection
	// evasion) escalates instead of staying an opaque structural blob.
	if decode {
		out = append(out, decodeAndRescan(text, fileRel, analyzer, 0)...)
	}

	return dedupeFindings(out)
}

// extractHost pulls the host out of the first URL in a line, if any.
func extractHost(line string) string {
	m := hostFromURL.FindStringSubmatch(line)
	if len(m) < 2 {
		return ""
	}
	return strings.ToLower(m[1])
}

// resolveRegistryHost returns the registry/index host for the rewrite directive on
// lines[idx]. If the directive points at a literal URL the host is taken directly;
// if it points at a shell variable (e.g. `pip config set index-url "$MIRROR"`), the
// variable's assignment line elsewhere in the same script is resolved one level
// (e.g. MIRROR="${PIP_MIRROR:-https://pypi.internal.example.com/simple}") so an
// internal/benign mirror is not misclassified as a high-severity exfil rewrite.
// SAFETY: this only feeds the host into isInternalRegistryHost/isKnownBenignHost,
// both of which reject the defanged exfil host first, so a malicious host that is
// reached through a variable still escalates.
func resolveRegistryHost(lines []string, idx int) string {
	if h := extractHost(lines[idx]); h != "" {
		return h
	}
	for _, m := range shellVarRefRe.FindAllStringSubmatch(lines[idx], -1) {
		varName := m[1]
		for _, l := range lines {
			if assignsShellVar(l, varName) {
				if h := extractHost(l); h != "" {
					return h
				}
			}
		}
	}
	return ""
}

// assignsShellVar reports whether line is a shell assignment to name
// (VAR=..., export VAR=...).
func assignsShellVar(line, name string) bool {
	s := strings.TrimSpace(line)
	s = strings.TrimSpace(strings.TrimPrefix(s, "export "))
	return strings.HasPrefix(s, name+"=")
}

// matchedAny reports whether lowered text contains any of the substrings.
func matchedAny(lowerText string, terms []string) bool {
	for _, t := range terms {
		if strings.Contains(lowerText, t) {
			return true
		}
	}
	return false
}

// dedupeFindings collapses exact duplicates (same signal+line+file) keeping the
// highest severity, and returns them in a stable order.
func dedupeFindings(in []Finding) []Finding {
	if len(in) <= 1 {
		return in
	}
	type key struct {
		sig  string
		line int
		file string
	}
	best := map[key]Finding{}
	order := []key{}
	for _, f := range in {
		k := key{f.Signal, f.Line, f.File}
		if prev, ok := best[k]; ok {
			if f.Severity > prev.Severity {
				// keep corroboration if either had it
				f.Corroborated = f.Corroborated || prev.Corroborated
				best[k] = f
			}
			continue
		}
		best[k] = f
		order = append(order, k)
	}
	out := make([]Finding, 0, len(order))
	for _, k := range order {
		out = append(out, best[k])
	}
	sort.SliceStable(out, func(i, j int) bool {
		if out[i].Line != out[j].Line {
			return out[i].Line < out[j].Line
		}
		return out[i].Signal < out[j].Signal
	})
	return out
}

// paddingEvasionFinding builds the standard padding-evasion finding emitted by
// source analyzers when a file was truncated at the read cap with a high
// newline ratio (front-padding to push the payload past the scanner).
func paddingEvasionFinding(f *File, analyzer string) (Finding, bool) {
	if !f.Truncated {
		return Finding{}, false
	}
	if f.NewlineRatio < 0.30 {
		return Finding{}, false
	}
	return Finding{
		Analyzer:     analyzer,
		File:         f.RelPath,
		Signal:       "padding-evasion",
		Severity:     SevHigh,
		Detail:       "file exceeded read cap with a high newline ratio (front/newline padding)",
		Corroborated: true,
	}, true
}