source/pkg/bundle/analyzer_data.go

package bundle

import (
	"encoding/json"
	"regexp"
	"strings"
)

// DataAnalyzer inspects data/text siblings (json/yaml/csv/txt/.ipynb). A
// referenced data file IS the context-loader vector: an agent told to "read the
// steps in reference.yaml" will follow imperative directives embedded in it. So
// the CONTENT is scanned for imperative run/exec/fetch directives, exfil hosts,
// and base64/long-hex blobs. .ipynb is parsed as JSON and its code-cell source
// is scanned with the script vocabulary. A standalone present-but-unreferenced
// hit is SevMedium; IndirectionAnalyzer raises delegate-to-data to SevHigh when
// SKILL.md delegates to the file.
type DataAnalyzer struct{}

func (DataAnalyzer) Name() string { return "data" }

func (DataAnalyzer) Handles(kind FileKind) bool {
	return kind == KindData || kind == KindText
}

// imperativeDirectiveRe matches instructions telling the agent to run/execute/
// source/fetch something — the heart of the context-loader bypass.
var imperativeDirectiveRe = regexp.MustCompile(`(?i)\b(run|execute|exec|source|eval|invoke|fetch|download|curl|wget|pipe to (?:sh|bash)|chmod \+x|install (?:and run)?|launch|spawn)\b`)

func (DataAnalyzer) Analyze(f *File, b *Bundle) ([]Finding, error) {
	if f == nil {
		return nil, nil
	}
	text := string(f.Sniff)
	var out []Finding

	// Exfil host anywhere in a data file is critical.
	if exfilHostRe.MatchString(text) {
		line := firstLineMatching(text, exfilHostRe)
		out = append(out, Finding{
			Analyzer:     "data",
			File:         f.RelPath,
			Signal:       "exfil-host-reference",
			Severity:     SevCritical,
			Detail:       "data file references known exfiltration host",
			Line:         line,
			Corroborated: true,
		})
	}

	// .ipynb: parse code cells and scan their source with the script vocab.
	if strings.HasSuffix(strings.ToLower(f.RelPath), ".ipynb") {
		cells := scanNotebookCells(f.Sniff)
		if len(cells) == 0 && looksLikeJSON(text) {
			out = append(out, Finding{
				Analyzer: "data",
				File:     f.RelPath,
				Signal:   "opaque-notebook",
				Severity: SevMedium,
				Detail:   "could not parse notebook cells (malformed JSON)",
				Opaque:   true,
			})
		}
		for _, src := range cells {
			cellFindings := sharedIndicatorScan(src, f.RelPath, "data")
			out = append(out, cellFindings...)
		}
	}

	// Imperative directives embedded in data values.
	for _, dir := range scanImperativeDirectives(text) {
		out = append(out, Finding{
			Analyzer: "data",
			File:     f.RelPath,
			Signal:   "data-embedded-directive",
			Severity: SevMedium,
			Detail:   "data file embeds an imperative agent directive: " + dir,
		})
	}

	// base64/hex payload carriers inside data values.
	lines := strings.Split(text, "\n")
	for i, l := range lines {
		if base64BlobRe.MatchString(l) || longHexRe.MatchString(l) {
			out = append(out, Finding{
				Analyzer: "data",
				File:     f.RelPath,
				Signal:   "embedded-encoded-blob",
				Severity: SevMedium,
				Detail:   "long base64/hex blob inside data file",
				Line:     i + 1,
			})
		}
	}

	// Decode-and-rescan the data body (covers json/yaml/csv/txt carriers and the
	// split-runs obfuscation). A behavioral hit in the recovered bytes adds a
	// corroborated SevHigh decoded-* finding so an obfuscated directive escalates.
	out = append(out, decodeAndRescan(text, f.RelPath, "data", 0)...)

	if pe, ok := paddingEvasionFinding(f, "data"); ok {
		out = append(out, pe)
	}
	return dedupeFindings(out), nil
}

// scanImperativeDirectives returns deduped snippets of imperative directives
// found in the text (capped to keep output bounded).
func scanImperativeDirectives(text string) []string {
	seen := map[string]bool{}
	var out []string
	for _, l := range strings.Split(text, "\n") {
		if !imperativeDirectiveRe.MatchString(l) {
			continue
		}
		// Only flag lines that also carry a URL, a path, or a shell-ish token —
		// a prose word "run the report" alone is not a directive.
		low := strings.ToLower(l)
		if !urlRe.MatchString(l) &&
			!strings.Contains(low, "./") &&
			!strings.Contains(low, ".sh") &&
			!strings.Contains(low, ".py") &&
			!matchedAny(low, networkSinkTerms) &&
			!matchedAny(low, rceTerms) {
			continue
		}
		snippet := strings.TrimSpace(l)
		if len(snippet) > 120 {
			snippet = snippet[:120]
		}
		if !seen[snippet] {
			seen[snippet] = true
			out = append(out, snippet)
		}
		if len(out) >= 16 {
			break
		}
	}
	return out
}

// scanNotebookCells parses a Jupyter notebook and returns the concatenated
// source of each code cell. Degrades to empty on malformed JSON (never panics).
func scanNotebookCells(jsonBytes []byte) []string {
	var nb struct {
		Cells []struct {
			CellType string          `json:"cell_type"`
			Source   json.RawMessage `json:"source"`
		} `json:"cells"`
	}
	if err := json.Unmarshal(jsonBytes, &nb); err != nil {
		return nil
	}
	var out []string
	for _, c := range nb.Cells {
		if c.CellType != "code" {
			continue
		}
		out = append(out, joinSource(c.Source))
	}
	return out
}

// joinSource handles the notebook "source" field which may be a string or an
// array of strings.
func joinSource(raw json.RawMessage) string {
	if len(raw) == 0 {
		return ""
	}
	var asString string
	if err := json.Unmarshal(raw, &asString); err == nil {
		return asString
	}
	var asArray []string
	if err := json.Unmarshal(raw, &asArray); err == nil {
		return strings.Join(asArray, "")
	}
	return ""
}

func looksLikeJSON(text string) bool {
	t := strings.TrimSpace(text)
	return strings.HasPrefix(t, "{") || strings.HasPrefix(t, "[")
}

func firstLineMatching(text string, re *regexp.Regexp) int {
	for i, l := range strings.Split(text, "\n") {
		if re.MatchString(l) {
			return i + 1
		}
	}
	return 0
}