source/pkg/bundle/bundle.go

// Package bundle implements bundle-aware scanning of Claude Code skills.
//
// A skill is a DIRECTORY ("Bundle"): a SKILL.md plus sibling files it
// references (scripts, compiled artifacts, archives, binaries, images, data).
// The malicious payload frequently lives in the sibling files, not in the
// SKILL.md itself. This package walks the WHOLE directory (hidden/dot files
// included), classifies every sibling by SNIFFED MAGIC (not extension alone),
// resolves symlink targets, and resolves which siblings the SKILL.md
// references.
//
// bundle.go holds the pure data model (Bundle/File + FileKind) and pure helper
// functions with NO I/O, so the per-kind Analyzers (in sibling files) and the
// Scan walker can import it cycle-free.
package bundle

import (
	"path/filepath"
	"strings"

	"huggingface.co/turenlabs/Vigil/source/pkg/types"
)

// FileKind classifies a bundle member by what it actually is (sniffed magic
// first, then extension, then by-name special files).
type FileKind int

const (
	// KindSkillMd is the SKILL.md entry point itself.
	KindSkillMd FileKind = iota
	// KindMarkdown is a non-SKILL.md markdown file.
	KindMarkdown
	// KindShell is a .sh/.bash script or a shebang-detected shell script.
	KindShell
	// KindPythonSource is a .py source file.
	KindPythonSource
	// KindScriptOther covers .mjs/.cjs/.ts/.rb/.ps1/.bat/.lua/.pl and by-name
	// Makefile/Dockerfile/justfile: executable text the threat model says
	// scanners ignore.
	KindScriptOther
	// KindPyc is compiled Python bytecode (.pyc).
	KindPyc
	// KindArchive is a zip-family container (.zip/.docx/.xlsx/.pptx or any
	// PK-sniffed file).
	KindArchive
	// KindNativeBinary is an ELF/Mach-O object or .so/.dylib/.node.
	KindNativeBinary
	// KindWasm is a WebAssembly module (\0asm magic).
	KindWasm
	// KindImage is a raster image (.png/.jpg/.jpeg/.gif/.webp).
	KindImage
	// KindData is structured data: json/yaml/csv/.ipynb.
	KindData
	// KindText is plain text not otherwise classified.
	KindText
	// KindUnknown is a member whose kind could not be determined.
	KindUnknown
)

// String returns the stable lowercase token for a FileKind. These tokens are
// part of the manifest schema (FileKind.String() set) consumed by the trainer,
// so they must not change without a corpus migration.
func (k FileKind) String() string {
	switch k {
	case KindSkillMd:
		return "skill_md"
	case KindMarkdown:
		return "markdown"
	case KindShell:
		return "shell"
	case KindPythonSource:
		return "python"
	case KindScriptOther:
		return "script_other"
	case KindPyc:
		return "pyc"
	case KindArchive:
		return "archive"
	case KindNativeBinary:
		return "native_binary"
	case KindWasm:
		return "wasm"
	case KindImage:
		return "image"
	case KindData:
		return "data"
	case KindText:
		return "text"
	default:
		return "unknown"
	}
}

// File is a single member of a Bundle (a sibling of SKILL.md). SKILL.md itself
// is held on Bundle.Skill/SkillMdPath, not in Bundle.Files.
type File struct {
	RelPath        string   // path relative to Bundle.Dir, e.g. ".hidden/sync.sh"
	AbsPath        string   // absolute path on disk
	Kind           FileKind // sniffed-magic-first classification
	SizeBytes      int64    // full on-disk size (not the capped Sniff len)
	Hidden         bool     // any path segment begins with '.'
	Referenced     bool     // SKILL.md names this file
	RefSources     []string // "body"|"frontmatter"|"allowed-tools"|"markdown-link"|"code-span"
	Sniff          []byte   // first <= maxSniffBytes bytes, for analyzers
	Truncated      bool     // file exceeded maxSniffBytes; tail not in Sniff (flag-on-truncate)
	NewlineRatio   float64  // newline bytes / Sniff len, for padding-evasion detection
	IsSymlink      bool     // entry is a symlink
	SymlinkTarget  string   // EvalSymlinks result (resolved), empty if not a symlink
	SymlinkEscapes bool     // resolved target lies outside Bundle.Dir
	ScriptLang     string   // "sh"|"python"|"node"|"ruby"|"powershell"|"make"|... when a script kind
}

// Bundle is a parsed skill directory: the SKILL.md plus every sibling.
type Bundle struct {
	Dir          string
	SkillMdPath  string
	Skill        *types.SkillFile // parsed via parser.Parse (unchanged); nil if ParseErr != nil
	Files        []*File          // every sibling incl. hidden + symlinks, excludes SKILL.md itself
	SkillMdBytes int64
	PayloadBytes int64    // sum of non-SKILL.md file sizes
	AllowedTools []string // parsed from raw frontmatter (parser does not capture this)
	ParseErr     error
	Notes        []string // skipped-loop/unreadable notes; never a panic
}

// isHiddenName reports whether a single path segment is a hidden/dot file.
// The root "." or ".." segments are never themselves payload files.
func isHiddenName(name string) bool {
	return len(name) > 1 && name[0] == '.' && name != ".." && name != "."
}

// pathHasHiddenSegment reports whether any segment of a relative path is hidden.
func pathHasHiddenSegment(relPath string) bool {
	for _, seg := range strings.Split(filepath.ToSlash(relPath), "/") {
		if isHiddenName(seg) {
			return true
		}
	}
	return false
}

// sniffMagicKind classifies a file by leading magic bytes ALONE. Magic is
// authoritative over extension (a notes.txt that is really an ELF must classify
// as KindNativeBinary; an image1.png that is really a script must not classify
// as KindImage). The second return is false when no magic matched, leaving the
// caller to fall back to extension / by-name classification.
//
// Recognized magic:
//   - PK\x03\x04 / PK\x05\x06 / PK\x07\x08  -> zip family (KindArchive)
//   - \x7fELF                                -> ELF (KindNativeBinary)
//   - Mach-O (fat + thin, both endians)      -> Mach-O (KindNativeBinary)
//   - \0asm                                  -> wasm (KindWasm)
//   - PNG / JPEG / GIF / WEBP(RIFF....WEBP)  -> image (KindImage)
//   - CPython .pyc magic (low 2 bytes + \r\n at [2:4]) -> KindPyc
func sniffMagicKind(sniff []byte) (FileKind, bool) {
	if len(sniff) < 4 {
		return KindUnknown, false
	}

	// Zip family (also docx/xlsx/pptx, which are zips).
	if hasPrefix(sniff, []byte("PK\x03\x04")) ||
		hasPrefix(sniff, []byte("PK\x05\x06")) ||
		hasPrefix(sniff, []byte("PK\x07\x08")) {
		return KindArchive, true
	}

	// ELF.
	if hasPrefix(sniff, []byte("\x7fELF")) {
		return KindNativeBinary, true
	}

	// Mach-O: thin (0xFEEDFACE / 0xFEEDFACF, both byte orders) and fat
	// (0xCAFEBABE / 0xBEBAFECA). 0xCAFEBABE also collides with Java .class, but
	// .class is not in this threat model and is still opaque native-ish code, so
	// classifying it as KindNativeBinary is the safe (non-skip) choice.
	switch {
	case hasPrefix(sniff, []byte{0xFE, 0xED, 0xFA, 0xCE}),
		hasPrefix(sniff, []byte{0xFE, 0xED, 0xFA, 0xCF}),
		hasPrefix(sniff, []byte{0xCE, 0xFA, 0xED, 0xFE}),
		hasPrefix(sniff, []byte{0xCF, 0xFA, 0xED, 0xFE}),
		hasPrefix(sniff, []byte{0xCA, 0xFE, 0xBA, 0xBE}),
		hasPrefix(sniff, []byte{0xBE, 0xBA, 0xFE, 0xCA}):
		return KindNativeBinary, true
	}

	// WebAssembly.
	if hasPrefix(sniff, []byte{0x00, 0x61, 0x73, 0x6D}) { // \0asm
		return KindWasm, true
	}

	// Images.
	if hasPrefix(sniff, []byte{0x89, 0x50, 0x4E, 0x47}) { // PNG \x89PNG
		return KindImage, true
	}
	if hasPrefix(sniff, []byte{0xFF, 0xD8, 0xFF}) { // JPEG
		return KindImage, true
	}
	if hasPrefix(sniff, []byte("GIF87a")) || hasPrefix(sniff, []byte("GIF89a")) {
		return KindImage, true
	}
	if len(sniff) >= 12 && hasPrefix(sniff, []byte("RIFF")) &&
		string(sniff[8:12]) == "WEBP" {
		return KindImage, true
	}

	// CPython .pyc: 4-byte magic where bytes [2:4] are \r\n (0x0D 0x0A). The low
	// two bytes are a version-specific number that changes per release, so we key
	// on the stable \r\n pair plus a non-zero first byte (avoids matching a file
	// that merely starts with \0\0\r\n).
	if len(sniff) >= 4 && sniff[2] == 0x0D && sniff[3] == 0x0A &&
		!(sniff[0] == 0x00 && sniff[1] == 0x00) {
		return KindPyc, true
	}

	return KindUnknown, false
}

// hasPrefix reports whether data begins with prefix.
func hasPrefix(data, prefix []byte) bool {
	if len(data) < len(prefix) {
		return false
	}
	for i := range prefix {
		if data[i] != prefix[i] {
			return false
		}
	}
	return true
}

// hasShebang reports whether sniff begins with a "#!" shebang line.
func hasShebang(sniff []byte) bool {
	return len(sniff) >= 2 && sniff[0] == '#' && sniff[1] == '!'
}

// firstLine returns the first line of sniff (without the trailing newline),
// capped at 256 bytes so a pathological no-newline blob cannot be scanned whole.
func firstLine(sniff []byte) string {
	limit := len(sniff)
	if limit > 256 {
		limit = 256
	}
	for i := 0; i < limit; i++ {
		if sniff[i] == '\n' {
			return string(sniff[:i])
		}
	}
	return string(sniff[:limit])
}

// classifyKind classifies a file by SNIFFED MAGIC FIRST, then by extension,
// then by-name special files (Makefile/Dockerfile/justfile). name is the
// basename; sniff is the leading bytes (may be empty for unreadable files).
func classifyKind(name string, sniff []byte) FileKind {
	base := filepath.Base(name)
	lower := strings.ToLower(base)

	// 1. Magic sniff is authoritative.
	if k, ok := sniffMagicKind(sniff); ok {
		// SKILL.md is decided by the walker, never by magic; but a markdown file
		// that happens to begin with bytes resembling magic is vanishingly rare
		// and would already be handled below by extension. Magic wins for the
		// concrete binary/archive/image/pyc kinds returned by sniffMagicKind.
		return k
	}

	// 2. Shebang beats a misleading EXTENSION. A file with no binary/archive/image
	// magic that nonetheless begins with "#!" is an executable script regardless
	// of how its name is dressed (the threat model's "image1.png that is really a
	// script" case). This must run BEFORE extension classification so a .png/.txt
	// disguise cannot demote a real script to image/text. (A genuine PNG carries
	// PNG magic and was already returned in step 1, so this never mis-tags a real
	// image.)
	if hasShebang(sniff) {
		return shebangKind(sniff)
	}

	// 3. By-name special files (no extension, but executable-by-convention).
	switch {
	case lower == "makefile" || lower == "gnumakefile" || strings.HasPrefix(lower, "makefile."):
		return KindScriptOther
	case lower == "dockerfile" || strings.HasPrefix(lower, "dockerfile.") || strings.HasSuffix(lower, ".dockerfile"):
		return KindScriptOther
	case lower == "justfile" || lower == ".justfile":
		return KindScriptOther
	}

	// 4. Extension-based classification.
	ext := strings.ToLower(filepath.Ext(base))
	switch ext {
	case ".md", ".markdown":
		return KindMarkdown
	case ".sh", ".bash", ".zsh", ".ksh":
		return KindShell
	case ".py", ".pyw":
		return KindPythonSource
	case ".pyc", ".pyo":
		return KindPyc
	case ".mjs", ".cjs", ".js", ".ts", ".rb", ".ps1", ".psm1", ".bat", ".cmd", ".lua", ".pl", ".pm":
		return KindScriptOther
	case ".zip", ".docx", ".xlsx", ".pptx", ".jar", ".odt", ".ods", ".odp":
		return KindArchive
	case ".so", ".dylib", ".node", ".o", ".a":
		return KindNativeBinary
	case ".wasm":
		return KindWasm
	case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff":
		return KindImage
	case ".json", ".yaml", ".yml", ".csv", ".tsv", ".ipynb", ".toml":
		return KindData
	case ".txt", ".text", ".rst", ".log", ".cfg", ".ini", ".env":
		return KindText
	}

	// 5. Printable text with no other signal -> KindText; otherwise KindUnknown.
	// (A shebang was already handled in step 2, so any remaining script-shaped
	// file here is plain text.)
	if looksTextual(sniff) {
		return KindText
	}
	return KindUnknown
}

// shebangKind maps a "#!..." interpreter line to a script FileKind.
func shebangKind(sniff []byte) FileKind {
	line := firstLine(sniff)
	switch {
	case strings.Contains(line, "python"):
		return KindPythonSource
	case strings.Contains(line, "node"):
		return KindScriptOther
	case strings.Contains(line, "ruby"):
		return KindScriptOther
	case strings.Contains(line, "perl"):
		return KindScriptOther
	case strings.Contains(line, "sh"): // bash, sh, dash, zsh, ksh
		return KindShell
	default:
		return KindScriptOther
	}
}

// looksTextual reports whether a sniff prefix is mostly printable text. Empty
// input is treated as non-textual (unknown), so an unreadable file does not get
// misclassified as text.
func looksTextual(sniff []byte) bool {
	if len(sniff) == 0 {
		return false
	}
	probe := sniff
	if len(probe) > 512 {
		probe = probe[:512]
	}
	nonPrintable := 0
	for _, b := range probe {
		if b == 0 {
			return false // NUL byte => binary
		}
		if b < 0x09 || (b > 0x0D && b < 0x20) {
			nonPrintable++
		}
	}
	return nonPrintable*100/len(probe) < 10
}

// scriptLangFor returns a coarse language tag for a script-bearing kind, used by
// analyzers to pick the right indicator vocab. Empty string for non-script
// kinds. Resolution order matches classifyKind: by-name, then extension, then
// shebang.
func scriptLangFor(name string, sniff []byte) string {
	base := filepath.Base(name)
	lower := strings.ToLower(base)

	switch {
	case lower == "makefile" || lower == "gnumakefile" || strings.HasPrefix(lower, "makefile."):
		return "make"
	case lower == "dockerfile" || strings.HasPrefix(lower, "dockerfile.") || strings.HasSuffix(lower, ".dockerfile"):
		return "docker"
	case lower == "justfile" || lower == ".justfile":
		return "just"
	}

	switch strings.ToLower(filepath.Ext(base)) {
	case ".sh", ".bash", ".zsh", ".ksh":
		return "sh"
	case ".py", ".pyw":
		return "python"
	case ".pyc", ".pyo":
		return "python"
	case ".mjs", ".cjs", ".js", ".ts":
		return "node"
	case ".rb":
		return "ruby"
	case ".ps1", ".psm1":
		return "powershell"
	case ".bat", ".cmd":
		return "batch"
	case ".lua":
		return "lua"
	case ".pl", ".pm":
		return "perl"
	}

	if hasShebang(sniff) {
		line := strings.ToLower(firstLine(sniff))
		switch {
		case strings.Contains(line, "python"):
			return "python"
		case strings.Contains(line, "node"):
			return "node"
		case strings.Contains(line, "ruby"):
			return "ruby"
		case strings.Contains(line, "perl"):
			return "perl"
		case strings.Contains(line, "sh"):
			return "sh"
		default:
			return "sh"
		}
	}

	return ""
}