// Package compactmodel implements deterministic preprocessing for the compact // whole-package classifier. It deliberately contains no model or filesystem IO. package compactmodel import ( "crypto/sha256" _ "embed" "encoding/hex" "fmt" "unicode" "golang.org/x/text/cases" ) const ( WordFeatures = 1 << 15 CharFeatures = 1 << 15 StructuredFeatures = 16 TotalFeatures = WordFeatures + CharFeatures + StructuredFeatures MaxPackageBytes = 128 * 1024 MaxFileBytes = 48 * 1024 MaxPackageFiles = 96 ContractSchema = "vigil.compact-preprocessing.v1" SerializerName = "package-structure-v1" UnicodeVersion = "15.0.0" ) //go:embed contract.json var contractJSON []byte // ContractJSON returns a copy of the exact preprocessing contract. func ContractJSON() []byte { return append([]byte(nil), contractJSON...) } // ContractSHA256 identifies the exact preprocessing contract. The hash is // computed over the embedded contract.json bytes, including its final newline. func ContractSHA256() string { sum := sha256.Sum256(contractJSON) return hex.EncodeToString(sum[:]) } // ValidateRuntime ensures Unicode lowercasing, token categories and whitespace // categories cannot silently change when a future Go toolchain is used. func ValidateRuntime() error { if unicode.Version != UnicodeVersion || cases.UnicodeVersion != UnicodeVersion { return fmt.Errorf( "compactmodel: unsupported Unicode tables: stdlib=%s x/text=%s, want %s", unicode.Version, cases.UnicodeVersion, UnicodeVersion, ) } return nil } // Metadata is the minimum binding a future model loader must verify before it // can send vectors produced by this package to a scoring model. type Metadata struct { SchemaVersion string ContractSHA256 string WordFeatures int CharFeatures int StructuredFeatures int TotalFeatures int } // ValidateMetadata fails closed on any preprocessing or dimensional mismatch. func ValidateMetadata(metadata Metadata) error { if err := ValidateRuntime(); err != nil { return err } want := Metadata{ SchemaVersion: ContractSchema, ContractSHA256: ContractSHA256(), WordFeatures: WordFeatures, CharFeatures: CharFeatures, StructuredFeatures: StructuredFeatures, TotalFeatures: TotalFeatures, } if metadata != want { return fmt.Errorf("compactmodel: preprocessing metadata mismatch: got %+v, want %+v", metadata, want) } return nil }