source/pkg/compactmodel/contract.go

// Package compactmodel implements deterministic preprocessing for the compact
// whole-package classifier. It deliberately contains no model or filesystem IO.
package compactmodel

import (
	"crypto/sha256"
	_ "embed"
	"encoding/hex"
	"fmt"
	"unicode"

	"golang.org/x/text/cases"
)

const (
	WordFeatures       = 1 << 15
	CharFeatures       = 1 << 15
	StructuredFeatures = 16
	TotalFeatures      = WordFeatures + CharFeatures + StructuredFeatures

	MaxPackageBytes = 128 * 1024
	MaxFileBytes    = 48 * 1024
	MaxPackageFiles = 96

	ContractSchema = "vigil.compact-preprocessing.v1"
	SerializerName = "package-structure-v1"
	UnicodeVersion = "15.0.0"
)

//go:embed contract.json
var contractJSON []byte

// ContractJSON returns a copy of the exact preprocessing contract.
func ContractJSON() []byte {
	return append([]byte(nil), contractJSON...)
}

// ContractSHA256 identifies the exact preprocessing contract. The hash is
// computed over the embedded contract.json bytes, including its final newline.
func ContractSHA256() string {
	sum := sha256.Sum256(contractJSON)
	return hex.EncodeToString(sum[:])
}

// ValidateRuntime ensures Unicode lowercasing, token categories and whitespace
// categories cannot silently change when a future Go toolchain is used.
func ValidateRuntime() error {
	if unicode.Version != UnicodeVersion || cases.UnicodeVersion != UnicodeVersion {
		return fmt.Errorf(
			"compactmodel: unsupported Unicode tables: stdlib=%s x/text=%s, want %s",
			unicode.Version, cases.UnicodeVersion, UnicodeVersion,
		)
	}
	return nil
}

// Metadata is the minimum binding a future model loader must verify before it
// can send vectors produced by this package to a scoring model.
type Metadata struct {
	SchemaVersion      string
	ContractSHA256     string
	WordFeatures       int
	CharFeatures       int
	StructuredFeatures int
	TotalFeatures      int
}

// ValidateMetadata fails closed on any preprocessing or dimensional mismatch.
func ValidateMetadata(metadata Metadata) error {
	if err := ValidateRuntime(); err != nil {
		return err
	}
	want := Metadata{
		SchemaVersion:      ContractSchema,
		ContractSHA256:     ContractSHA256(),
		WordFeatures:       WordFeatures,
		CharFeatures:       CharFeatures,
		StructuredFeatures: StructuredFeatures,
		TotalFeatures:      TotalFeatures,
	}
	if metadata != want {
		return fmt.Errorf("compactmodel: preprocessing metadata mismatch: got %+v, want %+v", metadata, want)
	}
	return nil
}