File size: 8,138 Bytes
506f40d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import Foundation

public struct ModelConfiguration: Codable, Sendable, Equatable {
    public let patchSize: Int
    public let dModel: Int
    public let numHeads: Int
    public let mlpHiddenDim: Int
    public let blockSchedule: [String]
    public let quantileKnots: [Double]
    public let blockSize: Int
    public let contextLength: Int
    public let ropeTheta: Double
    public let ropePartialFactor: Double
    public let normEps: Double
    public let attnBias: Bool
    public let mlpBias: Bool
    public let timeAttnGate: String
    public let variateAttnGate: String
    public let minimumScale: Float
    public let useAsinh: Bool
    public let stabilizeWithGlobal: Bool
    public let covariateEnabled: Bool
    public let splitContextHorizonAttn: Bool
    public let attendMissingPatches: Bool
    public let quantileWidthFactor: Double

    public var headDim: Int { dModel / numHeads }
    public var ropeDim: Int { Int(Double(headDim) * ropePartialFactor) }
    public var numQuantiles: Int { quantileKnots.count }
    public var medianIndex: Int { quantileKnots.firstIndex(of: 0.5)! }
    public var supportsKnownCovariates: Bool {
        guard covariateEnabled else { return false }
        let axes = blockSchedule.flatMap { kind -> [Bool] in
            switch kind {
            case "time": return [true]
            case "variate": return [false]
            case "time_variate": return [true, false]
            default: return [false, true]
            }
        }
        for (index, time) in axes.enumerated() where time {
            let rest = axes.dropFirst(index + 1)
            if rest.contains(true) && rest.contains(false) { return true }
        }
        return false
    }

    public init(patchSize: Int, dModel: Int, numHeads: Int, mlpHiddenDim: Int,
                blockSchedule: [String], quantileKnots: [Double], blockSize: Int, contextLength: Int,
                ropeTheta: Double = 10000, ropePartialFactor: Double = 0.5, normEps: Double = 1e-6,
                attnBias: Bool = false, mlpBias: Bool = false, timeAttnGate: String = "sdpa_sigmoid",
                variateAttnGate: String = "sdpa_sigmoid", minimumScale: Float = 1e-6,
                useAsinh: Bool = true, stabilizeWithGlobal: Bool = false, covariateEnabled: Bool = true,
                splitContextHorizonAttn: Bool = true, attendMissingPatches: Bool = true,
                quantileWidthFactor: Double = 2) throws {
        self.patchSize = patchSize; self.dModel = dModel; self.numHeads = numHeads; self.mlpHiddenDim = mlpHiddenDim
        self.blockSchedule = blockSchedule; self.quantileKnots = quantileKnots; self.blockSize = blockSize; self.contextLength = contextLength
        self.ropeTheta = ropeTheta; self.ropePartialFactor = ropePartialFactor; self.normEps = normEps
        self.attnBias = attnBias; self.mlpBias = mlpBias; self.timeAttnGate = timeAttnGate; self.variateAttnGate = variateAttnGate
        self.minimumScale = minimumScale; self.useAsinh = useAsinh; self.stabilizeWithGlobal = stabilizeWithGlobal
        self.covariateEnabled = covariateEnabled; self.splitContextHorizonAttn = splitContextHorizonAttn
        self.attendMissingPatches = attendMissingPatches; self.quantileWidthFactor = quantileWidthFactor
        guard patchSize > 0, dModel > 0, numHeads > 0, mlpHiddenDim > 0, blockSize > 0, contextLength > 0,
              dModel % numHeads == 0, ropePartialFactor.isFinite, ropePartialFactor > 0, ropePartialFactor <= 1,
              ropeTheta.isFinite, ropeTheta > 0, normEps.isFinite, normEps > 0,
              minimumScale.isFinite, minimumScale > 0, quantileWidthFactor.isFinite, quantileWidthFactor > 0,
              !blockSchedule.isEmpty, ropeDim > 0, ropeDim % 2 == 0 else {
            throw ForecastError.invalidConfiguration("Invalid Timestral model dimensions or numerical settings.")
        }
        guard blockSchedule.allSatisfy({ ["time", "variate", "time_variate", "variate_time"].contains($0) }),
              [timeAttnGate, variateAttnGate].allSatisfy({ ["none", "sdpa_sigmoid"].contains($0) }),
              quantileKnots.contains(0.5), quantileKnots.allSatisfy({ $0.isFinite }),
              zip(quantileKnots, quantileKnots.dropFirst()).allSatisfy({ $0 < $1 }), !stabilizeWithGlobal else {
            throw ForecastError.invalidConfiguration("Unsupported Timestral schedule, gates, quantiles, or normalization frame.")
        }
    }

    public init(data: Data) throws { self = try JSONDecoder().decode(Self.self, from: data) }
    public static func load(from url: URL) throws -> Self { try Self(data: Data(contentsOf: url)) }

    enum CodingKeys: String, CodingKey, CaseIterable {
        case patchSize = "patch_size", dModel = "d_model", numHeads = "num_heads", mlpHiddenDim = "mlp_hidden_dim"
        case blockSchedule = "block_schedule", quantileKnots = "quantile_knots", blockSize = "block_size", contextLength = "context_length"
        case ropeTheta = "rope_theta", ropePartialFactor = "rope_partial_factor", normEps = "norm_eps"
        case attnBias = "attn_bias", mlpBias = "mlp_bias", timeAttnGate = "time_attn_gate", variateAttnGate = "variate_attn_gate"
        case minimumScale = "minimum_scale", useAsinh = "use_asinh", stabilizeWithGlobal = "stabilize_with_global"
        case covariateEnabled = "covariate_enabled", splitContextHorizonAttn = "split_context_horizon_attn"
        case attendMissingPatches = "attend_missing_patches", quantileWidthFactor = "quantile_width_factor"
    }
    struct AnyKey: CodingKey {
        let stringValue: String
        let intValue: Int? = nil
        init?(stringValue: String) { self.stringValue = stringValue }
        init?(intValue: Int) { return nil }
    }
    public init(from decoder: any Decoder) throws {
        let keys = try decoder.container(keyedBy: AnyKey.self).allKeys.map(\.stringValue)
        let allowed = Set(CodingKeys.allCases.map(\.rawValue))
        guard Set(keys).isSubset(of: allowed) else { throw ForecastError.invalidConfiguration("Unknown model configuration keys: \(Set(keys).subtracting(allowed).sorted())") }
        let c = try decoder.container(keyedBy: CodingKeys.self)
        try self.init(patchSize: c.decode(Int.self, forKey: .patchSize), dModel: c.decode(Int.self, forKey: .dModel),
                      numHeads: c.decode(Int.self, forKey: .numHeads), mlpHiddenDim: c.decode(Int.self, forKey: .mlpHiddenDim),
                      blockSchedule: c.decode([String].self, forKey: .blockSchedule), quantileKnots: c.decode([Double].self, forKey: .quantileKnots),
                      blockSize: c.decode(Int.self, forKey: .blockSize), contextLength: c.decode(Int.self, forKey: .contextLength),
                      ropeTheta: c.decodeIfPresent(Double.self, forKey: .ropeTheta) ?? 10000,
                      ropePartialFactor: c.decodeIfPresent(Double.self, forKey: .ropePartialFactor) ?? 0.5,
                      normEps: c.decodeIfPresent(Double.self, forKey: .normEps) ?? 1e-6,
                      attnBias: c.decodeIfPresent(Bool.self, forKey: .attnBias) ?? false,
                      mlpBias: c.decodeIfPresent(Bool.self, forKey: .mlpBias) ?? false,
                      timeAttnGate: c.decodeIfPresent(String.self, forKey: .timeAttnGate) ?? "sdpa_sigmoid",
                      variateAttnGate: c.decodeIfPresent(String.self, forKey: .variateAttnGate) ?? "sdpa_sigmoid",
                      minimumScale: c.decodeIfPresent(Float.self, forKey: .minimumScale) ?? 1e-6,
                      useAsinh: c.decodeIfPresent(Bool.self, forKey: .useAsinh) ?? true,
                      stabilizeWithGlobal: c.decodeIfPresent(Bool.self, forKey: .stabilizeWithGlobal) ?? false,
                      covariateEnabled: c.decodeIfPresent(Bool.self, forKey: .covariateEnabled) ?? true,
                      splitContextHorizonAttn: c.decodeIfPresent(Bool.self, forKey: .splitContextHorizonAttn) ?? true,
                      attendMissingPatches: c.decodeIfPresent(Bool.self, forKey: .attendMissingPatches) ?? true,
                      quantileWidthFactor: c.decodeIfPresent(Double.self, forKey: .quantileWidthFactor) ?? 2)
    }
}