giancap61 commited on
Commit
09c621d
·
verified ·
1 Parent(s): e9ba013

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/__init__.cpython-312.pyc +0 -0
  2. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_compatibility.cpython-312.pyc +0 -0
  3. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_lazy_graph_module.cpython-312.pyc +0 -0
  4. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_pytree.cpython-312.pyc +0 -0
  5. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_symbolic_trace.cpython-312.pyc +0 -0
  6. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_utils.cpython-312.pyc +0 -0
  7. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/__init__.cpython-312.pyc +0 -0
  8. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_backward_state.cpython-312.pyc +0 -0
  9. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_config.cpython-312.pyc +0 -0
  10. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_constant_symnode.cpython-312.pyc +0 -0
  11. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_dynamism.cpython-312.pyc +0 -0
  12. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/accelerator_partitioner.cpython-312.pyc +0 -0
  13. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/const_fold.cpython-312.pyc +0 -0
  14. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/debug.cpython-312.pyc +0 -0
  15. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/graph_gradual_typechecker.cpython-312.pyc +0 -0
  16. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/merge_matmul.cpython-312.pyc +0 -0
  17. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/meta_tracer.cpython-312.pyc +0 -0
  18. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/normalize.cpython-312.pyc +0 -0
  19. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/optimization.cpython-312.pyc +0 -0
  20. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/partitioner_utils.cpython-312.pyc +0 -0
  21. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/recording.cpython-312.pyc +0 -0
  22. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/refinement_types.cpython-312.pyc +0 -0
  23. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/rewriter.cpython-312.pyc +0 -0
  24. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/schema_type_annotation.cpython-312.pyc +0 -0
  25. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/sym_node.cpython-312.pyc +0 -0
  26. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/unify_refinements.cpython-312.pyc +0 -0
  27. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/validator.cpython-312.pyc +0 -0
  28. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__init__.py +0 -0
  29. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/__init__.cpython-312.pyc +0 -0
  30. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/constraint.cpython-312.pyc +0 -0
  31. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/constraint_generator.cpython-312.pyc +0 -0
  32. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/constraint_transformation.cpython-312.pyc +0 -0
  33. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/operation.cpython-312.pyc +0 -0
  34. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/transform_to_z3.cpython-312.pyc +0 -0
  35. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/util.cpython-312.pyc +0 -0
  36. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/z3_types.cpython-312.pyc +0 -0
  37. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/constraint.py +637 -0
  38. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/constraint_generator.py +1565 -0
  39. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/constraint_transformation.py +1322 -0
  40. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/operation.py +14 -0
  41. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/transform_to_z3.py +446 -0
  42. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/util.py +58 -0
  43. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/z3_types.py +30 -0
  44. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__init__.py +4 -0
  45. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/__init__.cpython-312.pyc +0 -0
  46. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/core.cpython-312.pyc +0 -0
  47. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/dispatch.cpython-312.pyc +0 -0
  48. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/match.cpython-312.pyc +0 -0
  49. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/more.cpython-312.pyc +0 -0
  50. TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/unification_tools.cpython-312.pyc +0 -0
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (4.37 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_compatibility.cpython-312.pyc ADDED
Binary file (1.89 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_lazy_graph_module.cpython-312.pyc ADDED
Binary file (8.26 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_pytree.cpython-312.pyc ADDED
Binary file (4.93 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_symbolic_trace.cpython-312.pyc ADDED
Binary file (56.9 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__pycache__/_utils.cpython-312.pyc ADDED
Binary file (2.94 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (210 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_backward_state.cpython-312.pyc ADDED
Binary file (1.44 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_config.cpython-312.pyc ADDED
Binary file (2.36 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_constant_symnode.cpython-312.pyc ADDED
Binary file (4.51 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/_dynamism.cpython-312.pyc ADDED
Binary file (6.16 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/accelerator_partitioner.cpython-312.pyc ADDED
Binary file (44.2 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/const_fold.cpython-312.pyc ADDED
Binary file (13.9 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/debug.cpython-312.pyc ADDED
Binary file (1.7 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/graph_gradual_typechecker.cpython-312.pyc ADDED
Binary file (46.9 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/merge_matmul.cpython-312.pyc ADDED
Binary file (6.36 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/meta_tracer.cpython-312.pyc ADDED
Binary file (15.7 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/normalize.cpython-312.pyc ADDED
Binary file (7.51 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/optimization.cpython-312.pyc ADDED
Binary file (24.7 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/partitioner_utils.cpython-312.pyc ADDED
Binary file (12.6 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/recording.cpython-312.pyc ADDED
Binary file (16.2 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/refinement_types.cpython-312.pyc ADDED
Binary file (1.38 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/rewriter.cpython-312.pyc ADDED
Binary file (7.81 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/schema_type_annotation.cpython-312.pyc ADDED
Binary file (6.55 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/sym_node.cpython-312.pyc ADDED
Binary file (67.5 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/unify_refinements.cpython-312.pyc ADDED
Binary file (4.43 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/__pycache__/validator.cpython-312.pyc ADDED
Binary file (41.4 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__init__.py ADDED
File without changes
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (232 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/constraint.cpython-312.pyc ADDED
Binary file (27.2 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/constraint_generator.cpython-312.pyc ADDED
Binary file (62.7 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/constraint_transformation.cpython-312.pyc ADDED
Binary file (45.7 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/operation.cpython-312.pyc ADDED
Binary file (543 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/transform_to_z3.cpython-312.pyc ADDED
Binary file (15.6 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/util.cpython-312.pyc ADDED
Binary file (2.4 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/__pycache__/z3_types.cpython-312.pyc ADDED
Binary file (1.44 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/constraint.py ADDED
@@ -0,0 +1,637 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from torch.fx.experimental.migrate_gradual_types.operation import (
3
+ op_add,
4
+ op_div,
5
+ op_eq,
6
+ op_gt,
7
+ op_lt,
8
+ op_mod,
9
+ op_mul,
10
+ op_neq,
11
+ op_sub,
12
+ )
13
+ from torch.fx.tensor_type import Dyn, TensorType
14
+
15
+
16
+ class Constraint:
17
+ pass
18
+
19
+
20
+ class Conj(Constraint):
21
+ def __init__(self, conjuncts):
22
+ """
23
+ :param conjuncts: Conjunction of constraints
24
+ """
25
+ self.conjucts = conjuncts
26
+
27
+ def __eq__(self, other):
28
+ if isinstance(other, Conj):
29
+ return self.conjucts == other.conjucts and self.conjucts == other.conjucts
30
+ else:
31
+ return False
32
+
33
+ def __repr__(self):
34
+ return f"And({self.conjucts})"
35
+
36
+
37
+ class Disj(Constraint):
38
+ def __init__(self, disjuncts):
39
+ """
40
+ :param disjuncts: Disjunction of constraints
41
+ """
42
+ self.disjuncts = disjuncts
43
+
44
+ def __eq__(self, other):
45
+ if isinstance(other, Disj):
46
+ return (
47
+ self.disjuncts == other.disjuncts and self.disjuncts == other.disjuncts
48
+ )
49
+ else:
50
+ return False
51
+
52
+ def __repr__(self):
53
+ return f"Or({self.disjuncts})"
54
+
55
+
56
+ class Prod(Constraint):
57
+ def __init__(self, products):
58
+ """
59
+ :param products: lists of dimensions to multiply
60
+ """
61
+ self.products = products
62
+
63
+ def __eq__(self, other):
64
+ if isinstance(other, Prod):
65
+ return self.products == other.products and self.products == other.products
66
+ else:
67
+ return False
68
+
69
+ def __repr__(self):
70
+ return f"Product({self.products})"
71
+
72
+
73
+ class T(Constraint):
74
+ """
75
+ True
76
+ """
77
+
78
+ def __init__(self) -> None:
79
+ pass
80
+
81
+ def __eq__(self, other):
82
+ return isinstance(other, T)
83
+
84
+ def __repr__(self):
85
+ return "True"
86
+
87
+
88
+ class F(Constraint):
89
+ """
90
+ False
91
+ """
92
+
93
+ def __init__(self) -> None:
94
+ pass
95
+
96
+ def __eq__(self, other):
97
+ return isinstance(other, F)
98
+
99
+ def __repr__(self):
100
+ return "False"
101
+
102
+
103
+ class BinaryConstraint(Constraint):
104
+ """
105
+ Represents all binary operations
106
+ """
107
+
108
+ def __init__(self, lhs, rhs, op):
109
+ """
110
+ :param lhs: lhs of the constraint
111
+ :param rhs: rhs of the constraint
112
+ :param op: string representing the operation
113
+ """
114
+ self.lhs = lhs
115
+ self.rhs = rhs
116
+ self.op = op
117
+
118
+ def __eq__(self, other):
119
+ if isinstance(other, BinaryConstraint):
120
+ return (
121
+ self.lhs == other.lhs and self.rhs == other.rhs and self.op == other.op
122
+ )
123
+ else:
124
+ return False
125
+
126
+ def __repr__(self):
127
+ return f"({self.lhs} {self.op} {self.rhs})"
128
+
129
+
130
+ class BinConstraintT(BinaryConstraint):
131
+ """
132
+ Binary constraints about tensors
133
+ """
134
+
135
+ def __init__(self, lhs, rhs, op):
136
+ assert (isinstance(lhs, (TVar, TensorType, int)) or lhs == Dyn) and (
137
+ isinstance(rhs, (TVar, TensorType, int)) or rhs == Dyn
138
+ )
139
+ super().__init__(lhs, rhs, op)
140
+
141
+
142
+ class BinConstraintD(BinaryConstraint):
143
+ """
144
+ Binary constraints about dimensions
145
+ """
146
+
147
+ def __init__(self, lhs, rhs, op):
148
+ assert is_algebraic_expression(lhs) or is_dim(lhs) or is_bool_expr(lhs)
149
+ assert is_algebraic_expression(rhs) or is_dim(rhs) or is_bool_expr(rhs)
150
+
151
+ super().__init__(lhs, rhs, op)
152
+
153
+
154
+ class TGreatestUpperBound(Constraint):
155
+ """
156
+ Greatest Upper bound for tensors with dynamic type
157
+ """
158
+
159
+ def __init__(self, res, rhs1, rhs2):
160
+ """
161
+ :param res: tensor variable that stores the result of the output
162
+ :param rhs1: tensor or tensor variable
163
+ :param rhs2: tensor or tensor variabke
164
+ """
165
+ self.res = res
166
+ self.rhs1 = rhs1
167
+ self.rhs2 = rhs2
168
+
169
+ def __repr__(self):
170
+ return f"{self.res} = {self.rhs1}\u2294*{self.rhs2}"
171
+
172
+ def __eq__(self, other):
173
+ if isinstance(other, TGreatestUpperBound):
174
+ return (
175
+ self.res == other.res
176
+ and self.rhs1 == other.rhs1
177
+ and self.rhs2 == other.rhs2
178
+ )
179
+ else:
180
+ return False
181
+
182
+
183
+ class DGreatestUpperBound(Constraint):
184
+ """
185
+ Greatest Upper bound for dimensions
186
+ """
187
+
188
+ def __init__(self, res, rhs1, rhs2):
189
+ """
190
+ :param res: Dimension variable to store the result
191
+ :param rhs1: dimension variable 1
192
+ :param rhs2: dimension variable 2
193
+ """
194
+ assert is_dim(res)
195
+ assert is_dim(rhs1)
196
+ assert is_dim(rhs2)
197
+
198
+ self.res = res
199
+ self.rhs1 = rhs1
200
+ self.rhs2 = rhs2
201
+
202
+ def __repr__(self):
203
+ return f"{self.res} = {self.rhs1}\u2294{self.rhs2}"
204
+
205
+ def __eq__(self, other):
206
+ if isinstance(other, DGreatestUpperBound):
207
+ return (
208
+ self.res == other.res
209
+ and self.rhs1 == other.rhs1
210
+ and self.rhs2 == other.rhs2
211
+ )
212
+ else:
213
+ return False
214
+
215
+
216
+ class CanReshape(Constraint):
217
+ """
218
+ can_reshape constraint
219
+ """
220
+
221
+ def __init__(self, src, target):
222
+ """
223
+ :param src: tensor variable
224
+ :param target: tensor
225
+ """
226
+ self.src = src
227
+ self.target = target
228
+
229
+ def __repr__(self):
230
+ return f"can-reshape({self.src}, {self.target})"
231
+
232
+ def __eq__(self, other):
233
+ if isinstance(other, CanReshape):
234
+ return self.src == other.src and self.target == other.target
235
+ else:
236
+ return False
237
+
238
+
239
+ class IndexSelect(Constraint):
240
+ def __init__(self, tensor_size, input_var, dim_replace, index, output):
241
+ """
242
+ Args:
243
+ input_var: input to index_select
244
+ tensor_size: tensor size we are considering
245
+ dim_replace: the dimension of the output at "index"
246
+ index: location of the dimensions to replace in the input
247
+ output: variable to store the result
248
+ """
249
+ assert isinstance(input_var, TVar)
250
+ assert isinstance(output, TVar)
251
+ assert isinstance(dim_replace, DVar) or dim_replace == Dyn
252
+ assert isinstance(index, int)
253
+
254
+ self.input_var = input_var
255
+ self.tensor_size = tensor_size
256
+ self.dim_replace = dim_replace
257
+ self.index = index
258
+ self.output = output
259
+
260
+ def __repr__(self):
261
+ return (
262
+ f" {self.output} = "
263
+ f"IndexSelect({self.input_var}, "
264
+ f"tensor_size: {self.tensor_size}, "
265
+ f"{self.dim_replace}, "
266
+ f"{self.index})"
267
+ )
268
+
269
+ def __eq__(self, other):
270
+ if isinstance(other, IndexSelect):
271
+ return (
272
+ self.tensor_size == other.tensor_size
273
+ and self.dim_replace == other.dim_replace
274
+ and self.index == other.index
275
+ and self.output == other.output
276
+ and self.input_var == other.input_var
277
+ )
278
+ else:
279
+ return False
280
+
281
+
282
+ class Transpose(Constraint):
283
+ def __init__(self, tensor_size, input_var, index1, index2, output):
284
+ """
285
+ Args:
286
+ tensor_size: current tensor size
287
+ input_var: variable to hold input
288
+ index1: dimension 1
289
+ index2: dimension 2
290
+ output: output that stores result
291
+ """
292
+ assert isinstance(input_var, TVar)
293
+ assert isinstance(output, TVar)
294
+ assert isinstance(index1, int)
295
+ assert isinstance(index2, int)
296
+
297
+ self.input_var = input_var
298
+ self.tensor_size = tensor_size
299
+ self.index1 = index1
300
+ self.index2 = index2
301
+ self.output = output
302
+
303
+ def __repr__(self):
304
+ return (
305
+ f" {self.output} = "
306
+ f"Transpose({self.input_var}, "
307
+ f"tensor_size: {self.tensor_size}, "
308
+ f"{self.index1}, "
309
+ f"{self.index2})"
310
+ )
311
+
312
+ def __eq__(self, other):
313
+ if isinstance(other, Transpose):
314
+ return (
315
+ self.tensor_size == other.tensor_size
316
+ and self.index1 == other.index1
317
+ and self.index2 == other.index2
318
+ and self.output == other.output
319
+ and self.input_var == other.input_var
320
+ )
321
+ else:
322
+ return False
323
+
324
+
325
+ class GetItem(Constraint):
326
+ def __init__(self, tensor_size, index, res, input_var):
327
+ """
328
+ Constraint for getting item given a tensor size
329
+ :param tensor_size: actual number
330
+ :param index: actual number representing the index
331
+ :param res: dimension variable to carry the item we get
332
+ :param input_var: a tensor variable from which we will get item
333
+ """
334
+ assert isinstance(res, DVar)
335
+
336
+ self.res = res
337
+ self.tensor_size = tensor_size
338
+ self.index = index
339
+ self.input_var = input_var
340
+
341
+ def __repr__(self):
342
+ return f" {self.res} = GetItem({self.input_var}, tensor_size: {self.tensor_size}, {self.index})"
343
+
344
+ def __eq__(self, other):
345
+ if isinstance(other, GetItem):
346
+ return (
347
+ self.res == other.res
348
+ and self.tensor_size == other.tensor_size
349
+ and self.index == other.index
350
+ and self.input_var == other.input_var
351
+ )
352
+ else:
353
+ return False
354
+
355
+
356
+ class GetItemTensor(Constraint):
357
+ def __init__(self, tensor_size, index_tuple, res, input_var):
358
+ """
359
+ Constraint for getting item given a tensor size
360
+ However, when the argument is a tuple, we will
361
+ expect a tensor
362
+ :param tensor_size: actual number representing the rank
363
+ :param index_tuple: tuple for indexing
364
+ :param res: tensor variable to carry the item we get
365
+ :param input_var: a tensor variable from which we will get item
366
+ """
367
+ assert isinstance(res, TVar)
368
+
369
+ self.res = res
370
+ self.tensor_size = tensor_size
371
+ self.index_tuple = index_tuple
372
+ self.input_var = input_var
373
+
374
+ def __repr__(self):
375
+ return f" {self.res} = GetItemT({self.input_var}, tensor_size: {self.tensor_size}, {self.index_tuple})"
376
+
377
+ def __eq__(self, other):
378
+ if isinstance(other, GetItemTensor):
379
+ return (
380
+ self.res == other.res
381
+ and self.tensor_size == other.tensor_size
382
+ and self.index_tuple == other.index_tuple
383
+ and self.input_var == other.input_var
384
+ )
385
+ else:
386
+ return False
387
+
388
+
389
+ class CalcConv(Constraint):
390
+ def __init__(
391
+ self,
392
+ conv_result,
393
+ input_var,
394
+ c_out,
395
+ kernel,
396
+ padding,
397
+ stride,
398
+ dilation,
399
+ matching_constraint_vars,
400
+ ):
401
+ """
402
+ :param conv_result: the convolution result
403
+ :param input_var: input to convolution
404
+ :param c_out: output channel type
405
+ :param kernel: kernel tuple
406
+ """
407
+ self.conv_result = conv_result
408
+ self.input_var = input_var
409
+ self.c_out = c_out
410
+ self.kernel = kernel
411
+ self.padding = padding
412
+ self.stride = stride
413
+ self.dilation = dilation
414
+ self.matching_constraint = matching_constraint_vars
415
+
416
+ def __repr__(self):
417
+ return (
418
+ f"{self.conv_result} ="
419
+ f" calc-conv({self.input_var},"
420
+ f" {self.c_out}, {self.kernel}, "
421
+ f"{self.padding}, {self.stride},"
422
+ f" {self.dilation})"
423
+ )
424
+
425
+ def __eq__(self, other):
426
+ if isinstance(other, CalcConv):
427
+ return (
428
+ self.conv_result == other.conv_result
429
+ and self.input_var == other.input_var
430
+ and self.c_out == other.c_out
431
+ and self.kernel == other.kernel
432
+ and self.padding == other.padding
433
+ and self.stride == other.stride
434
+ and self.dilation == other.dilation
435
+ and self.matching_constraint == other.matching_constraint
436
+ )
437
+ else:
438
+ return False
439
+
440
+
441
+ class CalcMaxPool(Constraint):
442
+ def __init__(
443
+ self,
444
+ maxpool_result,
445
+ input_var,
446
+ kernel,
447
+ padding,
448
+ stride,
449
+ dilation,
450
+ matching_constraint_vars,
451
+ ):
452
+ """
453
+ :param maxpool_result: the result of maxpool
454
+ :param input_var: input to convolution
455
+ :param kernel: kernel tuple
456
+ """
457
+ self.maxpool_result = maxpool_result
458
+ self.input_var = input_var
459
+ self.kernel = kernel
460
+ self.padding = padding
461
+ self.stride = stride
462
+ self.dilation = dilation
463
+ self.matching_constraint = matching_constraint_vars
464
+
465
+ def __repr__(self):
466
+ return (
467
+ f"{self.maxpool_result} ="
468
+ f" calc-maxpool({self.input_var},"
469
+ f" {self.kernel}, "
470
+ f"{self.padding}, {self.stride},"
471
+ f" {self.dilation})"
472
+ )
473
+
474
+ def __eq__(self, other):
475
+ if isinstance(other, CalcMaxPool):
476
+ return (
477
+ self.maxpool_result == other.maxpool_result
478
+ and self.input_var == other.input_var
479
+ and self.kernel == other.kernel
480
+ and self.padding == other.padding
481
+ and self.stride == other.stride
482
+ and self.dilation == other.dilation
483
+ and self.matching_constraint == other.matching_constraint
484
+ )
485
+ else:
486
+ return False
487
+
488
+
489
+ class ApplyBroadcasting(Constraint):
490
+ def __init__(self, res1, res2, input1, input2):
491
+ """
492
+ :param res1: resulting tensor 1
493
+ :param res2: resulting tensor 2
494
+ :param input1: tensor variable 1
495
+ :param input2: tensor variable 2
496
+ """
497
+ self.res1 = res1
498
+ self.res2 = res2
499
+ self.input1 = input1
500
+ self.input2 = input2
501
+
502
+ def __eq__(self, other):
503
+ if isinstance(other, ApplyBroadcasting):
504
+ return (
505
+ self.res1 == other.res1
506
+ and self.res2 == other.res2
507
+ and self.input1 == other.input1
508
+ and self.input2 == other.input2
509
+ )
510
+ else:
511
+ return False
512
+
513
+ def __repr__(self):
514
+ return (
515
+ f"{self.res1}, {self.res2} ="
516
+ f" apply-broadcasting({self.input1},"
517
+ f" {self.input2})"
518
+ )
519
+
520
+
521
+ class CalcProduct(Constraint):
522
+ """
523
+ Given correct dimensions, calculate the product for flatten accounting for Dyn
524
+ """
525
+
526
+ def __init__(self, start, end, flattened, dims_to_flatten):
527
+ """
528
+ :param start: start index
529
+ :param end: end index
530
+ :param flattened: variable to store the product
531
+ :param dims_to_flatten: the type which we will flatten
532
+ """
533
+ assert isinstance(dims_to_flatten, list)
534
+ assert isinstance(flattened, TVar)
535
+ assert isinstance(start, int)
536
+ assert isinstance(end, int)
537
+
538
+ self.start = start
539
+ self.end = end
540
+ self.dims_to_flatten = dims_to_flatten
541
+ self.flattened = flattened
542
+
543
+ def __eq__(self, other):
544
+ if isinstance(other, CalcProduct):
545
+ return (
546
+ self.start == other.start
547
+ and self.end == other.end
548
+ and self.dims_to_flatten == other.dims_to_flatten
549
+ and self.flattened == other.flattened
550
+ )
551
+
552
+ else:
553
+ return False
554
+
555
+ def __repr__(self):
556
+ return f"{self.flattened} = CalcProduct({self.start}, {self.end}, {self.dims_to_flatten})"
557
+
558
+
559
+ class TVar:
560
+ """
561
+ Tensor variable with no tensor constructor
562
+ """
563
+
564
+ def __init__(self, tvar):
565
+ """
566
+ :param tvar: tensor variable
567
+ """
568
+ self.tvar = tvar
569
+
570
+ def __repr__(self):
571
+ return f"TV({self.tvar})"
572
+
573
+ def __eq__(self, other):
574
+ if isinstance(other, TVar):
575
+ return self.tvar == other.tvar
576
+ else:
577
+ return False
578
+
579
+
580
+ class DVar:
581
+ """
582
+ Dimension variable
583
+ """
584
+
585
+ def __init__(self, c):
586
+ """
587
+ :param c: character or number
588
+ """
589
+ self.c = c
590
+
591
+ def __repr__(self):
592
+ return f"DV({self.c})"
593
+
594
+ def __eq__(self, other):
595
+ if isinstance(other, DVar):
596
+ return self.c == other.c
597
+ else:
598
+ return False
599
+
600
+
601
+ class BVar:
602
+ """
603
+ Boolean variable
604
+ """
605
+
606
+ def __init__(self, c):
607
+ """
608
+ :param c: character or number
609
+ """
610
+ self.c = c
611
+
612
+ def __repr__(self):
613
+ return f"BV({self.c})"
614
+
615
+ def __eq__(self, other):
616
+ if isinstance(other, BVar):
617
+ return self.c == other.c
618
+ else:
619
+ return False
620
+
621
+
622
+ def is_algebraic_expression(constraint):
623
+ if isinstance(constraint, BinConstraintD):
624
+ return constraint.op in [op_add, op_sub, op_div, op_mul, op_mod]
625
+ else:
626
+ return isinstance(constraint, Prod)
627
+
628
+
629
+ def is_bool_expr(constraint):
630
+ if isinstance(constraint, BinConstraintD):
631
+ return constraint.op in [op_gt, op_lt, op_neq, op_eq]
632
+ else:
633
+ return isinstance(constraint, (BVar, Conj, Disj))
634
+
635
+
636
+ def is_dim(d):
637
+ return isinstance(d, (DVar, int)) or d == Dyn
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/constraint_generator.py ADDED
@@ -0,0 +1,1565 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ import operator
3
+ import warnings
4
+ from collections.abc import Callable, Iterable
5
+ from typing import TypeVar
6
+ from typing_extensions import ParamSpec
7
+
8
+ import torch
9
+ from torch.fx._symbolic_trace import _assert_is_none
10
+ from torch.fx.experimental.migrate_gradual_types.constraint import (
11
+ ApplyBroadcasting,
12
+ BinConstraintD,
13
+ BinConstraintT,
14
+ CalcConv,
15
+ CalcMaxPool,
16
+ CalcProduct,
17
+ CanReshape,
18
+ Conj,
19
+ DGreatestUpperBound,
20
+ Disj,
21
+ DVar,
22
+ F,
23
+ GetItem,
24
+ GetItemTensor,
25
+ IndexSelect,
26
+ T,
27
+ TGreatestUpperBound,
28
+ Transpose,
29
+ TVar,
30
+ )
31
+ from torch.fx.experimental.migrate_gradual_types.operation import (
32
+ op_add,
33
+ op_consistency,
34
+ op_div,
35
+ op_eq,
36
+ op_gt,
37
+ op_leq,
38
+ op_lt,
39
+ op_matching,
40
+ op_mul,
41
+ op_neq,
42
+ op_precision,
43
+ op_sub,
44
+ )
45
+ from torch.fx.experimental.migrate_gradual_types.util import (
46
+ gen_bvar,
47
+ gen_dvar,
48
+ gen_nat_constraints,
49
+ gen_tensor_dims,
50
+ gen_tvar,
51
+ )
52
+ from torch.fx.node import Node, Target
53
+ from torch.fx.tensor_type import Dyn, TensorType
54
+ from torch.nn.modules.batchnorm import BatchNorm2d
55
+ from torch.nn.modules.conv import Conv2d
56
+
57
+
58
+ _T = TypeVar("_T")
59
+ _P = ParamSpec("_P")
60
+
61
+ _INFERENCE_RULES: dict[Target, Callable] = {}
62
+
63
+ MAX_TENSOR_RANK = 4
64
+
65
+ __all__ = [
66
+ "ConstraintGenerator",
67
+ "adaptive_inference_rule",
68
+ "add_layer_norm_constraints",
69
+ "add_linear_constraints",
70
+ "arange_inference_rule",
71
+ "assert_inference_rule",
72
+ "batchnorm_inference_rule",
73
+ "bmm_inference_rule",
74
+ "broadcasting_inference_rule",
75
+ "conv2d_inference_rule",
76
+ "cumsum_inference_rule",
77
+ "embedding_inference_rule",
78
+ "embedding_inference_rule_functional",
79
+ "eq_inference_rule",
80
+ "equality_inference_rule",
81
+ "expand_inference_rule",
82
+ "flatten_inference_rule",
83
+ "full_inference_rule",
84
+ "gen_broadcasting_constraints",
85
+ "gen_embedding_rules",
86
+ "gen_layer_norm_constraints",
87
+ "generate_flatten_constraints",
88
+ "get_attr_inference_rule",
89
+ "getitem_inference_rule",
90
+ "gt_inference_rule",
91
+ "index_select_inference_rule",
92
+ "layer_norm_functional",
93
+ "layer_norm_inference_rule",
94
+ "linear_constraints",
95
+ "linear_inference_rule",
96
+ "lt_inference_rule",
97
+ "masked_fill_inference_rule",
98
+ "maxpool_inference_rule",
99
+ "neq_inference_rule",
100
+ "range_check",
101
+ "register_inference_rule",
102
+ "relu_inference_rule",
103
+ "reshape_inference_rule",
104
+ "size_inference_rule",
105
+ "tensor_inference_rule",
106
+ "torch_dim_inference_rule",
107
+ "torch_linear_inference_rule",
108
+ "transpose_inference_rule",
109
+ "type_inference_rule",
110
+ "view_inference_rule",
111
+ ]
112
+
113
+
114
+ def register_inference_rule(
115
+ call_target: Target,
116
+ ) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
117
+ def register(fn: Callable[_P, _T]) -> Callable[_P, _T]:
118
+ if call_target in _INFERENCE_RULES:
119
+ raise RuntimeError(f"Inference rule already registered for {call_target}!")
120
+ _INFERENCE_RULES[call_target] = fn
121
+ return fn
122
+
123
+ return register
124
+
125
+
126
+ def generate_flatten_constraints(start_dim, end_dim, input, flattened, n, counter):
127
+ d, counter = gen_tensor_dims(n, counter)
128
+ c1 = BinConstraintT(input, TensorType(d), op_eq)
129
+ start_dim = n if start_dim == -1 else abs(start_dim)
130
+ end_dim = n + end_dim + 1 if end_dim < 0 else end_dim + 1
131
+ c2 = CalcProduct(start_dim, end_dim, flattened, d)
132
+ nat_constraints = gen_nat_constraints(d)
133
+ return Conj([c1, c2, *nat_constraints]), counter
134
+
135
+
136
+ @register_inference_rule(getattr)
137
+ def get_attr_inference_rule(n: Node, symbols, constraints, counter):
138
+ """
139
+ If the attribute is "device" then the tensor shape is preserved
140
+ """
141
+ assert isinstance(n.args[0], Node)
142
+ assert isinstance(n.args[1], str)
143
+ output, counter = gen_tvar(counter)
144
+ symbols[n] = output
145
+
146
+ input = symbols[n.args[0]]
147
+ attr = n.args[1]
148
+
149
+ if attr == "device":
150
+ return [BinConstraintT(input, output, op_eq)], counter
151
+ else:
152
+ raise NotImplementedError("Not yet implemented")
153
+
154
+
155
+ @register_inference_rule(torch.bmm)
156
+ def bmm_inference_rule(n: Node, symbols, constraints, counter):
157
+ """
158
+ Constraints that match the input to a size 3 tensor
159
+ and switch the dimensions according to the rules
160
+ of batch multiplication
161
+ """
162
+ assert isinstance(n.args[0], Node)
163
+ assert isinstance(n.args[1], Node)
164
+
165
+ bmm_output, counter = gen_tvar(counter)
166
+ symbols[n] = bmm_output
167
+
168
+ bmm_input1 = symbols[n.args[0]]
169
+ bmm_input2 = symbols[n.args[1]]
170
+
171
+ dims_input1, counter = gen_tensor_dims(3, counter)
172
+ dims_input2, counter = gen_tensor_dims(3, counter)
173
+
174
+ inputs_dyn = Conj(
175
+ [
176
+ BinConstraintT(bmm_input1, Dyn, op_eq),
177
+ BinConstraintT(bmm_input2, Dyn, op_eq),
178
+ BinConstraintT(bmm_output, Dyn, op_eq),
179
+ ]
180
+ )
181
+
182
+ input1_dyn = Conj(
183
+ [
184
+ BinConstraintT(bmm_input1, Dyn, op_eq),
185
+ BinConstraintT(bmm_input2, TensorType(dims_input2), op_eq),
186
+ BinConstraintT(
187
+ bmm_output, TensorType([dims_input2[0], Dyn, dims_input2[2]]), op_eq
188
+ ),
189
+ ]
190
+ )
191
+
192
+ input2_dyn = Conj(
193
+ [
194
+ BinConstraintT(bmm_input2, Dyn, op_eq),
195
+ BinConstraintT(bmm_input1, TensorType(dims_input1), op_eq),
196
+ BinConstraintT(
197
+ bmm_output, TensorType([dims_input1[0], dims_input1[1], Dyn]), op_eq
198
+ ),
199
+ ]
200
+ )
201
+
202
+ consistency_constraints = [
203
+ BinConstraintD(dims_input1[0], dims_input2[0], op_consistency)
204
+ ]
205
+
206
+ batch_size, counter = gen_dvar(counter)
207
+
208
+ inputs_are_tensors = Conj(
209
+ [
210
+ BinConstraintT(bmm_input1, TensorType(dims_input1), op_eq),
211
+ BinConstraintT(bmm_input2, TensorType(dims_input2), op_eq),
212
+ BinConstraintT(
213
+ bmm_output,
214
+ TensorType([batch_size, dims_input1[1], dims_input2[2]]),
215
+ op_eq,
216
+ ),
217
+ *consistency_constraints,
218
+ DGreatestUpperBound(batch_size, dims_input1[0], dims_input2[0]),
219
+ ]
220
+ )
221
+
222
+ return [Disj([inputs_dyn, input1_dyn, input2_dyn, inputs_are_tensors])], counter
223
+
224
+
225
+ @register_inference_rule("index_select")
226
+ def index_select_inference_rule(n: Node, symbols, constraints, counter):
227
+ """
228
+ We constrain the second argument to a vector or Dyn.
229
+ The output replaces the input with the shape of the vector
230
+ at the position given by the index (first argument)
231
+ """
232
+ # print(n.args)
233
+ assert isinstance(n.args[0], Node)
234
+ assert isinstance(n.args[1], int)
235
+ assert isinstance(n.args[2], Node)
236
+
237
+ index_select, counter = gen_tvar(counter)
238
+ symbols[n] = index_select
239
+
240
+ dims, counter = gen_tensor_dims(1, counter)
241
+
242
+ # equality constraint
243
+ is_size_1 = BinConstraintT(symbols[n.args[2]], TensorType(dims), op_eq)
244
+ is_dyn = BinConstraintT(symbols[n.args[2]], Dyn, op_eq)
245
+
246
+ c2 = Conj(
247
+ [
248
+ is_size_1,
249
+ Disj(
250
+ [
251
+ IndexSelect(
252
+ i + 1, symbols[n.args[0]], dims[0], n.args[1], index_select
253
+ )
254
+ for i in range(MAX_TENSOR_RANK)
255
+ ]
256
+ ),
257
+ ]
258
+ )
259
+ c3 = Conj(
260
+ [
261
+ is_dyn,
262
+ Disj(
263
+ [
264
+ IndexSelect(i + 1, symbols[n.args[0]], Dyn, n.args[1], index_select)
265
+ for i in range(MAX_TENSOR_RANK)
266
+ ]
267
+ ),
268
+ ]
269
+ )
270
+
271
+ return [Disj([c2, c3])], counter
272
+
273
+
274
+ @register_inference_rule("expand")
275
+ def expand_inference_rule(n: Node, symbols, constraints, counter):
276
+ """
277
+ We generate the exact constraints as we do for tensor additions but we constraint
278
+ the rank of this expression to be equal to len(n.args[1:]) so that only
279
+ those cases get considered for the output
280
+ """
281
+ assert isinstance(n.args[0], Node)
282
+
283
+ # define the output for expand
284
+ expand, counter = gen_tvar(counter)
285
+ symbols[n] = expand
286
+
287
+ # since we do not have two nodes here, we will construct an argument variable
288
+ e1 = symbols[n.args[0]]
289
+ e2, counter = gen_tvar(counter)
290
+
291
+ e2_nat_constraints = []
292
+ for arg in n.args[1:]:
293
+ assert isinstance(arg, (Node, int))
294
+ if isinstance(arg, Node):
295
+ assert isinstance(symbols[arg], DVar)
296
+ e2_nat_constraints.append(BinConstraintD(0, symbols[arg], op_leq))
297
+
298
+ e2_constraint = BinConstraintT(
299
+ e2,
300
+ TensorType(
301
+ [arg if isinstance(arg, int) else symbols[arg] for arg in n.args[1:]]
302
+ ),
303
+ op_eq,
304
+ )
305
+
306
+ constraints, counter = gen_broadcasting_constraints(
307
+ e1, e2, symbols, counter, expand
308
+ )
309
+
310
+ # constraint the output size
311
+ dims, counter = gen_tensor_dims(len(n.args[1:]), counter)
312
+ nat_constraints = gen_nat_constraints(dims)
313
+ c = [
314
+ BinConstraintT(expand, TensorType(dims), op_eq),
315
+ *nat_constraints,
316
+ e2_constraint,
317
+ *e2_nat_constraints,
318
+ ]
319
+ constraints += c
320
+
321
+ return constraints, counter
322
+
323
+
324
+ @register_inference_rule(torch.nn.functional.gelu)
325
+ @register_inference_rule(torch.nn.functional.dropout)
326
+ @register_inference_rule(torch.nn.functional.softmax)
327
+ @register_inference_rule("detach")
328
+ @register_inference_rule("to")
329
+ @register_inference_rule("int")
330
+ @register_inference_rule("long")
331
+ @register_inference_rule("contiguous")
332
+ @register_inference_rule(torch.ones)
333
+ @register_inference_rule(torch.zeros)
334
+ def equality_inference_rule(n: Node, symbols, constraints, counter):
335
+ """
336
+ We generate the constraint: input = output
337
+ """
338
+ output, counter = gen_tvar(counter)
339
+ symbols[n] = output
340
+
341
+ if isinstance(n.args[0], Node):
342
+ input = symbols[n.args[0]]
343
+ if isinstance(input, TVar):
344
+ return [BinConstraintT(input, output, op_eq)], counter
345
+
346
+ # then we have dimension variables
347
+ else:
348
+ for arg in n.args:
349
+ assert isinstance(symbols[arg], DVar)
350
+ my_size = [symbols[arg] for arg in n.args]
351
+ return [BinConstraintT(output, TensorType(my_size), op_eq)], counter
352
+
353
+ elif isinstance(n.args[0], tuple):
354
+ # then the tuple is the size
355
+ assert len(n.args[0]) <= 4
356
+ my_size = [symbols[arg] for arg in n.args[0]]
357
+ return [BinConstraintT(output, TensorType(my_size), op_eq)], counter
358
+ else:
359
+ raise NotImplementedError("Method not yet implemented")
360
+
361
+
362
+ @register_inference_rule("transpose")
363
+ def transpose_inference_rule(n: Node, symbols, constraints, counter):
364
+ """
365
+ Can be considered as a sequence of two index selects, so we generate constraints accordingly
366
+ """
367
+ assert isinstance(n.args[0], Node)
368
+ assert isinstance(n.args[1], int)
369
+ assert isinstance(n.args[2], int)
370
+
371
+ output, counter = gen_tvar(counter)
372
+ symbols[n] = output
373
+
374
+ from_arg = symbols[n.args[0]]
375
+ assert isinstance(from_arg, TVar)
376
+
377
+ # input and output are dyn
378
+ is_dyn = Conj(
379
+ [BinConstraintT(from_arg, Dyn, op_eq), BinConstraintT(output, Dyn, op_eq)]
380
+ )
381
+
382
+ # or input is a tensor and we actually do the replacement
383
+ c3 = Disj(
384
+ [
385
+ Transpose(i + 1, from_arg, n.args[1], n.args[2], output)
386
+ for i in range(MAX_TENSOR_RANK)
387
+ ]
388
+ )
389
+
390
+ return [Disj([is_dyn, c3])], counter
391
+
392
+
393
+ @register_inference_rule("type_as")
394
+ def type_inference_rule(n: Node, symbols, constraints, counter):
395
+ """
396
+ We generate the constraint: input = output
397
+ """
398
+ assert isinstance(n.args[0], Node)
399
+ assert isinstance(n.args[1], Node)
400
+
401
+ output, counter = gen_tvar(counter)
402
+ symbols[n] = output
403
+
404
+ from_arg = symbols[n.args[0]]
405
+ to_arg = symbols[n.args[1]]
406
+
407
+ assert isinstance(from_arg, TVar)
408
+ assert isinstance(to_arg, TVar)
409
+
410
+ return [
411
+ BinConstraintT(from_arg, to_arg, op_consistency),
412
+ BinConstraintT(output, to_arg, op_eq),
413
+ ], counter
414
+
415
+
416
+ @register_inference_rule("masked_fill_")
417
+ def masked_fill_inference_rule(n: Node, symbols, constraints, counter):
418
+ """
419
+ Similar to addition. For now we implement the constraints when
420
+ the argument is a boolean tensor. There is also a case for when
421
+ it is a condition. We will leave this out for now.
422
+ """
423
+
424
+ assert isinstance(n.args[0], Node)
425
+ assert isinstance(n.args[1], Node)
426
+
427
+ # We will retrieve the type variables from the symbol table
428
+ # and confirm they are tensor variables
429
+
430
+ e1 = symbols[n.args[0]]
431
+ e2 = symbols[n.args[1]]
432
+
433
+ if isinstance(e1, TVar) and isinstance(e2, TVar):
434
+ masked_fill_tensor, counter = gen_tvar(counter)
435
+ symbols[n] = masked_fill_tensor
436
+ return gen_broadcasting_constraints(
437
+ e1, e2, symbols, counter, masked_fill_tensor
438
+ )
439
+ else:
440
+ raise NotImplementedError("Not yet implemented")
441
+
442
+
443
+ @register_inference_rule(torch.nn.functional.embedding)
444
+ def embedding_inference_rule_functional(n: Node, symbols, constraints, counter):
445
+ assert isinstance(n.args[0], Node)
446
+
447
+ embedding_dim_weights = symbols[n.args[1]]
448
+
449
+ # will treat this as a static shape. So we will not use matching.
450
+ weight_dims, counter = gen_tensor_dims(2, counter)
451
+ equality_constraint = BinConstraintT(
452
+ embedding_dim_weights, TensorType(weight_dims), op_eq
453
+ )
454
+ embedding_dim = weight_dims[1]
455
+ constraints, counter = gen_embedding_rules(n, symbols, embedding_dim, counter)
456
+ return [equality_constraint] + constraints, counter
457
+
458
+
459
+ @register_inference_rule(torch.nn.modules.sparse.Embedding)
460
+ def embedding_inference_rule(n: Node, module_instance, symbols, constraints, counter):
461
+ """
462
+ The output shape differs from the input shape in the last dimension
463
+ """
464
+ assert isinstance(n.args[0], Node)
465
+ return gen_embedding_rules(n, symbols, module_instance.embedding_dim, counter)
466
+
467
+
468
+ def gen_embedding_rules(n: Node, symbols, embedding_dim, counter):
469
+ embedding_output, counter = gen_tvar(counter)
470
+ symbols[n] = embedding_output
471
+ embedding_input = symbols[n.args[0]]
472
+
473
+ input_dyn = BinConstraintT(embedding_input, Dyn, op_eq)
474
+ output_dyn = BinConstraintT(embedding_output, Dyn, op_eq)
475
+
476
+ c1 = Conj([input_dyn, output_dyn])
477
+ c2 = []
478
+
479
+ for i in range(1, MAX_TENSOR_RANK):
480
+ new_dims, counter = gen_tensor_dims(i, counter)
481
+ nat_constraints = gen_nat_constraints(new_dims)
482
+
483
+ # we consider all tensor sizes and append embedding_dim to the end of the output dimension in all cases
484
+ c_tensor_i = Conj(
485
+ [
486
+ BinConstraintT(embedding_input, TensorType(new_dims), op_eq),
487
+ BinConstraintT(
488
+ embedding_output, TensorType(new_dims + [embedding_dim]), op_eq
489
+ ),
490
+ ]
491
+ + nat_constraints
492
+ )
493
+ c2.append(c_tensor_i)
494
+
495
+ return [Disj([c1, Disj(c2)])], counter
496
+
497
+
498
+ @register_inference_rule(torch.tensor)
499
+ def tensor_inference_rule(n: Node, symbols, constraints, counter):
500
+ """
501
+ If the tensor is a scalar, we will skip it since we
502
+ do not support scalars yet. We will add support in the future
503
+ if it's needed. For our examples so far, scalars are not needed.
504
+ """
505
+ return [], counter
506
+
507
+
508
+ @register_inference_rule("reshape")
509
+ @register_inference_rule("view")
510
+ def view_inference_rule(n: Node, symbols, constraints, counter):
511
+ """
512
+ Similar to reshape but with an extra condition on the strides
513
+ """
514
+ assert isinstance(n.args[0], Node)
515
+
516
+ # generate the new variable
517
+ my_view, counter = gen_tvar(counter)
518
+ symbols[n] = my_view
519
+
520
+ src_var = symbols[n.args[0]]
521
+ t2 = [
522
+ symbols[elem] if isinstance(elem, Node) else elem for elem in n.args[1:]
523
+ ] # target shape
524
+ t2_type = []
525
+ num_constraints = []
526
+
527
+ for t in t2:
528
+ if t == -1:
529
+ var, counter = gen_dvar(counter)
530
+ t2_type.append(var)
531
+ # pyrefly: ignore [bad-argument-type]
532
+ num_constraints.append(BinConstraintD(var, Dyn, op_neq))
533
+
534
+ else:
535
+ # pyrefly: ignore [bad-argument-type]
536
+ num_constraints.append(BinConstraintD(t, Dyn, op_neq))
537
+ t2_type.append(t) # type: ignore[arg-type]
538
+
539
+ t2_type = TensorType(t2_type) # type: ignore[assignment]
540
+
541
+ c1 = BinConstraintT(my_view, t2_type, op_eq)
542
+ c2 = CanReshape(src_var, t2_type)
543
+
544
+ # TODO: add the extra check mentioned here:
545
+ # https://pytorch.org/docs/stable/generated/torch.Tensor.view.html#torch.Tensor.view
546
+
547
+ return [c1, c2] + num_constraints, counter # type: ignore[operator]
548
+
549
+
550
+ @register_inference_rule("size")
551
+ def size_inference_rule(n: Node, symbols, constraints, counter):
552
+ """
553
+ The constraint is just lhs = rhs.
554
+ Ex: size = input_ids.size()
555
+ """
556
+
557
+ if len(n.args) == 1:
558
+ # generate the new variable
559
+ size, counter = gen_tvar(counter)
560
+ symbols[n] = size
561
+ input = symbols[n.args[0]]
562
+ c = BinConstraintT(input, size, op_eq)
563
+ return [c], counter
564
+
565
+ elif len(n.args) == 2:
566
+ # TODO: review this rule; should input = dyn; output = dyn be included here?
567
+ if isinstance(n.args[1], int):
568
+ # generate the new variable
569
+ size_index, counter = gen_dvar(counter)
570
+ symbols[n] = size_index
571
+ input = symbols[n.args[0]]
572
+ c2 = [
573
+ GetItem(i + 1, n.args[1], size_index, input)
574
+ for i in range(MAX_TENSOR_RANK)
575
+ ]
576
+ c3 = BinConstraintD(0, size_index, op_leq)
577
+
578
+ input_dyn = BinConstraintT(input, Dyn, op_eq)
579
+ output_dyn = BinConstraintD(size_index, Dyn, op_eq)
580
+ c1 = Conj([input_dyn, output_dyn])
581
+
582
+ return [Disj([c1, Conj([Disj(c2), c3])])], counter
583
+
584
+ else:
585
+ raise NotImplementedError
586
+
587
+ else:
588
+ raise NotImplementedError
589
+
590
+
591
+ def range_check(i, n):
592
+ """
593
+ Checks if an index i is within range of a size n list
594
+ Args:
595
+ i: index
596
+ n: list size
597
+
598
+ Returns: Boolean
599
+ """
600
+ if i >= 0:
601
+ return T() if i < n else F()
602
+ else:
603
+ return T() if i >= n else F()
604
+
605
+
606
+ @register_inference_rule(torch.cumsum)
607
+ def cumsum_inference_rule(n: Node, symbols, constraints, counter):
608
+ """
609
+ Input and output shapes should be equal
610
+ We should verify that the index is valid
611
+ """
612
+ assert isinstance(n.args[0], Node)
613
+ arg_1 = n.args[1] if len(n.args) > 1 else n.kwargs["dim"]
614
+ assert isinstance(arg_1, int)
615
+
616
+ output, counter = gen_tvar(counter)
617
+ symbols[n] = output
618
+ input = symbols[n.args[0]]
619
+
620
+ input_dyn = BinConstraintT(input, Dyn, op_eq)
621
+ output_dyn = BinConstraintT(output, Dyn, op_eq)
622
+ c1 = Conj([input_dyn, output_dyn])
623
+ c2 = []
624
+ for i in range(1, MAX_TENSOR_RANK + 1):
625
+ new_dims, counter = gen_tensor_dims(i, counter)
626
+
627
+ nat_constraints = gen_nat_constraints(new_dims)
628
+
629
+ c_tensor_i = Conj(
630
+ [
631
+ BinConstraintT(input, TensorType(new_dims), op_eq),
632
+ BinConstraintT(output, TensorType(new_dims), op_eq),
633
+ ]
634
+ + [range_check(arg_1, i)]
635
+ + nat_constraints
636
+ )
637
+
638
+ c2.append(c_tensor_i)
639
+ dyn_or_tensor = Disj([c1, Disj(c2)])
640
+ return [dyn_or_tensor], counter
641
+
642
+
643
+ @register_inference_rule(_assert_is_none)
644
+ def assert_inference_rule(n: Node, symbols, constraints, counter):
645
+ assert len(n.users) == 0
646
+ return [], counter
647
+
648
+
649
+ @register_inference_rule(operator.getitem)
650
+ def getitem_inference_rule(n: Node, symbols, constraints, counter):
651
+ assert isinstance(n.args[0], Node)
652
+
653
+ # dimension output case
654
+ if isinstance(n.args[1], int):
655
+ # create and store the new dimension variable
656
+ get_item_output, counter = gen_dvar(counter)
657
+ symbols[n] = get_item_output
658
+
659
+ # retrieve arg variables
660
+ get_item_arg = symbols[n.args[0]]
661
+ assert isinstance(get_item_arg, TVar)
662
+
663
+ # if the input is dynamic, we accept any index and return
664
+ # a dynamic dimension as output
665
+ input_dyn = BinConstraintT(get_item_arg, Dyn, op_eq)
666
+ output_dyn = BinConstraintD(get_item_output, Dyn, op_eq)
667
+ c1 = Conj([input_dyn, output_dyn])
668
+
669
+ # if the input is a tensor,
670
+ # generate a getItem constraint which will be expanded based on the
671
+ # tensor dimension.
672
+
673
+ c2 = [
674
+ GetItem(i + 1, n.args[1], get_item_output, get_item_arg)
675
+ for i in range(MAX_TENSOR_RANK)
676
+ ]
677
+
678
+ # since the output is a dimension, we make sure it's a natural number
679
+ # added as a conjunction to the disjunction of c2
680
+ c3 = BinConstraintD(0, get_item_output, op_leq)
681
+ return [Disj([c1, Conj([Disj(c2), c3])])], counter
682
+
683
+ # tensor output case
684
+ elif isinstance(n.args[1], tuple):
685
+ # create and store the new tensor variable
686
+ get_item_output, counter = gen_tvar(counter) # type: ignore[arg-type,assignment]
687
+ symbols[n] = get_item_output
688
+
689
+ # retrieve arg variables
690
+ if n.args[0] in symbols:
691
+ get_item_arg = symbols[n.args[0]]
692
+ assert isinstance(get_item_arg, TVar)
693
+
694
+ input_dyn = BinConstraintT(get_item_arg, Dyn, op_eq)
695
+ output_dyn = BinConstraintT(get_item_output, Dyn, op_eq) # type: ignore[assignment]
696
+ c1 = Conj([input_dyn, output_dyn])
697
+
698
+ c2 = [
699
+ GetItemTensor(i + 1, n.args[1], get_item_output, get_item_arg) # type: ignore[misc]
700
+ for i in range(MAX_TENSOR_RANK)
701
+ ]
702
+ else:
703
+ # TODO: we should figure out why there is a key-error here.
704
+ return [], counter
705
+
706
+ return [Disj([c1, *c2])], counter
707
+
708
+ else:
709
+ raise RuntimeError("Method not yet implemented")
710
+
711
+
712
+ @register_inference_rule(operator.gt)
713
+ def gt_inference_rule(n: Node, symbols, constraints, counter):
714
+ assert isinstance(n.args[0], (Node, int))
715
+ assert isinstance(n.args[1], (Node, int))
716
+
717
+ # We make sure this node will not be used again. We do not
718
+ # generate a constraint about that node. Only about the operands.
719
+
720
+ e1 = symbols[n.args[0]] if isinstance(n.args[0], Node) else n.args[0]
721
+ e2 = symbols[n.args[1]] if isinstance(n.args[1], Node) else n.args[1]
722
+
723
+ if isinstance(n.args[0], Node) and isinstance(n.args[1], Node):
724
+ if isinstance(e1, TVar) and isinstance(e2, TVar):
725
+ gt_tensor, counter = gen_tvar(counter)
726
+ symbols[n] = gt_tensor
727
+ return gen_broadcasting_constraints(e1, e2, symbols, counter, gt_tensor)
728
+
729
+ elif isinstance(e1, DVar) and isinstance(e2, DVar):
730
+ # This is meant to be used for flow analysis only
731
+ gt_constraint = BinConstraintD(e1, e2, op_gt)
732
+
733
+ my_gt, counter = gen_bvar(counter)
734
+ equality_constraint = BinConstraintD(my_gt, gt_constraint, op_eq)
735
+ return [equality_constraint], counter
736
+
737
+ else:
738
+ raise RuntimeError("Sort Mismatch")
739
+
740
+ elif isinstance(n.args[0], Node) and not isinstance(n.args[1], Node):
741
+ if isinstance(e1, DVar):
742
+ # This is meant to be used for flow analysis only
743
+ gt_constraint = BinConstraintD(e1, e2, op_gt)
744
+
745
+ my_gt, counter = gen_bvar(counter)
746
+ equality_constraint = BinConstraintD(my_gt, gt_constraint, op_eq)
747
+ return [equality_constraint], counter
748
+
749
+ elif isinstance(e1, TVar) and isinstance(e2, int):
750
+ # then we made the wrong assumption about the argument being a tensor
751
+ # so we should fix the assumption
752
+ warnings.warn(
753
+ f"Made the wrong assumption for node {n}. Correctness not guaranteed."
754
+ )
755
+
756
+ new_e1, counter = gen_dvar(counter)
757
+ symbols[n.args[0]] = new_e1
758
+ symbols[n.args[0]]
759
+
760
+ gt_constraint = BinConstraintD(new_e1, e2, op_gt)
761
+
762
+ my_gt, counter = gen_bvar(counter)
763
+ equality_constraint = BinConstraintD(my_gt, gt_constraint, op_eq)
764
+ return [equality_constraint], counter
765
+
766
+ else:
767
+ raise NotImplementedError("Method not yet implemented")
768
+
769
+ else:
770
+ raise NotImplementedError("Method not yet implemented")
771
+
772
+
773
+ @register_inference_rule(operator.eq)
774
+ def eq_inference_rule(n: Node, symbols, constraints, counter):
775
+ assert isinstance(n.args[0], (Node, int))
776
+ assert isinstance(n.args[1], (Node, int))
777
+
778
+ e1 = symbols[n.args[0]] if isinstance(n.args[0], Node) else n.args[0]
779
+ e2 = symbols[n.args[1]] if isinstance(n.args[1], Node) else n.args[1]
780
+
781
+ if isinstance(n.args[0], Node) and isinstance(n.args[1], Node):
782
+ if isinstance(e1, TVar) and isinstance(e2, TVar):
783
+ eq_tensor, counter = gen_tvar(counter)
784
+ symbols[n] = eq_tensor
785
+ return gen_broadcasting_constraints(e1, e2, symbols, counter, eq_tensor)
786
+
787
+ elif isinstance(e1, DVar) and isinstance(e2, DVar):
788
+ # This is meant to be used for flow analysis only
789
+ eq_constraint = BinConstraintD(e1, e2, op_eq)
790
+
791
+ my_eq, counter = gen_bvar(counter)
792
+ equality_constraint = BinConstraintD(my_eq, eq_constraint, op_eq)
793
+ return [equality_constraint], counter
794
+
795
+ else:
796
+ raise RuntimeError("Sort Mismatch")
797
+
798
+ elif isinstance(n.args[0], Node) and not isinstance(n.args[1], Node):
799
+ if isinstance(e1, DVar):
800
+ # This is meant to be used for flow analysis only
801
+ eq_constraint = BinConstraintD(e1, e2, op_eq)
802
+
803
+ my_eq, counter = gen_bvar(counter)
804
+ equality_constraint = BinConstraintD(my_eq, eq_constraint, op_eq)
805
+ return [equality_constraint], counter
806
+ else:
807
+ raise NotImplementedError("Method not yet implemented")
808
+ else:
809
+ raise NotImplementedError("Method not yet implemented")
810
+
811
+
812
+ @register_inference_rule(operator.ne)
813
+ def neq_inference_rule(n: Node, symbols, constraints, counter):
814
+ """
815
+ Translates to inconsistent in gradual types.
816
+ To prove inequality, we should prove that
817
+ tensors are either different sizes or
818
+ disagree on at least one dimension
819
+
820
+ This is a WIP (works when the condition
821
+ is false. We are working on making this operation work
822
+ when the condition is true as well)
823
+ """
824
+ assert isinstance(n.args[0], Node)
825
+ assert isinstance(n.args[1], tuple)
826
+
827
+ # implementing for size 3 and 4
828
+ if len(n.args[1]) == 3:
829
+ assert isinstance(n.args[1][0], (Node, int))
830
+ assert isinstance(n.args[1][1], (Node, int))
831
+ assert isinstance(n.args[1][2], (Node, int))
832
+
833
+ lhs = symbols[n.args[0]]
834
+
835
+ b, counter = gen_tensor_dims(4, counter)
836
+ input_is_size3 = BinConstraintT(lhs, TensorType([b[0], b[1], b[2]]), op_eq)
837
+
838
+ d1 = n.args[1][0] if isinstance(n.args[1][0], int) else symbols[n.args[1][0]]
839
+ d2 = n.args[1][1] if isinstance(n.args[1][1], int) else symbols[n.args[1][1]]
840
+ d3 = n.args[1][2] if isinstance(n.args[1][2], int) else symbols[n.args[1][2]]
841
+
842
+ # dimensions not equal
843
+ my_ne, counter = gen_bvar(counter)
844
+ neq_1 = BinConstraintD(d1, b[0], op_neq)
845
+ neq_2 = BinConstraintD(d2, b[1], op_neq)
846
+ neq_3 = BinConstraintD(d3, b[2], op_neq)
847
+
848
+ # dimensions inconsistent
849
+ dims_inconsistent1 = Conj(
850
+ [BinConstraintD(d1, Dyn, op_neq), BinConstraintD(b[0], Dyn, op_neq), neq_1]
851
+ )
852
+ dims_inconsistent2 = Conj(
853
+ [BinConstraintD(d2, Dyn, op_neq), BinConstraintD(b[1], Dyn, op_neq), neq_2]
854
+ )
855
+ dims_inconsistent3 = Conj(
856
+ [BinConstraintD(d3, Dyn, op_neq), BinConstraintD(b[2], Dyn, op_neq), neq_3]
857
+ )
858
+
859
+ dims_inconsistent = Disj(
860
+ [dims_inconsistent1, dims_inconsistent2, dims_inconsistent3]
861
+ )
862
+
863
+ # we are covering size 3 and 4 only for now
864
+ ne_constraint = Conj([input_is_size3, dims_inconsistent])
865
+
866
+ my_ne, counter = gen_bvar(counter)
867
+ equality_constraint = BinConstraintD(my_ne, ne_constraint, op_eq)
868
+
869
+ elif len(n.args[1]) == 4:
870
+ assert isinstance(n.args[1][0], (Node, int))
871
+ assert isinstance(n.args[1][1], (Node, int))
872
+ assert isinstance(n.args[1][2], (Node, int))
873
+ assert isinstance(n.args[1][3], (Node, int))
874
+
875
+ lhs = symbols[n.args[0]]
876
+
877
+ b1, counter = gen_dvar(counter)
878
+ b2, counter = gen_dvar(counter)
879
+ b3, counter = gen_dvar(counter)
880
+ b4, counter = gen_dvar(counter)
881
+
882
+ input_is_size4 = BinConstraintT(lhs, TensorType([b1, b2, b3, b4]), op_eq)
883
+
884
+ d1 = n.args[1][0] if isinstance(n.args[1][0], int) else symbols[n.args[1][0]]
885
+ d2 = n.args[1][1] if isinstance(n.args[1][1], int) else symbols[n.args[1][1]]
886
+ d3 = n.args[1][2] if isinstance(n.args[1][2], int) else symbols[n.args[1][2]]
887
+ d4 = n.args[1][3] if isinstance(n.args[1][3], int) else symbols[n.args[1][3]]
888
+
889
+ # dimensions not equal
890
+ my_ne, counter = gen_bvar(counter)
891
+ neq_1 = BinConstraintD(d1, b1, op_neq)
892
+ neq_2 = BinConstraintD(d2, b2, op_neq)
893
+ neq_3 = BinConstraintD(d3, b3, op_neq)
894
+ neq_4 = BinConstraintD(d4, b4, op_neq)
895
+
896
+ # dimensions to inconsistent
897
+ dims_inconsistent1 = Conj(
898
+ [BinConstraintD(d1, Dyn, op_neq), BinConstraintD(b1, Dyn, op_neq), neq_1]
899
+ )
900
+ dims_inconsistent2 = Conj(
901
+ [BinConstraintD(d2, Dyn, op_neq), BinConstraintD(b2, Dyn, op_neq), neq_2]
902
+ )
903
+ dims_inconsistent3 = Conj(
904
+ [BinConstraintD(d3, Dyn, op_neq), BinConstraintD(b3, Dyn, op_neq), neq_3]
905
+ )
906
+ dims_inconsistent4 = Conj(
907
+ [BinConstraintD(d4, Dyn, op_neq), BinConstraintD(b3, Dyn, op_neq), neq_4]
908
+ )
909
+
910
+ dims_inconsistent = Disj(
911
+ [
912
+ dims_inconsistent1,
913
+ dims_inconsistent2,
914
+ dims_inconsistent3,
915
+ dims_inconsistent4,
916
+ ]
917
+ )
918
+
919
+ ne_constraint = Conj([input_is_size4, dims_inconsistent])
920
+
921
+ my_ne, counter = gen_bvar(counter)
922
+
923
+ equality_constraint = BinConstraintD(my_ne, ne_constraint, op_eq)
924
+
925
+ else:
926
+ raise NotImplementedError("Method not yet implemented")
927
+
928
+ return [equality_constraint], counter
929
+
930
+
931
+ @register_inference_rule(operator.lt)
932
+ def lt_inference_rule(n: Node, symbols, constraints, counter):
933
+ assert isinstance(n.args[0], (Node, int))
934
+ assert isinstance(n.args[1], (Node, int))
935
+
936
+ # We make sure this node will not be used again. We do not
937
+ # generate a constraint about that node. Only about the operands.
938
+
939
+ e1 = symbols[n.args[0]] if isinstance(n.args[0], Node) else n.args[0]
940
+ e2 = symbols[n.args[1]] if isinstance(n.args[1], Node) else n.args[1]
941
+
942
+ if isinstance(n.args[0], Node) and isinstance(n.args[1], Node):
943
+ if isinstance(e1, TVar) and isinstance(e2, TVar):
944
+ lt_tensor, counter = gen_tvar(counter)
945
+ symbols[n] = lt_tensor
946
+ return gen_broadcasting_constraints(e1, e2, symbols, counter, lt_tensor)
947
+
948
+ elif isinstance(e1, DVar) and isinstance(e2, DVar):
949
+ # This is meant to be used for flow analysis only
950
+ lt_constraint = BinConstraintD(e1, e2, op_lt)
951
+
952
+ my_lt, counter = gen_bvar(counter)
953
+ equality_constraint = BinConstraintD(my_lt, lt_constraint, op_eq)
954
+ return [equality_constraint], counter
955
+
956
+ else:
957
+ raise RuntimeError("Sort Mismatch")
958
+
959
+ elif isinstance(n.args[0], Node) and not isinstance(n.args[1], Node):
960
+ if isinstance(e1, DVar):
961
+ # This is meant to be used for flow analysis only
962
+ lt_constraint = BinConstraintD(e1, e2, op_lt)
963
+
964
+ my_lt, counter = gen_bvar(counter)
965
+ equality_constraint = BinConstraintD(my_lt, lt_constraint, op_eq)
966
+ return [equality_constraint], counter
967
+ else:
968
+ raise NotImplementedError("Method not yet implemented")
969
+
970
+ else:
971
+ raise NotImplementedError("Method not yet implemented")
972
+
973
+
974
+ @register_inference_rule(torch.full)
975
+ def full_inference_rule(n: Node, symbols, constraints, counter):
976
+ full, counter = gen_tvar(counter)
977
+ symbols[n] = full
978
+ res = []
979
+
980
+ assert isinstance(n.args[0], Iterable)
981
+ for arg in n.args[0]:
982
+ dim = arg if isinstance(arg, int) else symbols[arg]
983
+ res.append(dim)
984
+ c = BinConstraintT(full, TensorType(list(res)), op_eq) # type: ignore[arg-type]
985
+ return [c], counter
986
+
987
+
988
+ # TODO normalize index
989
+ @register_inference_rule(torch.arange)
990
+ def arange_inference_rule(n: Node, symbols, constraints, counter):
991
+ start = 0
992
+ step = 1
993
+
994
+ if len(n.args) == 1:
995
+ end = symbols[n.args[0]]
996
+ else:
997
+ raise NotImplementedError("Not yet implemented")
998
+
999
+ # int((end - start) / step)
1000
+ d1, counter = gen_dvar(counter)
1001
+ size_constraint = BinConstraintD(
1002
+ d1, BinConstraintD(BinConstraintD(end, start, op_sub), step, op_div), op_eq
1003
+ )
1004
+ arange, counter = gen_tvar(counter)
1005
+ symbols[n] = arange
1006
+
1007
+ # either the a parameter is a number or it is Dyn
1008
+ c1 = Disj(
1009
+ [
1010
+ BinConstraintD(end, Dyn, op_eq),
1011
+ BinConstraintD(start, Dyn, op_eq),
1012
+ BinConstraintD(step, Dyn, op_eq),
1013
+ ]
1014
+ )
1015
+ c2 = BinConstraintD(d1, Dyn, op_eq)
1016
+ both_dyn = Conj([c1, c2])
1017
+
1018
+ c11 = Conj(
1019
+ [
1020
+ BinConstraintD(end, Dyn, op_neq),
1021
+ BinConstraintD(start, Dyn, op_neq),
1022
+ BinConstraintD(step, Dyn, op_neq),
1023
+ ]
1024
+ )
1025
+ c22 = BinConstraintD(d1, Dyn, op_neq)
1026
+ both_numbers = Conj([c11, c22, size_constraint])
1027
+
1028
+ return [
1029
+ BinConstraintT(arange, TensorType([d1]), op_eq),
1030
+ Disj([both_dyn, both_numbers]),
1031
+ ], counter
1032
+
1033
+
1034
+ def gen_broadcasting_constraints(e1, e2, symbols, counter, output_var):
1035
+ # additional vars that don't correspond to expressions
1036
+ e11, counter = gen_tvar(counter)
1037
+ e22, counter = gen_tvar(counter)
1038
+
1039
+ # generate constraints
1040
+ c1 = TGreatestUpperBound(output_var, e11, e22)
1041
+ c2 = ApplyBroadcasting(e11, e22, e1, e2)
1042
+ c3 = BinConstraintT(e11, e22, op_consistency)
1043
+ return [c1, c2, c3], counter
1044
+
1045
+
1046
+ @register_inference_rule(operator.mul)
1047
+ @register_inference_rule(torch.ne)
1048
+ @register_inference_rule("ne")
1049
+ @register_inference_rule(torch.add)
1050
+ @register_inference_rule(operator.add)
1051
+ def broadcasting_inference_rule(n: Node, symbols, constraints, counter):
1052
+ op_code = None
1053
+ if n.target is operator.add or n.target is torch.add:
1054
+ op_code = op_add
1055
+ elif n.target is operator.mul:
1056
+ op_code = op_mul
1057
+
1058
+ if isinstance(n.args[0], Node) and isinstance(n.args[1], Node):
1059
+ if isinstance(symbols[n.args[0]], TVar) and isinstance(
1060
+ symbols[n.args[1]], TVar
1061
+ ):
1062
+ my_output, counter = gen_tvar(counter)
1063
+ symbols[n] = my_output
1064
+ e1 = symbols[n.args[0]]
1065
+ e2 = symbols[n.args[1]]
1066
+
1067
+ return gen_broadcasting_constraints(e1, e2, symbols, counter, my_output)
1068
+ else:
1069
+ raise NotImplementedError("Method not yet implemented")
1070
+
1071
+ elif isinstance(n.args[0], Node) and isinstance(n.args[1], (int, float)):
1072
+ if isinstance(symbols[n.args[0]], TVar):
1073
+ my_output, counter = gen_tvar(counter)
1074
+ symbols[n] = my_output
1075
+ e1 = symbols[n.args[0]]
1076
+ return [BinConstraintT(my_output, e1, op_eq)], counter
1077
+ elif isinstance(symbols[n.args[0]], DVar):
1078
+ my_output, counter = gen_dvar(counter) # type: ignore[arg-type,assignment]
1079
+ symbols[n] = my_output
1080
+ e1 = symbols[n.args[0]]
1081
+
1082
+ # we will propagate the runtime value here since this is regular addition
1083
+ c = Conj(
1084
+ [
1085
+ BinConstraintD(
1086
+ my_output, BinConstraintD(e1, n.args[1], op_code), op_eq
1087
+ ),
1088
+ BinConstraintD(0, my_output, op_leq),
1089
+ ]
1090
+ )
1091
+ return [c], counter
1092
+
1093
+ elif isinstance(n.args[1], Node) and isinstance(n.args[0], (int, float)):
1094
+ if isinstance(symbols[n.args[1]], TVar):
1095
+ my_output, counter = gen_tvar(counter)
1096
+ symbols[n] = my_output
1097
+ e2 = symbols[n.args[1]]
1098
+ return [BinConstraintT(my_output, e2, op_eq)], counter
1099
+ elif isinstance(symbols[n.args[1]], DVar):
1100
+ my_output, counter = gen_dvar(counter) # type: ignore[arg-type,assignment]
1101
+ symbols[n] = my_output
1102
+ e2 = symbols[n.args[1]]
1103
+
1104
+ # we will propagate the runtime value here since this is regular addition
1105
+ c = Conj(
1106
+ [
1107
+ BinConstraintD(
1108
+ my_output, BinConstraintD(e2, n.args[0], op_code), op_eq
1109
+ ),
1110
+ BinConstraintD(0, my_output, op_leq),
1111
+ ]
1112
+ )
1113
+ return [c], counter
1114
+
1115
+ else:
1116
+ raise NotImplementedError("Method not yet implemented")
1117
+
1118
+ else:
1119
+ # TODO generate add constraints for scalar addition
1120
+ raise NotImplementedError("Addition not yet implemented")
1121
+
1122
+
1123
+ @register_inference_rule(torch.flatten)
1124
+ def flatten_inference_rule(n: Node, symbols, constraints, counter):
1125
+ assert isinstance(n.args[0], Node)
1126
+
1127
+ # generate the new variable
1128
+ flattened, counter = gen_tvar(counter)
1129
+ symbols[n] = flattened
1130
+
1131
+ input = symbols[n.args[0]]
1132
+
1133
+ # set the default start and end dims
1134
+ start_dim = 1
1135
+ end_dim = -1
1136
+
1137
+ if len(n.args) > 1:
1138
+ assert isinstance(n.args[1], int)
1139
+ start_dim = n.args[1]
1140
+
1141
+ if len(n.args) > 2:
1142
+ assert isinstance(n.args[2], int)
1143
+ end_dim = n.args[2]
1144
+
1145
+ c1 = BinConstraintT(input, Dyn, op_eq)
1146
+ c2 = BinConstraintT(flattened, Dyn, op_eq)
1147
+ both_dyn = Conj([c1, c2])
1148
+
1149
+ const = []
1150
+ for i in range(1, MAX_TENSOR_RANK + 1):
1151
+ c, counter = generate_flatten_constraints(
1152
+ start_dim, end_dim, input, flattened, i, counter
1153
+ )
1154
+ const.append(c)
1155
+
1156
+ return [Disj([both_dyn, *const])], counter
1157
+
1158
+
1159
+ @register_inference_rule(torch.nn.functional.layer_norm)
1160
+ def layer_norm_functional(n: Node, symbols, constraints, counter):
1161
+ """
1162
+ We generate the constraint: input = output
1163
+ """
1164
+ assert isinstance(n.args[0], Node)
1165
+ return gen_layer_norm_constraints(n, n.args[1], symbols, counter)
1166
+
1167
+
1168
+ @register_inference_rule(torch.nn.LayerNorm)
1169
+ def layer_norm_inference_rule(n: Node, module_instance, symbols, constraints, counter):
1170
+ """
1171
+ Input and output shapes should be equal.
1172
+ Input should be consistent with the normalized_shape
1173
+ """
1174
+ assert isinstance(n.args[0], Node)
1175
+ return gen_layer_norm_constraints(
1176
+ n, module_instance.normalized_shape, symbols, counter
1177
+ )
1178
+
1179
+
1180
+ def gen_layer_norm_constraints(n: Node, normalized_shape, symbols, counter):
1181
+ output, counter = gen_tvar(counter)
1182
+ symbols[n] = output
1183
+ input = symbols[n.args[0]]
1184
+
1185
+ input_dyn = BinConstraintT(input, Dyn, op_eq)
1186
+ output_dyn = BinConstraintT(output, Dyn, op_eq)
1187
+
1188
+ c1 = Conj([input_dyn, output_dyn])
1189
+
1190
+ c2 = []
1191
+ for i in range(1, MAX_TENSOR_RANK + 1):
1192
+ new_dims_rhs, counter = gen_tensor_dims(i, counter)
1193
+ nat_constraints = gen_nat_constraints(new_dims_rhs)
1194
+
1195
+ c_tensor_i = Conj(
1196
+ [
1197
+ BinConstraintT(input, TensorType(new_dims_rhs), op_eq),
1198
+ BinConstraintT(output, TensorType(new_dims_rhs), op_eq),
1199
+ ]
1200
+ + add_layer_norm_constraints(new_dims_rhs, list(normalized_shape))
1201
+ + nat_constraints
1202
+ )
1203
+ c2.append(c_tensor_i)
1204
+ return [Disj([c1, Disj(c2)])], counter
1205
+
1206
+
1207
+ @register_inference_rule(torch.nn.Dropout)
1208
+ @register_inference_rule(torch.nn.ReLU)
1209
+ def relu_inference_rule(n: Node, module_instance, symbols, constraints, counter):
1210
+ """
1211
+ Input and output shapes should be equal.
1212
+ """
1213
+ assert isinstance(n.args[0], Node)
1214
+ output, counter = gen_tvar(counter)
1215
+ symbols[n] = output
1216
+ input = symbols[n.args[0]]
1217
+ assert isinstance(input, TVar)
1218
+ return [BinConstraintT(input, output, op_eq)], counter
1219
+
1220
+
1221
+ @register_inference_rule(torch.nn.Linear)
1222
+ def linear_inference_rule(n: Node, module_instance, symbols, constraints, counter):
1223
+ """
1224
+ Input and output sizes should be the same except for the last dimension
1225
+ If the input is Dyn, then so should the output
1226
+ """
1227
+ assert isinstance(n.args[0], Node)
1228
+ return linear_constraints(
1229
+ n, module_instance.in_features, module_instance.out_features, symbols, counter
1230
+ )
1231
+
1232
+
1233
+ @register_inference_rule("dim")
1234
+ def torch_dim_inference_rule(n: Node, symbols, constraints, counter):
1235
+ assert isinstance(n.args[0], Node)
1236
+ my_dim, counter = gen_dvar(counter)
1237
+ symbols[n] = my_dim
1238
+ input = symbols[n.args[0]]
1239
+
1240
+ input_dyn = BinConstraintT(input, Dyn, op_eq)
1241
+ output_dyn = BinConstraintD(my_dim, Dyn, op_eq)
1242
+
1243
+ c1 = []
1244
+
1245
+ for i in range(1, MAX_TENSOR_RANK + 1):
1246
+ new_dims_rhs_1, counter = gen_tensor_dims(i, counter)
1247
+
1248
+ c_tensor_i = Conj(
1249
+ [
1250
+ BinConstraintT(input, TensorType(new_dims_rhs_1), op_eq),
1251
+ BinConstraintD(my_dim, i, op_eq),
1252
+ ]
1253
+ )
1254
+ c1.append(c_tensor_i)
1255
+
1256
+ return [Disj([Conj([input_dyn, output_dyn]), Disj(c1)])], counter
1257
+
1258
+
1259
+ @register_inference_rule(torch._C._nn.linear)
1260
+ def torch_linear_inference_rule(n: Node, symbols, constraints, counter):
1261
+ assert isinstance(n.args[0], Node)
1262
+ weight_dims, counter = gen_tensor_dims(2, counter)
1263
+ equality_constraint = BinConstraintT(
1264
+ symbols[n.args[1]], TensorType(weight_dims), op_eq
1265
+ )
1266
+ constraints, counter = linear_constraints(
1267
+ n, weight_dims[1], weight_dims[0], symbols, counter
1268
+ )
1269
+ return [equality_constraint] + constraints, counter
1270
+
1271
+
1272
+ def linear_constraints(n: Node, in_features, out_features, symbols, counter):
1273
+ linear_output, counter = gen_tvar(counter)
1274
+ symbols[n] = linear_output
1275
+ linear_input = symbols[n.args[0]]
1276
+
1277
+ input_dyn = BinConstraintT(linear_input, Dyn, op_eq)
1278
+ output_dyn = BinConstraintT(linear_output, Dyn, op_eq)
1279
+
1280
+ c1 = Conj([input_dyn, output_dyn])
1281
+
1282
+ c2 = []
1283
+ for i in range(1, MAX_TENSOR_RANK + 1):
1284
+ new_dims_rhs_1, counter = gen_tensor_dims(i, counter)
1285
+ new_dims_rhs_2, counter = gen_tensor_dims(i, counter)
1286
+
1287
+ nat_constraints = gen_nat_constraints(new_dims_rhs_1 + new_dims_rhs_2)
1288
+
1289
+ c_tensor_i = Conj(
1290
+ [
1291
+ BinConstraintT(linear_input, TensorType(new_dims_rhs_1), op_eq),
1292
+ BinConstraintT(linear_output, TensorType(new_dims_rhs_2), op_eq),
1293
+ ]
1294
+ + add_linear_constraints(
1295
+ new_dims_rhs_1, new_dims_rhs_2, in_features, out_features
1296
+ )
1297
+ + nat_constraints
1298
+ )
1299
+ c2.append(c_tensor_i)
1300
+ return [Disj([c1, Disj(c2)])], counter
1301
+
1302
+
1303
+ def add_layer_norm_constraints(input_dim, normalized_dim):
1304
+ """
1305
+ The constraints say that the type has te form: [*, 1024, 1024]
1306
+ while the normalized_dim have the form [1024, 1024]
1307
+ Args:
1308
+ input_dim: Input shape of layer norm
1309
+ normalized_dim: normalized_dim parameter of the module instance
1310
+
1311
+ """
1312
+
1313
+ # in this case we return false since there's a pattern mismatch
1314
+ if len(normalized_dim) > len(input_dim):
1315
+ return [F()]
1316
+
1317
+ else:
1318
+ constraints = []
1319
+ for i, n in zip(reversed(input_dim), reversed(normalized_dim)):
1320
+ constraints.append(BinConstraintD(i, n, op_consistency))
1321
+ return constraints
1322
+
1323
+
1324
+ def add_linear_constraints(dims1, dims2, in_features, out_features):
1325
+ assert len(dims1) == len(dims2)
1326
+ constraints = []
1327
+ for i in range(len(dims1)):
1328
+ if i == len(dims1) - 1:
1329
+ constraints.append(BinConstraintD(dims1[i], in_features, op_consistency))
1330
+ constraints.append(BinConstraintD(dims2[i], out_features, op_eq))
1331
+ else:
1332
+ constraints.append(BinConstraintD(dims1[i], dims2[i], op_eq))
1333
+
1334
+ return constraints
1335
+
1336
+
1337
+ @register_inference_rule(torch.reshape)
1338
+ def reshape_inference_rule(n: Node, symbols, constraints, counter):
1339
+ assert isinstance(n.args[0], Node)
1340
+
1341
+ # generate the new variable
1342
+ my_reshape, counter = gen_tvar(counter)
1343
+ symbols[n] = my_reshape
1344
+
1345
+ src_var = symbols[n.args[0]]
1346
+ t2 = n.args[1]
1347
+ t2_type = TensorType([Dyn if elem == -1 else elem for elem in t2]) # type: ignore[union-attr]
1348
+ c1 = BinConstraintT(my_reshape, t2_type, op_eq) # type: ignore[union-attr]
1349
+ c2 = CanReshape(src_var, t2_type)
1350
+
1351
+ return [c1, c2], counter
1352
+
1353
+
1354
+ @register_inference_rule(BatchNorm2d)
1355
+ def batchnorm_inference_rule(n: Node, module_instance, symbols, constraints, counter):
1356
+ assert isinstance(n.args[0], Node)
1357
+
1358
+ # generate the new variable
1359
+ batchnorm_output, counter = gen_tvar(counter)
1360
+ symbols[n] = batchnorm_output
1361
+ batchnorm_input = symbols[n.args[0]]
1362
+
1363
+ # dim vars
1364
+ d1, counter = gen_dvar(counter)
1365
+ d2, counter = gen_dvar(counter)
1366
+ d3, counter = gen_dvar(counter)
1367
+ d4, counter = gen_dvar(counter)
1368
+
1369
+ nat_constraints = gen_nat_constraints([d1, d2, d3, d4])
1370
+
1371
+ c1 = BinConstraintT(batchnorm_input, TensorType([d1, d2, d3, d4]), op_matching)
1372
+ c2 = BinConstraintT(batchnorm_input, batchnorm_output, op_eq)
1373
+ return [c1, c2, *nat_constraints], counter
1374
+
1375
+
1376
+ @register_inference_rule(torch.nn.AdaptiveAvgPool2d)
1377
+ def adaptive_inference_rule(n: Node, module_instance, symbols, constraints, counter):
1378
+ assert isinstance(n.args[0], Node)
1379
+
1380
+ avg_pool, counter = gen_tvar(counter)
1381
+
1382
+ symbols[n] = avg_pool
1383
+ input_var = symbols[n.args[0]]
1384
+
1385
+ # dim vars
1386
+ d1, counter = gen_dvar(counter)
1387
+ d2, counter = gen_dvar(counter)
1388
+ d3, counter = gen_dvar(counter)
1389
+ d4, counter = gen_dvar(counter)
1390
+ nat_constraints = gen_nat_constraints([d1, d2, d3, d4])
1391
+ c1 = BinConstraintT(input_var, TensorType([d1, d2, d3, d4]), op_matching)
1392
+ c2 = BinConstraintT(
1393
+ avg_pool,
1394
+ TensorType(
1395
+ [d1, d2, module_instance.output_size[0], module_instance.output_size[1]]
1396
+ ),
1397
+ op_eq,
1398
+ )
1399
+
1400
+ return [c1, c2, *nat_constraints], counter
1401
+
1402
+
1403
+ @register_inference_rule(Conv2d)
1404
+ def conv2d_inference_rule(n: Node, module_instance, symbols, constraints, counter):
1405
+ assert isinstance(n.args[0], Node)
1406
+
1407
+ my_conv, counter = gen_tvar(counter)
1408
+ symbols[n] = my_conv
1409
+ input_var = symbols[n.args[0]]
1410
+
1411
+ # dim vars
1412
+ [d1, d2, d3, d4], counter = gen_tensor_dims(MAX_TENSOR_RANK, counter)
1413
+
1414
+ # c1 = Matching(input_var, TensorType([d1, d2, d3, d4]))
1415
+ c1 = BinConstraintT(input_var, TensorType([d1, d2, d3, d4]), op_matching)
1416
+
1417
+ # c2 = DConsistency(module_instance.in_channels, d2)
1418
+ c2 = BinConstraintD(module_instance.in_channels, d2, op_consistency)
1419
+
1420
+ c3 = CalcConv(
1421
+ my_conv,
1422
+ input_var,
1423
+ module_instance.out_channels,
1424
+ module_instance.kernel_size,
1425
+ module_instance.padding,
1426
+ module_instance.stride,
1427
+ module_instance.dilation,
1428
+ [d1, d2, d3, d4],
1429
+ )
1430
+
1431
+ nat_constraints = gen_nat_constraints([d1, d2, d3, d4])
1432
+
1433
+ return [c1, c2, c3, *nat_constraints], counter
1434
+
1435
+
1436
+ @register_inference_rule(torch.nn.MaxPool2d)
1437
+ def maxpool_inference_rule(n: Node, module_instance, symbols, constraints, counter):
1438
+ assert isinstance(n.args[0], Node)
1439
+ maxpool, counter = gen_tvar(counter)
1440
+ symbols[n] = maxpool
1441
+ input_var = symbols[n.args[0]]
1442
+
1443
+ # dim vars
1444
+ [d1, d2, d3, d4], counter = gen_tensor_dims(MAX_TENSOR_RANK, counter)
1445
+
1446
+ c1 = BinConstraintT(input_var, TensorType([d1, d2, d3, d4]), op_matching)
1447
+
1448
+ c2 = CalcMaxPool(
1449
+ maxpool,
1450
+ input_var,
1451
+ module_instance.kernel_size,
1452
+ module_instance.padding,
1453
+ module_instance.stride,
1454
+ module_instance.dilation,
1455
+ [d1, d2, d3, d4],
1456
+ )
1457
+
1458
+ nat_constraints = gen_nat_constraints([d1, d2, d3, d4])
1459
+
1460
+ return [c1, c2, *nat_constraints], counter
1461
+
1462
+
1463
+ class ConstraintGenerator:
1464
+ def __init__(self, traced, graph=None):
1465
+ self.traced = traced # traced or tracer.root
1466
+ self.traced_params = dict(self.traced.named_parameters())
1467
+ self.constraints = []
1468
+ self.symbol_dict = {}
1469
+ self.graph = traced.graph if hasattr(traced, "graph") else graph
1470
+
1471
+ def generate_constraints(self, counter=0):
1472
+ """
1473
+ Iterate through every node and generate constraints
1474
+ Effect: self.constraints will be populated with the final constraints
1475
+ """
1476
+ graph = self.graph
1477
+
1478
+ all_constraints = []
1479
+
1480
+ # pyrefly: ignore [missing-attribute]
1481
+ for n in graph.nodes:
1482
+ (constraints, counter) = self.generate_constraints_node(n, counter)
1483
+ all_constraints += constraints
1484
+
1485
+ return Conj(all_constraints), counter
1486
+
1487
+ def generate_constraints_node(self, n: Node, counter):
1488
+ """
1489
+ Generate constraints the given node:
1490
+ Currently supported operations:
1491
+ - Reshape
1492
+ - Add
1493
+ - conv2d
1494
+ """
1495
+
1496
+ if n.op == "placeholder":
1497
+ x, counter = gen_tvar(counter)
1498
+ self.symbol_dict[n] = x
1499
+
1500
+ my_type = n.type
1501
+
1502
+ if n.type != Dyn and (not isinstance(n.type, TensorType)):
1503
+ if n.type == torch.nn.parameter.Parameter:
1504
+ # since we have a parameter, the shape must be static
1505
+ assert "example_value" in n.meta
1506
+ my_type = TensorType(n.meta["example_value"].size())
1507
+ else:
1508
+ my_type = Dyn
1509
+
1510
+ c1 = BinConstraintT(my_type, x, op_precision)
1511
+ c2 = BinConstraintT(x, MAX_TENSOR_RANK, op_leq)
1512
+ return [c1, c2], counter
1513
+
1514
+ elif n.op == "call_function":
1515
+ if n.target in _INFERENCE_RULES:
1516
+ return _INFERENCE_RULES[n.target](
1517
+ n, self.symbol_dict, self.constraints, counter
1518
+ )
1519
+ else:
1520
+ raise RuntimeError(
1521
+ f"No inference rule registered for target {n.target}!"
1522
+ )
1523
+
1524
+ elif n.op == "call_module":
1525
+ module_instance = self.traced.get_submodule(n.target)
1526
+ if type(module_instance) in _INFERENCE_RULES:
1527
+ return _INFERENCE_RULES[type(module_instance)](
1528
+ n, module_instance, self.symbol_dict, self.constraints, counter
1529
+ )
1530
+ else:
1531
+ raise RuntimeError(
1532
+ f"No inference rule registered for class {type(module_instance)}!"
1533
+ )
1534
+
1535
+ elif n.op == "call_method":
1536
+ if n.target in _INFERENCE_RULES:
1537
+ return _INFERENCE_RULES[n.target](
1538
+ n, self.symbol_dict, self.constraints, counter
1539
+ )
1540
+ else:
1541
+ raise RuntimeError(
1542
+ f"No inference rule registered for target {n.target}!"
1543
+ )
1544
+
1545
+ elif n.op == "get_attr":
1546
+ t = self.traced_params.get(n.target, None)
1547
+
1548
+ if isinstance(t, torch.Tensor):
1549
+ if len(t.shape) > 0:
1550
+ res = list(t.shape)
1551
+ attr_type = TensorType(res)
1552
+ output, counter = gen_tvar(counter)
1553
+ self.symbol_dict[n] = output
1554
+ return [BinConstraintT(output, attr_type, op_eq)], counter
1555
+ else:
1556
+ # scalar?
1557
+ return [], counter
1558
+ else:
1559
+ return [], counter
1560
+
1561
+ elif n.op == "output":
1562
+ return [], counter
1563
+
1564
+ else:
1565
+ raise NotImplementedError(f"Method {n.op} not yet implemented")
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/constraint_transformation.py ADDED
@@ -0,0 +1,1322 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: ignore-errors
2
+ import copy
3
+ import itertools
4
+ from collections.abc import Callable
5
+
6
+ from torch.fx.experimental.migrate_gradual_types.constraint import (
7
+ ApplyBroadcasting,
8
+ BinConstraintD,
9
+ CalcConv,
10
+ CalcMaxPool,
11
+ CalcProduct,
12
+ CanReshape,
13
+ Conj,
14
+ Constraint,
15
+ DGreatestUpperBound,
16
+ Disj,
17
+ DVar,
18
+ F,
19
+ GetItem,
20
+ GetItemTensor,
21
+ IndexSelect,
22
+ Prod,
23
+ T,
24
+ TGreatestUpperBound,
25
+ Transpose,
26
+ TVar,
27
+ )
28
+ from torch.fx.experimental.migrate_gradual_types.constraint_generator import (
29
+ BinConstraintT,
30
+ MAX_TENSOR_RANK,
31
+ )
32
+ from torch.fx.experimental.migrate_gradual_types.operation import (
33
+ op_add,
34
+ op_consistency,
35
+ op_div,
36
+ op_eq,
37
+ op_leq,
38
+ op_matching,
39
+ op_mod,
40
+ op_mul,
41
+ op_neq,
42
+ op_precision,
43
+ op_sub,
44
+ )
45
+ from torch.fx.experimental.migrate_gradual_types.util import (
46
+ gen_dvar,
47
+ gen_nat_constraints,
48
+ gen_tensor_dims,
49
+ )
50
+ from torch.fx.tensor_type import Dyn, TensorType
51
+
52
+
53
+ _TRANSFORMATION_RULES: dict[Constraint, Callable] = {}
54
+
55
+
56
+ def register_transformation_rule(call_target):
57
+ def register(fn):
58
+ if call_target in _TRANSFORMATION_RULES:
59
+ raise RuntimeError(
60
+ f"Transformation rule already registered for {call_target}!"
61
+ )
62
+ _TRANSFORMATION_RULES[call_target] = fn
63
+ return fn
64
+
65
+ return register
66
+
67
+
68
+ def valid_index(index, dims):
69
+ """
70
+ Given a list of dimensions, checks if an index is valid in the list
71
+ """
72
+ try:
73
+ dims[index]
74
+ return T()
75
+ except IndexError:
76
+ return F()
77
+
78
+
79
+ @register_transformation_rule(Transpose)
80
+ def transform_transpose(constraint, counter):
81
+ """
82
+ Similar to a sequence of two index-selects
83
+ """
84
+ dims, counter = gen_tensor_dims(constraint.tensor_size, counter)
85
+ is_valid_index1 = valid_index(constraint.index1, dims)
86
+ is_valid_index2 = valid_index(constraint.index2, dims)
87
+ new_dims = copy.deepcopy(dims)
88
+ nat_constraints = gen_nat_constraints(dims)
89
+
90
+ if is_valid_index1 == T() and is_valid_index2 == T():
91
+ new_dims[constraint.index1] = dims[constraint.index2]
92
+ new_dims[constraint.index2] = dims[constraint.index1]
93
+
94
+ transformed_constraint = Conj(
95
+ [
96
+ BinConstraintT(constraint.input_var, TensorType(dims), op_eq),
97
+ *nat_constraints,
98
+ is_valid_index1,
99
+ is_valid_index2,
100
+ BinConstraintT(constraint.output, TensorType(new_dims), op_eq),
101
+ ]
102
+ )
103
+ return transformed_constraint, counter
104
+
105
+
106
+ @register_transformation_rule(IndexSelect)
107
+ def transform_index_select(constraint, counter):
108
+ """
109
+ The constraints consider the given tensor size, checks if the index is valid
110
+ and if so, generates a constraint for replacing the input dimension
111
+ with the required dimension
112
+ """
113
+ dims, counter = gen_tensor_dims(constraint.tensor_size, counter)
114
+ is_valid_index = valid_index(constraint.index, dims)
115
+ nat_constraints = gen_nat_constraints(dims)
116
+
117
+ # if the index is valid then replace the input dimension with the new dimension
118
+ # otherwise the dimension will not be replaced and the clause will contain False
119
+ if is_valid_index == T():
120
+ new_dims = copy.deepcopy(dims)
121
+ new_dims[constraint.index] = constraint.dim_replace
122
+
123
+ transformed_constraint = Conj(
124
+ [
125
+ BinConstraintT(constraint.input_var, TensorType(dims), op_eq),
126
+ *nat_constraints,
127
+ is_valid_index,
128
+ BinConstraintT(constraint.output, TensorType(new_dims), op_eq),
129
+ ]
130
+ )
131
+
132
+ # print(constraints)
133
+ return transformed_constraint, counter
134
+
135
+
136
+ @register_transformation_rule(GetItem)
137
+ def transform_get_item(constraint, counter):
138
+ """
139
+ generate an equality of the form:
140
+ t = [a1, ..., an]
141
+ then generate constraints that check if the given index is valid
142
+ given this particular tensor size.
143
+ If the index is valid, generate a constraint to get the item
144
+ Note that we already handled the Dyn input case in the previous
145
+ step.
146
+ Args:
147
+ constraint: GetItem which assumes we are getting an item from a tensor (not Dyn)
148
+ counter: variable tracking
149
+ Returns: simplified constraints for GetItem
150
+
151
+ """
152
+ dims, counter = gen_tensor_dims(constraint.tensor_size, counter)
153
+ nat_constraints = gen_nat_constraints(dims)
154
+
155
+ is_valid_index = valid_index(constraint.index, dims)
156
+
157
+ all_constraints = [
158
+ BinConstraintT(constraint.input_var, TensorType(dims), op_eq),
159
+ *nat_constraints,
160
+ is_valid_index,
161
+ ]
162
+
163
+ # if the index is valid, we generate a constraint for getting an item
164
+ # otherwise this clause will have been UNSAT due to the wrong index
165
+ if is_valid_index == T():
166
+ all_constraints.append(
167
+ BinConstraintD(constraint.res, dims[constraint.index], op_eq)
168
+ )
169
+
170
+ return Conj(all_constraints), counter
171
+
172
+
173
+ def valid_index_tensor(index, dims):
174
+ """
175
+ if the slice instances exceed the length of the dimensions
176
+ then this is a type error so we return False
177
+ """
178
+ slice_count = 0
179
+ for s in index:
180
+ if isinstance(s, slice):
181
+ slice_count += 1
182
+ if slice_count > len(dims):
183
+ return F()
184
+ else:
185
+ return T()
186
+
187
+
188
+ @register_transformation_rule(GetItemTensor)
189
+ def transform_get_item_tensor(constraint, counter):
190
+ """
191
+ When the index is a tuple, then the output will be a tensor
192
+ TODO: we have to check if this is the case for all HF models
193
+
194
+ The cases we are covering here are a tuple with one of:
195
+ - slice with default argument
196
+ - None
197
+
198
+ None appends 1 to the input tensor dimensions
199
+ so each occurrence of 'None' increases the rank by 1
200
+
201
+ slice with default arguments does not change the rank
202
+ """
203
+ assert isinstance(constraint.index_tuple, tuple)
204
+
205
+ # generate a result tensor of the expected size
206
+ dims, counter = gen_tensor_dims(constraint.tensor_size, counter)
207
+ nat_constraints = gen_nat_constraints(dims)
208
+
209
+ # generate a place-holder list of the right rank
210
+ # where "slice" does not contribute to the rank and "None" does
211
+ none_c = constraint.index_tuple.count(None)
212
+ resulting_tensor_dims = (none_c + len(dims)) * [None]
213
+
214
+ dim_index = 0
215
+ for i in range(len(constraint.index_tuple)):
216
+ # append 1 to the right location of the resulting tensor
217
+ if constraint.index_tuple[i] is None:
218
+ resulting_tensor_dims[i] = 1
219
+
220
+ elif constraint.index_tuple[i] == slice(None, None, None):
221
+ pass
222
+
223
+ else:
224
+ raise NotImplementedError("Method not yet implemented")
225
+
226
+ # append the remaining dimensions to the right location
227
+ dim_index = 0
228
+ for i in range(len(resulting_tensor_dims)):
229
+ if resulting_tensor_dims[i] is None:
230
+ resulting_tensor_dims[i] = dims[dim_index]
231
+ dim_index += 1
232
+
233
+ # check if the index is valid
234
+ is_valid_index = valid_index_tensor(constraint.index_tuple, dims)
235
+
236
+ # check if the resulting tensor is within bounds
237
+ if len(resulting_tensor_dims) > 4:
238
+ return F(), counter
239
+
240
+ else:
241
+ constraints = [
242
+ BinConstraintT(constraint.input_var, TensorType(dims), op_eq),
243
+ BinConstraintT(constraint.res, TensorType(resulting_tensor_dims), op_eq),
244
+ *nat_constraints,
245
+ is_valid_index,
246
+ ]
247
+ return Conj(constraints), counter
248
+
249
+
250
+ @register_transformation_rule(BinConstraintT)
251
+ def generate_binconstraint_t(constraint, counter):
252
+ """
253
+ Transform binary constraints for tensors
254
+ """
255
+
256
+ # precision constraints
257
+ if constraint.op == op_precision:
258
+ if constraint.lhs == Dyn:
259
+ return T(), counter
260
+ elif isinstance(constraint.lhs, TensorType):
261
+ is_fully_static = all(d != Dyn for d in constraint.lhs.__args__)
262
+ if is_fully_static:
263
+ return BinConstraintT(constraint.lhs, constraint.rhs, op_eq), counter
264
+ else:
265
+ new_dims = []
266
+
267
+ for _ in range(len(constraint.lhs.__args__)):
268
+ dim, counter = gen_dvar(counter)
269
+ new_dims.append(dim)
270
+
271
+ new_dim_constraints = (
272
+ [
273
+ BinConstraintD(old_dim, new_dim, op_precision)
274
+ for new_dim, old_dim in zip(new_dims, constraint.lhs.__args__)
275
+ ]
276
+ + [BinConstraintT(constraint.rhs, TensorType(new_dims), op_eq)]
277
+ + [BinConstraintD(1, new_dim, op_leq) for new_dim in new_dims]
278
+ )
279
+ return Conj(new_dim_constraints), counter
280
+
281
+ # matching
282
+ elif constraint.op == op_matching:
283
+ assert isinstance(constraint.rhs, TensorType)
284
+ d1 = constraint.rhs.__args__[0]
285
+ d2 = constraint.rhs.__args__[1]
286
+ d3 = constraint.rhs.__args__[2]
287
+ d4 = constraint.rhs.__args__[3]
288
+
289
+ conj = [
290
+ BinConstraintT(constraint.lhs, Dyn, op_eq),
291
+ BinConstraintD(d1, Dyn, op_eq),
292
+ BinConstraintD(d2, Dyn, op_eq),
293
+ BinConstraintD(d3, Dyn, op_eq),
294
+ BinConstraintD(d4, Dyn, op_eq),
295
+ ]
296
+ return (
297
+ Disj(
298
+ [
299
+ Conj(conj),
300
+ BinConstraintT(constraint.lhs, TensorType([d1, d2, d3, d4]), op_eq),
301
+ ]
302
+ ),
303
+ counter,
304
+ )
305
+
306
+ elif constraint.op == op_consistency:
307
+ c_dyn = Disj(
308
+ [
309
+ BinConstraintT(constraint.lhs, Dyn, op_eq),
310
+ BinConstraintT(constraint.rhs, Dyn, op_eq),
311
+ ]
312
+ )
313
+ (
314
+ (
315
+ c_tensor_1,
316
+ c_tensor_2,
317
+ c_tensor_3,
318
+ c_tensor_4,
319
+ ),
320
+ counter,
321
+ ) = gen_consistency_constraints(constraint, counter)
322
+
323
+ return Disj([c_dyn, c_tensor_1, c_tensor_2, c_tensor_3, c_tensor_4]), counter
324
+
325
+ elif constraint.op == op_leq:
326
+ assert isinstance(constraint.rhs, int)
327
+ disj = [BinConstraintT(constraint.lhs, Dyn, op_eq)]
328
+ for i in range(1, constraint.rhs + 1):
329
+ dims = []
330
+ for _ in range(1, i + 1):
331
+ dim_var, counter = gen_dvar(counter)
332
+ dims.append(dim_var)
333
+ disj.append(BinConstraintT(constraint.lhs, TensorType(dims), op_eq))
334
+ return Disj(disj), counter
335
+ else:
336
+ return constraint, counter
337
+
338
+
339
+ @register_transformation_rule(BinConstraintD)
340
+ def generate_binconstraint_d(constraint, counter):
341
+ """
342
+ Transform binary constraints for dimensions
343
+ """
344
+ if constraint.op == op_precision:
345
+ if isinstance(constraint.lhs, int):
346
+ return BinConstraintD(constraint.lhs, constraint.rhs, op_eq), counter
347
+ elif constraint.lhs == Dyn:
348
+ return T(), counter
349
+
350
+ elif constraint.op == op_consistency:
351
+ return (
352
+ Disj(
353
+ [
354
+ BinConstraintD(constraint.lhs, constraint.rhs, op_eq),
355
+ BinConstraintD(constraint.rhs, Dyn, op_eq),
356
+ BinConstraintD(constraint.lhs, Dyn, op_eq),
357
+ ]
358
+ ),
359
+ counter,
360
+ )
361
+
362
+ else:
363
+ return constraint, counter
364
+
365
+
366
+ @register_transformation_rule(Conj)
367
+ def generate_conj(constraint, counter):
368
+ """
369
+ Transform conjunctions
370
+ """
371
+ new = []
372
+ for c in constraint.conjucts:
373
+ new_c, counter = transform_constraint(c, counter)
374
+ new.append(new_c)
375
+ return Conj(new), counter
376
+
377
+
378
+ @register_transformation_rule(Disj)
379
+ def generate_disj(constraint, counter):
380
+ """
381
+ Transform disjunctions
382
+ """
383
+ new = []
384
+ for c in constraint.disjuncts:
385
+ new_c, counter = transform_constraint(c, counter)
386
+ new.append(new_c)
387
+ return Disj(new), counter
388
+
389
+
390
+ @register_transformation_rule(TGreatestUpperBound)
391
+ def generate_gub(constraint, counter):
392
+ """
393
+ Transform greatest upper bound for tensors. Results in equality and Greatest Upper Bound
394
+ on dimensions
395
+ """
396
+ c1 = Conj(
397
+ [
398
+ Disj(
399
+ [
400
+ BinConstraintT(constraint.rhs1, Dyn, op_eq),
401
+ BinConstraintT(constraint.rhs2, Dyn, op_eq),
402
+ ]
403
+ ),
404
+ BinConstraintT(constraint.res, Dyn, op_eq),
405
+ ]
406
+ )
407
+
408
+ [c2, c3, c4, c5], counter = gen_greatest_upper_bound(constraint, counter)
409
+
410
+ return Disj([c1, c2, c3, c4, c5]), counter
411
+
412
+
413
+ @register_transformation_rule(DGreatestUpperBound)
414
+ def generate_d_gub(constraint, counter):
415
+ """
416
+ Transform greatest upper bound for dimensions into equality constraints
417
+ """
418
+ c1 = Conj(
419
+ [
420
+ BinConstraintD(constraint.rhs1, Dyn, op_eq),
421
+ BinConstraintD(constraint.res, constraint.rhs2, op_eq),
422
+ ]
423
+ )
424
+ c2 = Conj(
425
+ [
426
+ BinConstraintD(constraint.rhs2, Dyn, op_eq),
427
+ BinConstraintD(constraint.res, constraint.rhs1, op_eq),
428
+ ]
429
+ )
430
+ c3 = Conj(
431
+ [
432
+ BinConstraintD(constraint.rhs2, constraint.rhs1, op_eq),
433
+ BinConstraintD(constraint.res, constraint.rhs1, op_eq),
434
+ ]
435
+ )
436
+ return Disj([c1, c2, c3]), counter
437
+
438
+
439
+ @register_transformation_rule(CalcConv)
440
+ def generate_calc_conv(constraint, counter):
441
+ d, counter = gen_tensor_dims(4, counter)
442
+ conv_result = TensorType([d[0], d[1], d[2], d[3]])
443
+
444
+ # the convolution result is a tensor of size 4
445
+ c1 = BinConstraintT(constraint.conv_result, conv_result, op_eq)
446
+
447
+ # the second dimension of the output is equal to the output channels
448
+ c2 = Conj(
449
+ [
450
+ BinConstraintD(d[1], constraint.c_out, op_eq),
451
+ BinConstraintD(d[1], Dyn, op_neq),
452
+ ]
453
+ )
454
+
455
+ # the input corresponds to the output in the first dimension of the convolution
456
+ c3 = BinConstraintD(constraint.matching_constraint[0], d[0], op_eq)
457
+
458
+ c4, c5 = calc_last_two_dims(constraint, d)
459
+
460
+ leq_constraints = Conj(
461
+ [
462
+ BinConstraintD(0, d[0], op_leq),
463
+ BinConstraintD(0, d[1], op_leq),
464
+ BinConstraintD(0, d[2], op_leq),
465
+ BinConstraintD(0, d[3], op_leq),
466
+ ]
467
+ )
468
+
469
+ return Conj([c1, c2, c3, c4, c5, leq_constraints]), counter
470
+
471
+
472
+ @register_transformation_rule(CalcMaxPool)
473
+ def generate_calc_maxpool(constraint, counter):
474
+ """
475
+ Transform maxpool constraints
476
+ """
477
+ d, counter = gen_tensor_dims(4, counter)
478
+ maxpool_result = TensorType([d[0], d[1], d[2], d[3]])
479
+
480
+ # the maxpool result is a tensor of size 4
481
+ c1 = BinConstraintT(constraint.maxpool_result, maxpool_result, op_eq)
482
+
483
+ # the input corresponds to the output in the first and second dimension of maxpool
484
+ c2 = BinConstraintD(constraint.matching_constraint[1], d[1], op_eq)
485
+ c3 = BinConstraintD(constraint.matching_constraint[0], d[0], op_eq)
486
+ c4, c5 = calc_last_two_dims(constraint, d)
487
+
488
+ leq_constraints = Conj(
489
+ [
490
+ BinConstraintD(0, d[0], op_leq),
491
+ BinConstraintD(0, d[1], op_leq),
492
+ BinConstraintD(0, d[2], op_leq),
493
+ BinConstraintD(0, d[3], op_leq),
494
+ ]
495
+ )
496
+
497
+ return Conj([c1, c2, c3, c4, c5, leq_constraints]), counter
498
+
499
+
500
+ @register_transformation_rule(CalcProduct)
501
+ def generate_calc_product(constraint, counter):
502
+ """
503
+ Transform flatten constraints
504
+ """
505
+ start = constraint.start
506
+ end = constraint.end
507
+ dims = constraint.dims_to_flatten
508
+ flattened = constraint.flattened
509
+ n = len(constraint.dims_to_flatten)
510
+
511
+ # this will be evaluated right here
512
+ boundary_check = 0 <= start and start < end and end <= n
513
+
514
+ c_boundary = T() if boundary_check else F()
515
+
516
+ lhs = dims[0:start]
517
+ rhs = dims[end:]
518
+ mid = dims[start:end]
519
+
520
+ all_possibilities = generate_all_int_dyn_dim_possibilities(mid)
521
+
522
+ all_constraints = []
523
+
524
+ for p in all_possibilities:
525
+ p = list(p)
526
+ # this tells us there is a dynamic variable
527
+ contains_dyn = not all(constraint.op == op_neq for constraint in p)
528
+ if contains_dyn:
529
+ mid_var = [Dyn]
530
+ total_constraints = lhs + mid_var + rhs
531
+ if len(total_constraints) > 4:
532
+ all_constraints.append(F())
533
+ else:
534
+ all_constraints.append(
535
+ Conj(
536
+ [
537
+ BinConstraintT(
538
+ flattened, TensorType(lhs + mid_var + rhs), op_eq
539
+ )
540
+ ]
541
+ + p
542
+ )
543
+ )
544
+ else:
545
+ new_var, counter = gen_dvar(counter)
546
+ mid_eq_prod = Conj(
547
+ [
548
+ BinConstraintD(new_var, Prod(mid), op_eq),
549
+ BinConstraintD(new_var, Dyn, op_neq),
550
+ ]
551
+ )
552
+ mid_var = [new_var]
553
+ total_constraints = lhs + mid_var + rhs
554
+ if len(total_constraints) > 4:
555
+ all_constraints.append(F())
556
+ else:
557
+ all_constraints.append(
558
+ Conj(
559
+ [
560
+ BinConstraintT(
561
+ flattened, TensorType(lhs + mid_var + rhs), op_eq
562
+ ),
563
+ mid_eq_prod,
564
+ ]
565
+ + p
566
+ )
567
+ )
568
+
569
+ return Conj([Disj(all_constraints), c_boundary]), counter
570
+
571
+
572
+ @register_transformation_rule(CanReshape)
573
+ def generate_reshape(constraint, counter):
574
+ """
575
+ Transform reshape constraints
576
+ """
577
+ d, counter = gen_tensor_dims(4, counter)
578
+
579
+ d1 = d[0]
580
+ d2 = d[1]
581
+ d3 = d[2]
582
+ d4 = d[3]
583
+
584
+ target = constraint.target.__args__
585
+
586
+ is_fully_static = all(d != Dyn for d in target)
587
+
588
+ # dynamic tensor
589
+ c1_dyn = BinConstraintT(constraint.src, Dyn, op_eq)
590
+ c2_tensor1 = BinConstraintT(constraint.src, TensorType([d1]), op_eq)
591
+ c2_tensor2 = BinConstraintT(constraint.src, TensorType([d1, d2]), op_eq)
592
+ c2_tensor3 = BinConstraintT(constraint.src, TensorType([d1, d2, d3]), op_eq)
593
+ c2_tensor4 = BinConstraintT(constraint.src, TensorType([d1, d2, d3, d4]), op_eq)
594
+
595
+ d1_eq_dyn = BinConstraintD(d1, Dyn, op_eq)
596
+ d1_neq_dyn = BinConstraintD(d1, Dyn, op_neq)
597
+
598
+ d2_eq_dyn = BinConstraintD(d2, Dyn, op_eq)
599
+ d2_neq_dyn = BinConstraintD(d2, Dyn, op_neq)
600
+
601
+ d3_eq_dyn = BinConstraintD(d3, Dyn, op_eq)
602
+ d3_neq_dyn = BinConstraintD(d3, Dyn, op_neq)
603
+
604
+ d4_eq_dyn = BinConstraintD(d3, Dyn, op_eq)
605
+ d4_neq_dyn = BinConstraintD(d3, Dyn, op_neq)
606
+
607
+ nat_d1 = BinConstraintD(0, d1, op_leq)
608
+ nat_d2 = BinConstraintD(0, d2, op_leq)
609
+ nat_d3 = BinConstraintD(0, d3, op_leq)
610
+ nat_d4 = BinConstraintD(0, d4, op_leq)
611
+
612
+ if is_fully_static:
613
+ # size 1 tensor
614
+ c3_tensor1 = Disj(
615
+ [d1_eq_dyn, (Conj([d1_neq_dyn, BinConstraintD(d1, Prod(target), op_eq)]))]
616
+ )
617
+ all_tensor_1 = Conj([c2_tensor1, c3_tensor1])
618
+
619
+ # size 2 tensor
620
+ all_tensor_2 = Conj(
621
+ [c2_tensor2, gen_all_reshape_possibilities([d1, d2], target)]
622
+ )
623
+
624
+ # size 3 tensor
625
+ all_tensor_3 = Conj(
626
+ [c2_tensor3, gen_all_reshape_possibilities([d1, d2, d3], target)]
627
+ )
628
+
629
+ # size 4 tensor
630
+ all_tensor_4 = Conj(
631
+ [c2_tensor4, gen_all_reshape_possibilities([d1, d2, d3, d4], target)]
632
+ )
633
+
634
+ return (
635
+ Conj(
636
+ [
637
+ Disj(
638
+ [c1_dyn, all_tensor_1, all_tensor_2, all_tensor_3, all_tensor_4]
639
+ ),
640
+ nat_d1,
641
+ nat_d2,
642
+ nat_d3,
643
+ nat_d4,
644
+ ]
645
+ ),
646
+ counter,
647
+ )
648
+
649
+ # then there must be exactly one occurrence of dyn
650
+ else:
651
+ new_target = [n for n in target if n != Dyn]
652
+
653
+ # tensor 1
654
+ c3_tensor1 = Disj(
655
+ [d1_eq_dyn, (Conj([d1_neq_dyn, is_dim_div_by_target(new_target, d1)]))]
656
+ )
657
+ all_tensor_1 = Conj([c2_tensor1, c3_tensor1])
658
+
659
+ # tensor 2
660
+ c21 = Disj([d1_eq_dyn, d2_eq_dyn])
661
+ c22 = Conj(
662
+ [d1_neq_dyn, d2_neq_dyn, is_dim_div_by_target(new_target, Prod([d1, d2]))]
663
+ )
664
+ all_tensor_2 = Conj([c2_tensor2, Disj([c21, c22])])
665
+
666
+ # tensor 3
667
+ c31 = Disj([d1_eq_dyn, d2_eq_dyn, d3_eq_dyn])
668
+ c32 = Conj(
669
+ [
670
+ d1_neq_dyn,
671
+ d2_neq_dyn,
672
+ d3_neq_dyn,
673
+ is_dim_div_by_target(new_target, Prod([d1, d2, d3])),
674
+ ]
675
+ )
676
+ all_tensor_3 = Conj([c2_tensor3, Disj([c31, c32])])
677
+
678
+ # tensor 4
679
+ c41 = Disj([d1_eq_dyn, d2_eq_dyn, d3_eq_dyn, d4_eq_dyn])
680
+ c42 = Conj(
681
+ [
682
+ d1_neq_dyn,
683
+ d2_neq_dyn,
684
+ d3_neq_dyn,
685
+ d4_neq_dyn,
686
+ is_dim_div_by_target(new_target, Prod([d1, d2, d3, d4])),
687
+ ]
688
+ )
689
+ all_tensor_4 = Conj([c2_tensor4, Disj([c41, c42])])
690
+
691
+ return (
692
+ Conj(
693
+ [
694
+ Disj(
695
+ [c1_dyn, all_tensor_1, all_tensor_2, all_tensor_3, all_tensor_4]
696
+ ),
697
+ nat_d1,
698
+ nat_d2,
699
+ nat_d3,
700
+ nat_d4,
701
+ ]
702
+ ),
703
+ counter,
704
+ )
705
+
706
+
707
+ @register_transformation_rule(ApplyBroadcasting)
708
+ def generate_broadcasting(constraint, counter):
709
+ """
710
+ Transform broadcasting constraints
711
+ """
712
+ e11, e12 = constraint.res1, constraint.res2
713
+ e1, e2 = constraint.input1, constraint.input2
714
+
715
+ e1_dyn = BinConstraintT(e1, Dyn, op_eq)
716
+ e2_dyn = BinConstraintT(e2, Dyn, op_eq)
717
+
718
+ # Introduce dimensions
719
+ e1_equal_e11 = BinConstraintT(e1, e11, op_eq)
720
+ e2_equal_e12 = BinConstraintT(e2, e12, op_eq)
721
+
722
+ # dyn possibility
723
+ e1_dyn_constraint = Conj([e1_dyn, e1_equal_e11, e2_equal_e12])
724
+ e2_dyn_constraint = Conj([e2_dyn, e1_equal_e11, e2_equal_e12])
725
+
726
+ # tensor possibility
727
+ # generate dimensions to create tensors of size 1
728
+ final_tensor_1_constraint, _, _, nat_dims_1, counter = gen_broadcasting_constraints(
729
+ e1, e2, e11, e12, 1, counter
730
+ )
731
+
732
+ # generate dimensions to create tensors of size 2
733
+ (
734
+ final_tensor_2_constraint_no_padding,
735
+ final_tensor_2_constraint_padding_arg1,
736
+ final_tensor_2_constraint_padding_arg2,
737
+ nat_dims_2,
738
+ counter,
739
+ ) = gen_broadcasting_constraints(e1, e2, e11, e12, 2, counter)
740
+
741
+ # generate dimensions to create tensors of size 3
742
+ (
743
+ final_tensor_3_constraint_no_padding,
744
+ final_tensor_3_constraint_padding_arg1,
745
+ final_tensor_3_constraint_padding_arg2,
746
+ nat_dims_3,
747
+ counter,
748
+ ) = gen_broadcasting_constraints(e1, e2, e11, e12, 3, counter)
749
+
750
+ # generate dimensions to create tensors of size 4
751
+ (
752
+ final_tensor_4_constraint_no_padding,
753
+ final_tensor_4_constraint_padding_arg1,
754
+ final_tensor_4_constraint_padding_arg2,
755
+ nat_dims_4,
756
+ counter,
757
+ ) = gen_broadcasting_constraints(e1, e2, e11, e12, 4, counter)
758
+
759
+ final_result = Disj(
760
+ [
761
+ e1_dyn_constraint,
762
+ e2_dyn_constraint,
763
+ final_tensor_1_constraint,
764
+ final_tensor_2_constraint_no_padding,
765
+ final_tensor_2_constraint_padding_arg1,
766
+ final_tensor_2_constraint_padding_arg2,
767
+ final_tensor_3_constraint_no_padding,
768
+ final_tensor_3_constraint_padding_arg1,
769
+ final_tensor_3_constraint_padding_arg2,
770
+ final_tensor_4_constraint_no_padding,
771
+ final_tensor_4_constraint_padding_arg1,
772
+ final_tensor_4_constraint_padding_arg2,
773
+ ]
774
+ )
775
+
776
+ return (
777
+ Conj([final_result, *nat_dims_1, *nat_dims_2, *nat_dims_3, *nat_dims_4]),
778
+ counter,
779
+ )
780
+
781
+
782
+ def transform_constraint(constraint: Constraint, counter: int):
783
+ """
784
+ Transforms a constraint into a simpler constraint.
785
+ Ex: precision and consistency are transformed to equality
786
+ Args:
787
+ constraint: constraint to be transformed
788
+ counter: for variable tracking
789
+
790
+ Returns: Constraint
791
+
792
+ """
793
+ if type(constraint) in _TRANSFORMATION_RULES:
794
+ return _TRANSFORMATION_RULES[type(constraint)](constraint, counter)
795
+
796
+ else:
797
+ return constraint, counter
798
+
799
+
800
+ def calc_last_two_dims(constraint, d: list[DVar]):
801
+ """
802
+ Generates constraints for the last two dimensions of a convolution or a maxpool output
803
+ Args:
804
+ constraint: CalcConv or CalcMaxPool
805
+ d: The list of output dimensions
806
+
807
+ Returns: Constraints for calculating the last two dimensions of the output
808
+
809
+ """
810
+
811
+ assert isinstance(constraint, (CalcConv, CalcMaxPool))
812
+
813
+ b3 = constraint.matching_constraint[2]
814
+ b4 = constraint.matching_constraint[3]
815
+
816
+ b3_dyn = Conj([BinConstraintD(d[2], Dyn, op_eq), BinConstraintD(b3, Dyn, op_eq)])
817
+ b4_dyn = Conj([BinConstraintD(d[3], Dyn, op_eq), BinConstraintD(b4, Dyn, op_eq)])
818
+
819
+ d3_not_dyn = Conj(
820
+ [BinConstraintD(d[2], Dyn, op_neq), BinConstraintD(b3, Dyn, op_neq)]
821
+ )
822
+ d4_not_dyn = Conj(
823
+ [BinConstraintD(d[3], Dyn, op_neq), BinConstraintD(b4, Dyn, op_neq)]
824
+ )
825
+
826
+ # transform parameters into tuples in case they are not already
827
+ padding = (
828
+ (constraint.padding, constraint.padding)
829
+ if isinstance(constraint.padding, int)
830
+ else constraint.padding
831
+ )
832
+ kernel = (
833
+ (constraint.kernel, constraint.kernel)
834
+ if isinstance(constraint.kernel, int)
835
+ else constraint.kernel
836
+ )
837
+ stride = (
838
+ (constraint.stride, constraint.stride)
839
+ if isinstance(constraint.stride, int)
840
+ else constraint.stride
841
+ )
842
+ dilation = (
843
+ (constraint.dilation, constraint.dilation)
844
+ if isinstance(constraint.dilation, int)
845
+ else constraint.dilation
846
+ )
847
+
848
+ f1 = BinConstraintD(b3, BinConstraintD(2, padding[0], op_mul), op_add)
849
+ f2 = BinConstraintD(dilation[0], BinConstraintD(kernel[0], 1, op_sub), op_mul)
850
+ f3 = BinConstraintD(
851
+ BinConstraintD(BinConstraintD(f1, f2, op_sub), 1, op_sub), stride[0], op_div
852
+ )
853
+ f4 = BinConstraintD(f3, 1, op_add)
854
+
855
+ c4 = Disj([b3_dyn, Conj([d3_not_dyn, BinConstraintD(d[2], f4, op_eq)])])
856
+
857
+ f11 = BinConstraintD(b4, BinConstraintD(2, padding[1], op_mul), op_add)
858
+ f22 = BinConstraintD(dilation[1], BinConstraintD(kernel[1], 1, op_sub), op_mul)
859
+ f33 = BinConstraintD(
860
+ BinConstraintD(BinConstraintD(f11, f22, op_sub), 1, op_sub), stride[1], op_div
861
+ )
862
+ f44 = BinConstraintD(f33, 1, op_add)
863
+
864
+ c5 = Disj([b4_dyn, Conj([d4_not_dyn, BinConstraintD(d[3], f44, op_eq)])])
865
+
866
+ return c4, c5
867
+
868
+
869
+ def generate_all_int_dyn_dim_possibilities(my_list: list[DVar]):
870
+ """
871
+ Generate all possibilities of being equal or not equal to dyn for my_list
872
+ Args:
873
+ my_list: List of tensor dimensions
874
+
875
+ Returns: A list of a list of constraints. Each list of constraints corresponds to
876
+ one possibility about the values of the dimension variables
877
+ """
878
+ # generate all possibilities of being equal or not equal to dyn for my_list
879
+ eq_possibilities = [
880
+ BinConstraintD(my_list[i], Dyn, op_eq) for i in range(len(my_list))
881
+ ]
882
+ neq_possibilities = [
883
+ BinConstraintD(my_list[i], Dyn, op_neq) for i in range(len(my_list))
884
+ ]
885
+
886
+ d_possibilities = [list(i) for i in zip(eq_possibilities, neq_possibilities)]
887
+ all_possibilities = list(itertools.product(*d_possibilities))
888
+ return all_possibilities
889
+
890
+
891
+ def is_target_div_by_dim(target: list[int], dim: list[DVar]):
892
+ """
893
+ Generate constraints to check if the target dimensions are divisible by the input dimensions
894
+ Args:
895
+ target: Target dimensions
896
+ dim: Input dimensions
897
+
898
+ Returns: Constraints to check divisibility
899
+
900
+ """
901
+ return BinConstraintD(BinConstraintD(Prod(target), dim, op_mod), 0, op_eq)
902
+
903
+
904
+ def is_dim_div_by_target(target: list[int], dim: list[DVar]):
905
+ """
906
+ Generate constraints to check if the input dimensions is divisible by the target dimensions
907
+ Args:
908
+ target: Target dimensions
909
+ dim: Input dimensions
910
+
911
+ Returns: Constraints to check divisibility
912
+
913
+ """
914
+ return BinConstraintD(BinConstraintD(dim, Prod(target), op_mod), 0, op_eq)
915
+
916
+
917
+ def gen_all_reshape_possibilities(list_of_dims, target):
918
+ """
919
+ Consider all possibilities what the input dimensions could be (number or dynamic)
920
+ Then generate the appropriate constraints using multiplication or mod depending on the possibility
921
+ The possibilities we consider here are the cross product of being equal to dyn or not equal to dyn
922
+ for the input. Target is fixed because at most one dimension could be dyn.
923
+ We have different cases for this.
924
+
925
+ Args:
926
+ list_of_dims: The input list of dimensions
927
+ target: The tensor we want to reshape to
928
+
929
+ Returns: A disjunction of transformed reshape constraints
930
+
931
+ """
932
+ all_possibilities = generate_all_int_dyn_dim_possibilities(list_of_dims)
933
+
934
+ all_constraints = []
935
+
936
+ for p in all_possibilities:
937
+ to_multiply = []
938
+
939
+ p = list(p)
940
+
941
+ for constraint in p:
942
+ assert isinstance(constraint, BinConstraintD)
943
+ if constraint.op == op_neq:
944
+ to_multiply.append(constraint.lhs)
945
+
946
+ if not to_multiply:
947
+ all_constraints.append(Conj(p))
948
+
949
+ elif len(to_multiply) < len(list_of_dims):
950
+ all_constraints.append(
951
+ Conj(p + [is_target_div_by_dim(target, Prod(to_multiply))])
952
+ )
953
+ else:
954
+ all_constraints.append(
955
+ Conj(p + [BinConstraintD(Prod(list_of_dims), Prod(target), op_eq)])
956
+ )
957
+
958
+ return Disj(all_constraints)
959
+
960
+
961
+ def broadcast_dim(tensor_input1, tensor_input2, res1, res2, index, padding=False):
962
+ """
963
+ Apply broadcasting to the 'index' dimension of tensor_input1.
964
+ Args:
965
+ tensor_input1: should represent [d1, ..., d_index, ...] where d_index = 1
966
+ tensor_input2: represents the second input
967
+ res1: broadcasted result 1
968
+ res2: broadcasted result 2
969
+ index: the index to broadcast
970
+ padding: If padding was used, then tensor_input1[index] does not exist
971
+
972
+ Returns:
973
+
974
+ """
975
+ if tensor_input1[index] is None:
976
+ assert padding
977
+
978
+ if not padding:
979
+ # then the inputs are the same length so they all have dimensions at "index"
980
+ return Conj(
981
+ [
982
+ BinConstraintD(tensor_input1[index], 1, op_eq),
983
+ BinConstraintD(res1[index], res2[index], op_eq),
984
+ BinConstraintD(res2[index], tensor_input2[index], op_eq),
985
+ ]
986
+ )
987
+
988
+ else:
989
+ # we don't set the input dimension to 1, since it doesn't exist.
990
+ return Conj(
991
+ [
992
+ BinConstraintD(res1[index], res2[index], op_eq),
993
+ BinConstraintD(res2[index], tensor_input2[index], op_eq),
994
+ ]
995
+ )
996
+
997
+
998
+ def apply_padding(
999
+ e1_var: TVar,
1000
+ e11: BinConstraintT,
1001
+ e2: BinConstraintT,
1002
+ e12: BinConstraintT,
1003
+ d2: list[DVar],
1004
+ d11: list[DVar],
1005
+ d12: list[DVar],
1006
+ counter: int,
1007
+ ):
1008
+ """
1009
+ We are considering the possibility where one input has less dimensions than
1010
+ another input, so we apply padding to the broadcasted results
1011
+
1012
+ Args:
1013
+ e1_var: Variable representing the first input where padding will be
1014
+ e11: constraint of the form e11 = Tensortype[d1, ..., dn]
1015
+ e2: constraint of the form e2 = Tensortype[d1, ..., dn]
1016
+ e12: constraint of the form e11 = Tensortype[d1, ..., dn]
1017
+ d2: Tensor variables for the second input
1018
+ d11: Tensor variables for the broadcasted first input
1019
+ d12: Tensor variables for the broadcasted second input
1020
+ counter: variable tracking
1021
+
1022
+ Returns: A new constraint whose goal is to apply padding to the broadcasted result
1023
+
1024
+ """
1025
+
1026
+ res = []
1027
+
1028
+ # pad the shorter input with None so we can pass it to the broadcasting helper function
1029
+ for i in range(1, len(d2)):
1030
+ d1, counter = gen_tensor_dims(i, counter)
1031
+
1032
+ nat_constraints = gen_nat_constraints(d1 + d2 + d11 + d12)
1033
+
1034
+ e1 = BinConstraintT(e1_var, TensorType(d1), op_eq)
1035
+
1036
+ simulate_padding = [None] * (len(d2) - i)
1037
+
1038
+ assert len(simulate_padding + d1) == len(d2)
1039
+
1040
+ # for every padding size, we also consider broadcasting
1041
+ broadcast_padding = [
1042
+ broadcast_dim(simulate_padding, d2, d11, d12, j, True)
1043
+ for j in range(len(d2) - i)
1044
+ ]
1045
+
1046
+ # we consider the possibilities for broadcasting for every dimension. Since we already
1047
+ # padded d1, we do not consider it while broadcasting
1048
+ all_broadcasting_possibilities = (
1049
+ generate_all_broadcasting_possibilities_no_padding(
1050
+ d1, d2[(len(d2) - i) :], d11[(len(d2) - i) :], d12[(len(d2) - i) :]
1051
+ )
1052
+ )
1053
+ # combine all constraints into a conjunction
1054
+ c = Conj(
1055
+ [
1056
+ e1,
1057
+ e11,
1058
+ e2,
1059
+ e12,
1060
+ *broadcast_padding,
1061
+ all_broadcasting_possibilities,
1062
+ *nat_constraints,
1063
+ ]
1064
+ )
1065
+ res.append(c)
1066
+
1067
+ return Disj(res), counter
1068
+
1069
+
1070
+ def no_broadcast_dim_with_index(
1071
+ d1: list[DVar], d2: list[DVar], d3: list[DVar], d4: list[DVar], i: int
1072
+ ):
1073
+ """
1074
+ Args:
1075
+ d1: input 1
1076
+ d2: input 2
1077
+ d3: simulated broadcasting for input 1
1078
+ d4: simulated broadcasting for input 2
1079
+ i: the rank of the resulting tensor addition
1080
+
1081
+ Returns: Constraints for when no broadcasting occurs
1082
+ """
1083
+ return Conj(
1084
+ [
1085
+ Disj(
1086
+ [
1087
+ Conj(
1088
+ [
1089
+ BinConstraintD(d1[i], 1, op_eq),
1090
+ BinConstraintD(d2[i], 1, op_eq),
1091
+ ]
1092
+ ),
1093
+ Conj(
1094
+ [
1095
+ BinConstraintD(d1[i], 1, op_neq),
1096
+ BinConstraintD(d2[i], 1, op_neq),
1097
+ ]
1098
+ ),
1099
+ ]
1100
+ ),
1101
+ BinConstraintD(d1[i], d3[i], op_eq),
1102
+ BinConstraintD(d2[i], d4[i], op_eq),
1103
+ ]
1104
+ )
1105
+
1106
+
1107
+ def gen_lists_of_dims(num_tensors: int, dim_size: int, counter: int):
1108
+ """
1109
+ Generate lists of DVar to represent tensor dimensions
1110
+ Args:
1111
+ num_tensors: the required number of tensors
1112
+ dim_size: the number of dimensions for each tensor
1113
+ counter: variable tracking
1114
+
1115
+ Returns: A list of a list of tensor dimensions
1116
+
1117
+ """
1118
+ res = []
1119
+
1120
+ for _ in range(num_tensors):
1121
+ dims, counter = gen_tensor_dims(dim_size, counter)
1122
+ res.append(dims)
1123
+
1124
+ return res, counter
1125
+
1126
+
1127
+ def create_equality_constraints_for_broadcasting(
1128
+ e1: TVar,
1129
+ e2: TVar,
1130
+ e11: TVar,
1131
+ e12: TVar,
1132
+ d1: list[DVar],
1133
+ d2: list[DVar],
1134
+ d11: list[DVar],
1135
+ d12: list[DVar],
1136
+ ):
1137
+ """
1138
+ Create equality constraints for when no broadcasting occurs
1139
+ Args:
1140
+ e1: Input 1
1141
+ e2: Input 2
1142
+ e11: Broadcasted input 1
1143
+ e12: Broadcasted input 2
1144
+ d1: Variables that store dimensions for e1
1145
+ d2: Variables that store dimensions for e2
1146
+ d11: Variables that store dimensions for e11
1147
+ d12: Variables that store dimensions for e22
1148
+
1149
+ Returns: Four equality constraints
1150
+
1151
+ """
1152
+
1153
+ e1_tensor = BinConstraintT(e1, TensorType(d1), op_eq)
1154
+ e11_tensor = BinConstraintT(e11, TensorType(d11), op_eq)
1155
+ e2_tensor = BinConstraintT(e2, TensorType(d2), op_eq)
1156
+ e12_tensor = BinConstraintT(e12, TensorType(d12), op_eq)
1157
+ return [e1_tensor, e11_tensor, e2_tensor, e12_tensor]
1158
+
1159
+
1160
+ def gen_consistency_constraints(constraint: Constraint, counter: int):
1161
+ """
1162
+ Args:
1163
+ constraint: Consistency constraint on tensors
1164
+ counter: for variable tracking
1165
+
1166
+ Returns: Equality and consistency constraints on dimensions
1167
+
1168
+ """
1169
+
1170
+ all_constraints = []
1171
+
1172
+ for i in range(1, MAX_TENSOR_RANK + 1):
1173
+ new_dims_rhs_1, counter = gen_tensor_dims(i, counter)
1174
+ new_dims_rhs_2, counter = gen_tensor_dims(i, counter)
1175
+
1176
+ nat_constraints = gen_nat_constraints(new_dims_rhs_1 + new_dims_rhs_2)
1177
+
1178
+ c_tensor_i = Conj(
1179
+ [
1180
+ BinConstraintT(constraint.lhs, TensorType(new_dims_rhs_1), op_eq),
1181
+ BinConstraintT(constraint.rhs, TensorType(new_dims_rhs_2), op_eq),
1182
+ ]
1183
+ + [
1184
+ BinConstraintD(d1, d2, op_consistency)
1185
+ for d1, d2 in zip(new_dims_rhs_1, new_dims_rhs_2)
1186
+ ]
1187
+ + nat_constraints
1188
+ )
1189
+
1190
+ all_constraints.append(c_tensor_i)
1191
+
1192
+ return all_constraints, counter
1193
+
1194
+
1195
+ def gen_greatest_upper_bound(constraint: TGreatestUpperBound, counter: int):
1196
+ """
1197
+ Args:
1198
+ constraint: Greatest upper bound on tensors
1199
+ counter: variable tracking
1200
+
1201
+ Returns: A set of equality constraints and DGreatestUpperBound constraints
1202
+
1203
+ """
1204
+
1205
+ all_constraints = []
1206
+
1207
+ for i in range(1, MAX_TENSOR_RANK + 1):
1208
+ c = []
1209
+ dims1, counter = gen_tensor_dims(i, counter)
1210
+ c1tensor = TensorType(dims1)
1211
+
1212
+ dims2, counter = gen_tensor_dims(i, counter)
1213
+ c2tensor = TensorType(dims2)
1214
+
1215
+ dims3, counter = gen_tensor_dims(i, counter)
1216
+ c3tensor = TensorType(dims3)
1217
+
1218
+ c += [
1219
+ BinConstraintT(constraint.rhs1, c1tensor, op_eq),
1220
+ BinConstraintT(constraint.rhs2, c2tensor, op_eq),
1221
+ BinConstraintT(constraint.res, c3tensor, op_eq),
1222
+ ] + gen_nat_constraints(dims1 + dims2 + dims3)
1223
+
1224
+ assert (
1225
+ len(c3tensor.__args__) == len(c1tensor.__args__) == len(c2tensor.__args__)
1226
+ )
1227
+ for i in range(len(c3tensor.__args__)):
1228
+ c.append(
1229
+ DGreatestUpperBound(
1230
+ c3tensor.__args__[i], c1tensor.__args__[i], c2tensor.__args__[i]
1231
+ )
1232
+ )
1233
+
1234
+ all_constraints.append(Conj(c))
1235
+ return all_constraints, counter
1236
+
1237
+
1238
+ def generate_all_broadcasting_possibilities_no_padding(
1239
+ d1: list[DVar], d2: list[DVar], d11: list[DVar], d12: list[DVar]
1240
+ ):
1241
+ """
1242
+ Generate broadcasting constraints assuming no padding. Broadcasting can happen at any dimension.
1243
+ We look at all combinations for all dimensions in d1 and d2
1244
+ Args:
1245
+ d1: input1 dimensions
1246
+ d2: input2 dimensions
1247
+ d11: broadcasted input1 dimensions
1248
+ d12: broadcasted input2 dimensions
1249
+
1250
+ Returns: broadcasting constraints relating the input dimensions to the broadcasted dimensions
1251
+
1252
+ """
1253
+
1254
+ size = len(d1)
1255
+
1256
+ res2 = []
1257
+
1258
+ for i in range(size):
1259
+ t1 = broadcast_dim(d1, d2, d11, d12, i)
1260
+ t2 = broadcast_dim(d2, d1, d12, d11, i)
1261
+ t3 = no_broadcast_dim_with_index(d1, d2, d11, d12, i)
1262
+
1263
+ res2.append(Disj([t1, t2, t3]))
1264
+
1265
+ return Conj(res2)
1266
+
1267
+
1268
+ def gen_broadcasting_constraints(
1269
+ e1: TVar, e2: TVar, e11: TVar, e12: TVar, i: int, counter: int
1270
+ ):
1271
+ """
1272
+ Simulates broadcasting on e1 and e2 and returns the results
1273
+ respectively in e11 and e12. Because of gradual types,
1274
+ e1 and e2 may not be equal. Similarly, e11 and e12 may not
1275
+ be equal. e11 and e12 should be guaranteed to be consistent
1276
+ as they represent the shapes of the tensors to be added after
1277
+ broadcasting.
1278
+ Args:
1279
+ e1: TVar representing the type of input 1
1280
+ e2: TVar representing the type of input 2
1281
+ e11: TVar representing the representing broadcasted input 1
1282
+ e12: TVar representing the representing broadcasted input 2
1283
+ i: The rank of the resulting type of addition
1284
+ counter: for variable tracking
1285
+
1286
+ Returns: Simplified broadcasting constraints
1287
+
1288
+ """
1289
+ dims, counter = gen_lists_of_dims(4, i, counter)
1290
+ [d1, d2, d3, d4] = dims
1291
+ nat_dims_i = gen_nat_constraints(list(itertools.chain.from_iterable(dims)))
1292
+
1293
+ initialize_tensors_constraints = create_equality_constraints_for_broadcasting(
1294
+ e1, e2, e11, e12, d1, d2, d3, d4
1295
+ )
1296
+
1297
+ [e1_tensor, e11_tensor, e2_tensor, e12_tensor] = initialize_tensors_constraints
1298
+
1299
+ # without padding, broadcast all possibilities for tensors of size i
1300
+ final_tensor_constraint_no_padding = Conj(
1301
+ [
1302
+ *initialize_tensors_constraints,
1303
+ generate_all_broadcasting_possibilities_no_padding(d1, d2, d3, d4),
1304
+ ]
1305
+ )
1306
+
1307
+ # with padding, broadcast all possibilities for tensors of size i
1308
+ final_tensor_constraint_padding_arg1, counter = apply_padding(
1309
+ e1, e11_tensor, e2_tensor, e12_tensor, d2, d3, d4, counter
1310
+ )
1311
+
1312
+ final_tensor_constraint_padding_arg2, counter = apply_padding(
1313
+ e2, e12_tensor, e1_tensor, e11_tensor, d1, d4, d3, counter
1314
+ )
1315
+
1316
+ return (
1317
+ final_tensor_constraint_no_padding,
1318
+ final_tensor_constraint_padding_arg1,
1319
+ final_tensor_constraint_padding_arg2,
1320
+ nat_dims_i,
1321
+ counter,
1322
+ )
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/operation.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ op_add = "+"
2
+ op_sub = "-"
3
+ op_mul = "*"
4
+ op_div = "/"
5
+ op_eq = "="
6
+ op_neq = "!="
7
+ op_imp = "=>"
8
+ op_matching = "\u22b3" # (contains)
9
+ op_consistency = "~"
10
+ op_precision = "\u2291" # (square image of or equal to)
11
+ op_leq = "\u2264" # less-than or equal to
12
+ op_lt = "<"
13
+ op_gt = ">"
14
+ op_mod = "%"
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/transform_to_z3.py ADDED
@@ -0,0 +1,446 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # mypy: allow-untyped-defs
2
+ from torch.fx.experimental.migrate_gradual_types.constraint import (
3
+ BinConstraintD,
4
+ BinConstraintT,
5
+ BVar,
6
+ Conj,
7
+ Disj,
8
+ DVar,
9
+ F,
10
+ is_algebraic_expression,
11
+ is_bool_expr,
12
+ is_dim,
13
+ Prod,
14
+ T,
15
+ TVar,
16
+ )
17
+ from torch.fx.experimental.migrate_gradual_types.constraint_generator import (
18
+ ConstraintGenerator,
19
+ )
20
+ from torch.fx.experimental.migrate_gradual_types.constraint_transformation import (
21
+ transform_constraint,
22
+ )
23
+ from torch.fx.experimental.migrate_gradual_types.operation import (
24
+ op_add,
25
+ op_div,
26
+ op_eq,
27
+ op_gt,
28
+ op_leq,
29
+ op_lt,
30
+ op_mod,
31
+ op_mul,
32
+ op_neq,
33
+ op_sub,
34
+ )
35
+ from torch.fx.tensor_type import Dyn, TensorType
36
+
37
+
38
+ try:
39
+ import z3 # type: ignore[import]
40
+
41
+ from torch.fx.experimental.migrate_gradual_types.z3_types import (
42
+ D,
43
+ tensor_type,
44
+ z3_dyn,
45
+ )
46
+
47
+ HAS_Z3 = True
48
+
49
+ def transform_to_z3(constraint, counter, dimension_dict):
50
+ if isinstance(constraint, Conj):
51
+ conjuncts = []
52
+ for c in constraint.conjucts:
53
+ new_c, counter = transform_to_z3(c, counter, dimension_dict)
54
+ conjuncts.append(new_c)
55
+ return z3.And(conjuncts), counter
56
+
57
+ elif isinstance(constraint, Disj):
58
+ disjuncts = []
59
+ for c in constraint.disjuncts:
60
+ new_c, counter = transform_to_z3(c, counter, dimension_dict)
61
+ disjuncts.append(new_c)
62
+ return z3.Or(disjuncts), counter
63
+
64
+ elif isinstance(constraint, T):
65
+ return True, counter
66
+
67
+ elif isinstance(constraint, F):
68
+ return False, counter
69
+
70
+ elif isinstance(constraint, BinConstraintT):
71
+ if constraint.op == op_eq:
72
+ lhs, counter = transform_var(constraint.lhs, counter, dimension_dict)
73
+ rhs, counter = transform_var(constraint.rhs, counter, dimension_dict)
74
+ return (lhs == rhs), counter
75
+
76
+ else:
77
+ raise NotImplementedError("Method not yet implemented")
78
+
79
+ elif isinstance(constraint, BinConstraintD):
80
+ if constraint.op == op_eq:
81
+ if isinstance(constraint.lhs, BVar) and is_bool_expr(constraint.rhs):
82
+ transformed_rhs, counter = transform_to_z3(
83
+ constraint.rhs, counter, dimension_dict
84
+ )
85
+ transformed_lhs = z3.Bool(constraint.lhs.c)
86
+ return transformed_lhs == transformed_rhs, counter
87
+
88
+ elif is_dim(constraint.lhs) and is_dim(constraint.rhs):
89
+ # with dimension transformations we consider the encoding
90
+ lhs, counter = transform_dimension(
91
+ constraint.lhs, counter, dimension_dict
92
+ )
93
+ rhs, counter = transform_dimension(
94
+ constraint.rhs, counter, dimension_dict
95
+ )
96
+ return lhs == rhs, counter
97
+
98
+ else:
99
+ # then we have an algebraic expression which means that we disregard the
100
+ # first element of the encoding
101
+ lhs, counter = transform_algebraic_expression(
102
+ constraint.lhs, counter, dimension_dict
103
+ )
104
+ rhs, counter = transform_algebraic_expression(
105
+ constraint.rhs, counter, dimension_dict
106
+ )
107
+ return lhs == rhs, counter
108
+
109
+ # The assumption here is that the LHS and RHS must be dimensions
110
+ elif constraint.op == op_neq:
111
+ assert is_dim(constraint.lhs)
112
+ assert is_dim(constraint.rhs)
113
+ lhs, counter = transform_dimension(
114
+ constraint.lhs, counter, dimension_dict
115
+ )
116
+ rhs, counter = transform_dimension(
117
+ constraint.rhs, counter, dimension_dict
118
+ )
119
+ if constraint.rhs == Dyn or constraint.lhs == Dyn:
120
+ if constraint.rhs == Dyn:
121
+ return lhs.arg(0) == 1, counter
122
+ elif constraint.lhs == Dyn:
123
+ return rhs.arg(0) == 1, counter
124
+
125
+ # if one of the instances is a number
126
+ elif isinstance(constraint.lhs, int) or isinstance(constraint.rhs, int):
127
+ if isinstance(constraint.lhs, int):
128
+ return (
129
+ z3.Or(
130
+ [
131
+ rhs.arg(0) == 0,
132
+ z3.And([rhs.arg(0) == 1, lhs.arg(1) != rhs.arg(1)]),
133
+ ]
134
+ ),
135
+ counter,
136
+ )
137
+
138
+ elif isinstance(constraint.rhs, int):
139
+ return (
140
+ z3.Or(
141
+ [
142
+ lhs.arg(0) == 0,
143
+ z3.And([lhs.arg(0) == 1, lhs.arg(1) != rhs.arg(1)]),
144
+ ]
145
+ ),
146
+ counter,
147
+ )
148
+
149
+ else:
150
+ return (
151
+ z3.Or(
152
+ [
153
+ z3.And([lhs.arg(0) == 0, rhs.arg(0) != 0]),
154
+ z3.And([lhs.arg(0) != 0, rhs.arg(0) == 0]),
155
+ z3.And(
156
+ [
157
+ lhs.arg(0) != 0,
158
+ rhs.arg(0) != 0,
159
+ lhs.arg(1) != rhs.arg(1),
160
+ ]
161
+ ),
162
+ ]
163
+ ),
164
+ counter,
165
+ )
166
+
167
+ elif constraint.op == op_leq:
168
+ # if the dimensions are not dyn, this will come into effect
169
+ # there would have been another constraint specifying if a given dimension
170
+ # is dyn or not
171
+ assert is_dim(constraint.lhs) and is_dim(constraint.rhs)
172
+ lhs, counter = transform_algebraic_expression(
173
+ constraint.lhs, counter, dimension_dict
174
+ )
175
+ rhs, counter = transform_algebraic_expression(
176
+ constraint.rhs, counter, dimension_dict
177
+ )
178
+ return lhs <= rhs, counter
179
+
180
+ elif constraint.op == op_gt:
181
+ assert is_dim(constraint.lhs) and is_dim(constraint.rhs)
182
+ lhs, counter = transform_algebraic_expression(
183
+ constraint.lhs, counter, dimension_dict
184
+ )
185
+ rhs, counter = transform_algebraic_expression(
186
+ constraint.rhs, counter, dimension_dict
187
+ )
188
+ return lhs > rhs, counter
189
+
190
+ elif constraint.op == op_lt:
191
+ assert is_dim(constraint.lhs) and is_dim(constraint.rhs)
192
+ lhs, counter = transform_algebraic_expression(
193
+ constraint.lhs, counter, dimension_dict
194
+ )
195
+ rhs, counter = transform_algebraic_expression(
196
+ constraint.rhs, counter, dimension_dict
197
+ )
198
+ return lhs < rhs, counter
199
+
200
+ else:
201
+ raise NotImplementedError("operation not yet implemented")
202
+
203
+ else:
204
+ raise NotImplementedError("Operation not yet implemented")
205
+
206
+ def transform_var(tensor, counter, dimension_dict):
207
+ """
208
+ Transforms tensor variables to a format understood by z3
209
+ Args:
210
+ tensor: Tensor variable or a tensor type potentially with variable dimensions
211
+ Returns: Transformed variable to a z3 format
212
+
213
+ """
214
+ if isinstance(tensor, TensorType):
215
+ res = []
216
+ for t in tensor.__args__:
217
+ transformed, counter = transform_dimension(t, counter, dimension_dict)
218
+ res.append(transformed)
219
+
220
+ assert len(res) <= 4
221
+ if len(tensor.__args__) == 1:
222
+ return tensor_type.tensor1(res[0]), counter
223
+ elif len(tensor.__args__) == 2:
224
+ return tensor_type.tensor2(res[0], res[1]), counter
225
+ elif len(tensor.__args__) == 3:
226
+ return tensor_type.tensor3(res[0], res[1], res[2]), counter
227
+ elif len(tensor.__args__) == 4:
228
+ return tensor_type.tensor4(res[0], res[1], res[2], res[3]), counter
229
+
230
+ elif tensor == Dyn:
231
+ return z3_dyn, counter
232
+
233
+ elif isinstance(tensor, TVar):
234
+ return z3.Const(tensor.tvar, tensor_type), counter
235
+
236
+ def transform_dimension(dimension, counter, dimension_dict):
237
+ """
238
+ Takes a dimension variable or a number and transforms it to a tuple
239
+ according to our scheme
240
+ Args:
241
+ dimension: The dimension to be transformed
242
+ counter: variable tracking
243
+
244
+ Returns: tuple and the current counter
245
+
246
+ """
247
+ if dimension == Dyn:
248
+ counter += 1
249
+ return D(0, z3.Int(counter)), counter
250
+ elif isinstance(dimension, int):
251
+ return D(1, dimension), counter
252
+ elif isinstance(dimension, DVar):
253
+ if dimension.c in dimension_dict:
254
+ return (
255
+ D(z3.Int(dimension_dict[dimension.c]), z3.Int(dimension.c)),
256
+ counter,
257
+ )
258
+ else:
259
+ counter += 1
260
+ dimension_dict[dimension.c] = counter
261
+ return D(z3.Int(counter), z3.Int(dimension.c)), counter
262
+
263
+ def transform_algebraic_expression(expr, counter, dimension_dict):
264
+ """
265
+ Transforms an algebraic expression to z3 format
266
+ Args:
267
+ expr: An expression is either a dimension variable or an algebraic-expression
268
+
269
+
270
+ Returns: the transformed expression
271
+
272
+ """
273
+ assert is_algebraic_expression(expr) or is_dim(expr)
274
+
275
+ if is_dim(expr):
276
+ transformed, counter = transform_dimension(expr, counter, dimension_dict)
277
+ return transformed.arg(1), counter
278
+
279
+ elif isinstance(expr, Prod):
280
+ dims = []
281
+ for dim in expr.products:
282
+ assert is_dim(dim)
283
+ d, counter = transform_dimension(dim, counter, dimension_dict)
284
+ dims.append(d.arg(1))
285
+ return z3.Product(dims), counter
286
+
287
+ elif is_algebraic_expression(expr):
288
+ lhs, counter = transform_algebraic_expression(
289
+ expr.lhs, counter, dimension_dict
290
+ )
291
+ rhs, counter = transform_algebraic_expression(
292
+ expr.rhs, counter, dimension_dict
293
+ )
294
+
295
+ if expr.op == op_sub:
296
+ c = lhs - rhs
297
+
298
+ elif expr.op == op_add:
299
+ c = lhs + rhs
300
+
301
+ elif expr.op == op_div:
302
+ c = lhs / rhs
303
+
304
+ elif expr.op == op_mul:
305
+ c = lhs * rhs
306
+
307
+ elif expr.op == op_mod:
308
+ c = lhs % rhs
309
+
310
+ else:
311
+ raise NotImplementedError("operation not yet implemented")
312
+
313
+ return c, counter
314
+
315
+ else:
316
+ raise RuntimeError
317
+
318
+ def transform_all_constraints(traced, counter=0):
319
+ """
320
+ Given a trace, generates constraints and transforms them to z3 format
321
+
322
+ """
323
+ dimension_dict = {} # type: ignore[var-annotated]
324
+
325
+ generator = ConstraintGenerator(traced)
326
+ new_constraints, counter = generator.generate_constraints(counter)
327
+
328
+ # print(new_constraints.conjucts[0])
329
+ # print(*new_constraints.conjucts, sep='\n')
330
+
331
+ # transform precision, matching, consistency till obtaining a fixed point
332
+ new_constraints, counter = iterate_till_fixed_point(new_constraints, counter)
333
+ # print(new_constraints)
334
+ # print(new_constraints.conjucts)
335
+ # new_constraints.conjucts = new_constraints.conjucts[:-1]
336
+ # print(*new_constraints.conjucts, sep='\n')
337
+
338
+ transformed, counter = transform_to_z3(new_constraints, counter, dimension_dict)
339
+ # print(transformed)
340
+ return transformed
341
+
342
+ def iterate_till_fixed_point(constraints, counter):
343
+ """
344
+ Transform constraints till reaching a fixed point
345
+ """
346
+ old_c = None
347
+ while old_c != constraints:
348
+ old_c = constraints
349
+ constraints, counter = transform_constraint(constraints, counter)
350
+ return constraints, counter
351
+
352
+ def transform_all_constraints_trace_time(tracer_root, graph, node, counter=0):
353
+ """
354
+ Takes a node and a graph and generates two sets of constraints.
355
+ One set constraints the node's constraints and another set
356
+ constraints the negation of the node's constraints
357
+ Args:
358
+ tracer_root: the root for getting the module instances
359
+ graph: the graph so far in the tracing process
360
+ node: node that represents a conditional
361
+ counter: variable tracking
362
+
363
+ Returns: Two sets of constraints. One with a conjunction with the
364
+ the conditional constraint and the other with a conjunction with
365
+ its negation.
366
+
367
+ """
368
+ dimension_dict = {} # type: ignore[var-annotated]
369
+
370
+ generator = ConstraintGenerator(tracer_root, graph)
371
+ new_constraints, counter = generator.generate_constraints(counter)
372
+
373
+ condition_constraint = new_constraints.conjucts[-1]
374
+
375
+ # we know the constraint is a conjunction where the last constraint is about the conditional
376
+ # so remove the last constraint
377
+ new_constraints.conjucts = new_constraints.conjucts[:-1]
378
+
379
+ # transform precision, matching, consistency till obtaining a fixed point
380
+ new_constraints, counter = iterate_till_fixed_point(new_constraints, counter)
381
+
382
+ # since the function returns a list of one element, we get the first element
383
+ # we are only interested in the RHS in this case because the LHS just stores
384
+ # the result
385
+
386
+ # we make sure the constraint is of the form:
387
+ # c = b where b is a boolean expression
388
+ # and we consider b (constraint.rhs) for transformation
389
+ assert isinstance(condition_constraint.lhs, BVar)
390
+ assert is_bool_expr(condition_constraint.rhs)
391
+ condition_constraint_rhs = condition_constraint.rhs
392
+
393
+ # transform the condition constraint
394
+ condition_constraint_rhs, counter = iterate_till_fixed_point(
395
+ condition_constraint_rhs, counter
396
+ )
397
+
398
+ transformed, counter = transform_to_z3(new_constraints, counter, dimension_dict)
399
+
400
+ transformed_condition_constraint, counter = transform_to_z3(
401
+ condition_constraint_rhs, counter, dimension_dict
402
+ )
403
+
404
+ negation_transformed_condition_constraint = z3.Not(
405
+ transformed_condition_constraint
406
+ )
407
+
408
+ return z3.And([transformed, transformed_condition_constraint]), z3.And(
409
+ [transformed, negation_transformed_condition_constraint]
410
+ )
411
+
412
+ def evaluate_conditional_with_constraints(
413
+ tracer_root, graph, node, counter=0, user_constraints=None
414
+ ):
415
+ """
416
+ Given an IR and a node representing a conditional, evaluate the conditional
417
+ and its negation
418
+ Args:
419
+ tracer_root: Tracer root for module instances
420
+ node: The node to be evaluated
421
+
422
+ Returns: the results of evaluating the condition and the negation with
423
+ the rest of the constraints
424
+
425
+ """
426
+
427
+ (
428
+ transformed_positive,
429
+ transformed_negative,
430
+ ) = transform_all_constraints_trace_time(tracer_root, graph, node, counter)
431
+
432
+ s = z3.Solver()
433
+ s.add(transformed_positive)
434
+ if user_constraints is not None:
435
+ s.add(user_constraints)
436
+ condition = s.check()
437
+
438
+ s = z3.Solver()
439
+ s.add(transformed_negative)
440
+ if user_constraints is not None:
441
+ s.add(user_constraints)
442
+ negation = s.check()
443
+ return condition, negation
444
+
445
+ except ImportError:
446
+ HAS_Z3 = False
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/util.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch.fx.experimental.migrate_gradual_types.constraint import (
2
+ BinConstraintD,
3
+ BVar,
4
+ DVar,
5
+ TVar,
6
+ )
7
+ from torch.fx.experimental.migrate_gradual_types.operation import op_leq
8
+
9
+
10
+ def gen_tvar(curr: int) -> tuple[TVar, int]:
11
+ """
12
+ Generate a tensor variable
13
+ :param curr: The current counter
14
+ :return: a tensor variable and the updated counter
15
+ """
16
+ curr += 1
17
+ return TVar(curr), curr
18
+
19
+
20
+ def gen_dvar(curr: int) -> tuple[DVar, int]:
21
+ """
22
+ Generate a dimension variable
23
+ :param curr: the current counter
24
+ :return: a dimension variable and an updated counter
25
+ """
26
+ curr += 1
27
+ return DVar(curr), curr
28
+
29
+
30
+ def gen_bvar(curr: int) -> tuple[BVar, int]:
31
+ """
32
+ Generate a boolean variable
33
+ :param curr: the current counter
34
+ :return: a boolean variable and an updated counter
35
+ """
36
+ curr += 1
37
+ return BVar(curr), curr
38
+
39
+
40
+ def gen_tensor_dims(n: int, curr: int) -> tuple[list[DVar], int]:
41
+ """
42
+ Generate a list of tensor dimensions
43
+ :param n: the number of dimensions
44
+ :param curr: the current counter
45
+ :return: a list of dimension variables and an updated counter
46
+ """
47
+ dims = []
48
+ for _ in range(n):
49
+ dvar, curr = gen_dvar(curr)
50
+ dims.append(dvar)
51
+ return dims, curr
52
+
53
+
54
+ def gen_nat_constraints(list_of_dims: list[DVar]) -> list[BinConstraintD]:
55
+ """
56
+ Generate natural number constraints for dimensions
57
+ """
58
+ return [BinConstraintD(0, d, op_leq) for d in list_of_dims]
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/migrate_gradual_types/z3_types.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ try:
2
+ import z3 # type: ignore[import]
3
+
4
+ HAS_Z3 = True
5
+ # dynamic type
6
+ dyn = z3.DeclareSort("Dyn")
7
+ dyn_type = z3.Const("dyn", dyn)
8
+
9
+ # dimension
10
+ dim = z3.Datatype("dim")
11
+ dim.declare("dim", ("0", z3.IntSort()), ("1", z3.IntSort()))
12
+ dim = dim.create()
13
+
14
+ # tensors
15
+ tensor_type = z3.Datatype("TensorType")
16
+ tensor_type.declare("Dyn", ("dyn", dyn))
17
+ tensor_type.declare("tensor1", ("0", dim))
18
+ tensor_type.declare("tensor2", ("0", dim), ("1", dim))
19
+ tensor_type.declare("tensor3", ("0", dim), ("1", dim), ("2", dim))
20
+ tensor_type.declare("tensor4", ("0", dim), ("1", dim), ("2", dim), ("3", dim))
21
+ tensor_type = tensor_type.create()
22
+
23
+ # create dimension
24
+ D = dim.dim
25
+
26
+ z3_dyn = tensor_type.Dyn(dyn_type)
27
+
28
+
29
+ except ImportError:
30
+ HAS_Z3 = False
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # mypy: disable-error-code=attr-defined
2
+ from .core import reify, unify # noqa: F403
3
+ from .more import unifiable # noqa: F403
4
+ from .variable import isvar, Var, var, variables, vars # noqa: F403
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (419 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/core.cpython-312.pyc ADDED
Binary file (3.47 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/dispatch.cpython-312.pyc ADDED
Binary file (393 Bytes). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/match.cpython-312.pyc ADDED
Binary file (5.75 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/more.cpython-312.pyc ADDED
Binary file (4.73 kB). View file
 
TTS/Kokoro/.venv/Lib/site-packages/torch/fx/experimental/unification/__pycache__/unification_tools.cpython-312.pyc ADDED
Binary file (13.7 kB). View file