File size: 492,960 Bytes
9476ea2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 | # from tvm.script import ir as I
# from tvm.script import tir as T
# from tvm.script import relax as R
@I.ir_module
class Module:
I.module_attrs({"system_lib_prefix": "gpt_neox_q0f16_"})
@T.prim_func
def apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_seq = T.int32(is_size_var=True)
seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32")
bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32")
# with T.block("root"):
for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"):
for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size)
vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size)
T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size)
T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv])
T.writes(logits[seq_ids[vs], vv])
logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0))
@T.prim_func
def apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_token = T.int32(is_size_var=True)
pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32")
token_ids = T.match_buffer(var_token_ids, (num_token,), "int32")
logit_bias = T.match_buffer(var_logit_bias, (num_token,))
# with T.block("root"):
for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"):
for p1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vp = T.axis.spatial(num_token, p0 * 256 + p1)
T.where(p0 * 256 + p1 < num_token)
T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp])
T.writes(logits[pos2seq_id[vp], token_ids[vp]])
logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp]
@T.prim_func
def apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_seq = T.int32(is_size_var=True)
seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32")
num_token = T.int32(is_size_var=True)
pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32")
token_ids = T.match_buffer(var_token_ids, (num_token,), "int32")
token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32")
penalties = T.match_buffer(var_penalties, (num_seq, 3))
# with T.block("root"):
for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"):
for p1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vp = T.axis.spatial(num_token, p0 * 256 + p1)
T.where(p0 * 256 + p1 < num_token)
T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp])
T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]])
logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1])
logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2])
@T.prim_func
def batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
B = T.int32(is_size_var=True)
Q = T.match_buffer(Q_handle, (B, 12, 64), "float16")
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(pages_handle, (max_num_pages, 2, 12, 16, 64), "float16", offset_factor=1)
page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1)
output = T.match_buffer(output_handle, (B, 12, 64), "float16")
lse = T.match_buffer(lse_handle, (B, 12))
# with T.block("root"):
sm_scale: T.float32 = T.float32(0.18033688011112042)
for bx in T.thread_binding(B, thread="blockIdx.x"):
for fused_by_bz in T.thread_binding(12, thread="blockIdx.y"):
for ty in T.thread_binding(1, thread="threadIdx.y"):
for tx in T.thread_binding(16, thread="threadIdx.x"):
for tz in T.thread_binding(16, thread="threadIdx.z"):
with T.block("attn"):
T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz // 12 + ty + fused_by_bz % 12, tx * 4 - 32:tx * 4 - 32 + 68])
T.writes(output[bx, fused_by_bz % 12 + fused_by_bz // 12 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 12 + fused_by_bz // 12 + ty])
Q_local = T.alloc_buffer((4,), "float16", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared")
md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared")
S_reduce_local = T.alloc_buffer((1,), scope="local")
t0 = T.alloc_buffer((1,), scope="local")
S_local = T.alloc_buffer((1,), scope="local")
QK_local = T.alloc_buffer((4,), scope="local")
V_local = T.alloc_buffer((4,), "float16", scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_prev = T.alloc_buffer((1,), scope="local")
other_m = T.alloc_buffer((1,), scope="local")
other_d = T.alloc_buffer((1,), scope="local")
exp_mprev = T.alloc_buffer((1,), scope="local")
exp_otherm = T.alloc_buffer((1,), scope="local")
other_o = T.alloc_buffer((4,), scope="local")
st_m = T.alloc_buffer((1,), scope="local")
st_d = T.alloc_buffer((1,), scope="local")
O_local = T.alloc_buffer((4,), scope="local")
by: T.int32 = fused_by_bz % 12
bz: T.int32 = fused_by_bz // 12
batch_idx: T.int32 = bx
cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx]
cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0)
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for vec in T.vectorized(4):
freq = T.float32()
Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec])
for iterator in range((kv_chunk_len[0] + 15) // 16):
tile_start_s: T.int32 = tz + ty
tile_start_g: T.int32 = iterator * 16 + tz + ty
for j in range(1):
with T.block("KV_load"):
T.reads()
T.writes()
row_g: T.int32 = tile_start_g + j
if row_g < kv_chunk_len[0]:
seq_offset: T.int32 = row_g
page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
for vec in T.vectorized(4):
freq = T.float32()
K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec])
V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec]
else:
for vec in T.vectorized(4):
K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
T.tvm_storage_sync("shared")
m_prev[0] = st_m[0]
for j in range(1):
for vec in T.vectorized(4):
QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale
S_reduce_local[0] = T.float32(0.0)
for vec in T.unroll(4):
S_reduce_local[0] = S_reduce_local[0] + QK_local[vec]
with T.block("block_cross_thread"):
T.reads(S_reduce_local[0])
T.writes(t0[0])
T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0)))
T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx)
S_local[j] = T.float32(-50000.0)
if iterator * 16 + tz + j < kv_chunk_len[0]:
S_local[j] = t0[0]
st_m[0] = T.max(st_m[0], S_local[j])
o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0])
st_d[0] = st_d[0] * o_scale
for j in range(1):
S_local[j] = T.exp2(S_local[j] - st_m[0])
st_d[0] = st_d[0] + S_local[j]
for j in T.vectorized(4):
O_local[j] = O_local[j] * o_scale
for j in range(1):
for vec in T.vectorized(4):
V_local[vec] = V_smem[tz + j, tx * 4 + vec]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j]
for vec in T.vectorized(4):
O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec]
md_allreduce[tz, ty, 0] = st_m[0]
md_allreduce[tz, ty, 1] = st_d[0]
T.tvm_storage_sync("shared")
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for j in range(16):
m_prev[0] = st_m[0]
d_prev[0] = st_d[0]
other_m[0] = md_allreduce[j, ty, 0]
other_d[0] = md_allreduce[j, ty, 1]
for vec in T.vectorized(4):
other_o[vec] = O_allreduce[j, ty, tx * 4 + vec]
st_m[0] = T.max(st_m[0], other_m[0])
st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0])
exp_mprev[0] = T.exp2(m_prev[0] - st_m[0])
exp_otherm[0] = T.exp2(other_m[0] - st_m[0])
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] / st_d[0]
for vec in T.vectorized(4):
output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec])
lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0])
@T.prim_func
def batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
B = T.int32(is_size_var=True)
Q = T.match_buffer(Q_handle, (B, 12, 64), "float16")
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(pages_handle, (max_num_pages, 2, 12, 16, 64), "float16", offset_factor=1)
page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1)
output = T.match_buffer(output_handle, (B, 12, 64), "float16")
lse = T.match_buffer(lse_handle, (B, 12))
# with T.block("root"):
sm_scale: T.float32 = T.float32(0.18033688011112042)
for bx in T.thread_binding(B, thread="blockIdx.x"):
for fused_by_bz in T.thread_binding(12, thread="blockIdx.y"):
for ty in T.thread_binding(1, thread="threadIdx.y"):
for tx in T.thread_binding(16, thread="threadIdx.x"):
for tz in T.thread_binding(16, thread="threadIdx.z"):
with T.block("attn"):
T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz // 12 + ty + fused_by_bz % 12, tx * 4 - 32:tx * 4 - 32 + 68])
T.writes(output[bx, fused_by_bz % 12 + fused_by_bz // 12 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 12 + fused_by_bz // 12 + ty])
Q_local = T.alloc_buffer((4,), "float16", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared")
md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared")
S_reduce_local = T.alloc_buffer((1,), scope="local")
t0 = T.alloc_buffer((1,), scope="local")
S_local = T.alloc_buffer((1,), scope="local")
QK_local = T.alloc_buffer((4,), scope="local")
V_local = T.alloc_buffer((4,), "float16", scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_prev = T.alloc_buffer((1,), scope="local")
other_m = T.alloc_buffer((1,), scope="local")
other_d = T.alloc_buffer((1,), scope="local")
exp_mprev = T.alloc_buffer((1,), scope="local")
exp_otherm = T.alloc_buffer((1,), scope="local")
other_o = T.alloc_buffer((4,), scope="local")
st_m = T.alloc_buffer((1,), scope="local")
st_d = T.alloc_buffer((1,), scope="local")
O_local = T.alloc_buffer((4,), scope="local")
by: T.int32 = fused_by_bz % 12
bz: T.int32 = fused_by_bz // 12
batch_idx: T.int32 = bx
cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx]
cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0)
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for vec in T.vectorized(4):
freq = T.float32()
Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec])
for iterator in range((kv_chunk_len[0] + 15) // 16):
tile_start_s: T.int32 = tz + ty
tile_start_g: T.int32 = iterator * 16 + tz + ty
for j in range(1):
with T.block("KV_load"):
T.reads()
T.writes()
row_g: T.int32 = tile_start_g + j
if row_g < kv_chunk_len[0]:
seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx])
page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
for vec in T.vectorized(4):
freq = T.float32()
K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec])
V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec]
else:
for vec in T.vectorized(4):
K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
T.tvm_storage_sync("shared")
m_prev[0] = st_m[0]
for j in range(1):
for vec in T.vectorized(4):
QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale
S_reduce_local[0] = T.float32(0.0)
for vec in T.unroll(4):
S_reduce_local[0] = S_reduce_local[0] + QK_local[vec]
with T.block("block_cross_thread"):
T.reads(S_reduce_local[0])
T.writes(t0[0])
T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0)))
T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx)
S_local[j] = T.float32(-50000.0)
if iterator * 16 + tz + j < kv_chunk_len[0]:
S_local[j] = t0[0]
st_m[0] = T.max(st_m[0], S_local[j])
o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0])
st_d[0] = st_d[0] * o_scale
for j in range(1):
S_local[j] = T.exp2(S_local[j] - st_m[0])
st_d[0] = st_d[0] + S_local[j]
for j in T.vectorized(4):
O_local[j] = O_local[j] * o_scale
for j in range(1):
for vec in T.vectorized(4):
V_local[vec] = V_smem[tz + j, tx * 4 + vec]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j]
for vec in T.vectorized(4):
O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec]
md_allreduce[tz, ty, 0] = st_m[0]
md_allreduce[tz, ty, 1] = st_d[0]
T.tvm_storage_sync("shared")
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for j in range(16):
m_prev[0] = st_m[0]
d_prev[0] = st_d[0]
other_m[0] = md_allreduce[j, ty, 0]
other_d[0] = md_allreduce[j, ty, 1]
for vec in T.vectorized(4):
other_o[vec] = O_allreduce[j, ty, tx * 4 + vec]
st_m[0] = T.max(st_m[0], other_m[0])
st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0])
exp_mprev[0] = T.exp2(m_prev[0] - st_m[0])
exp_otherm[0] = T.exp2(other_m[0] - st_m[0])
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] / st_d[0]
for vec in T.vectorized(4):
output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec])
lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0])
@T.prim_func
def batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
total_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (total_len, 12, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(var_pages, (max_num_pages, 2, 12, 16, 64), "float16", offset_factor=1)
page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (total_len, 12, 64), "float16")
lse = T.match_buffer(var_lse, (total_len, 12))
# with T.block("root"):
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(12, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 16), scope="shared")
S_local = T.alloc_buffer((32, 16), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
cur_page_indptr_begin: T.int32 = page_indptr[b_idx]
cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0)
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 15) // 16):
L_kv_start: T.int32 = iterator_1 * 16
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("K_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
freq = T.float32()
K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j])
else:
K_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("V_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
V_smem[i, j] = pages[page_no, 1, by, page_offset, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64])
T.writes(S_local[0:32, 0:16])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 2):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
k = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 2):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(16):
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i])
T.writes(S_smem[row, 0:16])
for j in range(16):
if row < 32:
row_: T.int32 = LH_start + row
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(16):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k = T.axis.reduce(16, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@T.prim_func
def batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
total_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (total_len, 12, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(var_pages, (max_num_pages, 2, 12, 16, 64), "float16", offset_factor=1)
page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (total_len, 12, 64), "float16")
lse = T.match_buffer(var_lse, (total_len, 12))
# with T.block("root"):
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(12, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 16), scope="shared")
S_local = T.alloc_buffer((32, 16), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
cur_page_indptr_begin: T.int32 = page_indptr[b_idx]
cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0)
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 15) // 16):
L_kv_start: T.int32 = iterator_1 * 16
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("K_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx])
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
freq = T.float32()
K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j])
else:
K_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("V_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx])
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
V_smem[i, j] = pages[page_no, 1, by, page_offset, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64])
T.writes(S_local[0:32, 0:16])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 2):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
k = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 2):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(16):
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i])
T.writes(S_smem[row, 0:16])
for j in range(16):
if row < 32:
row_: T.int32 = LH_start + row
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(16):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k = T.axis.reduce(16, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@T.prim_func
def batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
qo_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (qo_len, 12, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
kv_len = T.int32(is_size_var=True)
k = T.match_buffer(var_k, (kv_len, 12, 64), "float16")
v = T.match_buffer(var_v, (kv_len, 12, 64), "float16")
kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (qo_len, 12, 64), "float16")
lse = T.match_buffer(var_lse, (qo_len, 12))
# with T.block("root"):
for lbx in T.thread_binding(8, thread="blockIdx.x"):
for lby in T.thread_binding(12, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((64, 32), "float16", scope="shared")
V_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 32), scope="shared")
S_local = T.alloc_buffer((32, 32), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
q_indptr_val: T.int32 = q_indptr[b_idx]
LH_start: T.int32 = tile_id[0] * 32
kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1 in range(4):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_0, lj_0_0 in T.grid(1, 2):
for lj_1 in T.vectorized(8):
with T.block("Q_load"):
i = T.axis.spatial(32, li_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 4)
j = T.axis.spatial(64, lj_0_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 4 * 8 + lj_1)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 31) // 32):
L_kv_start: T.int32 = iterator_1 * 32
L_kv_base: T.int32 = kv_indptr[b_idx]
for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lz_0, ly_0_0 in T.grid(1, 2):
for ly_1 in T.vectorized(8):
with T.block("K_load"):
i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4)
j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1)
T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 32:j - 32 + 65])
T.writes(K_smem[j, i])
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
freq = T.float32()
K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[L_kv_base + cur_L, by, j + 32] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[L_kv_base + cur_L, by, j])
else:
K_smem[j, i] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lz_0, ly_0_0 in T.grid(1, 2):
for ly_1 in T.vectorized(8):
with T.block("V_load"):
i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4)
j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
V_smem[i, j] = v[L_kv_base + cur_L, by, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:64, 0:32])
T.writes(S_local[0:32, 0:32])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init in T.unroll(2):
for lj_1_0_init in T.unroll(1):
for lj_1_1_init in T.vectorized(4):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_0_init * 4 + lj_1_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0 in range(4):
for li_1 in T.unroll(2):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
for lk_1 in range(16):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1)
k_1 = T.axis.reduce(64, lk_0 * 16 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1 in range(2):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(32):
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i])
T.writes(S_smem[row, 0:32])
for j in range(32):
if row < 32:
row_: T.int32 = LH_start + row
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(32):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init in T.unroll(4):
for lj_1_0_init in T.unroll(1):
for lj_1_1_init in T.vectorized(4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_0_init * 4 + lj_1_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1 in T.grid(2, 16):
for li_1 in T.unroll(4):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1)
k_1 = T.axis.reduce(32, lk_0 * 16 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1 in range(4):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 8
@T.prim_func
def batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
qo_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (qo_len, 12, 64), "float16")
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
kv_len = T.int32(is_size_var=True)
k = T.match_buffer(var_k, (kv_len, 12, 64), "float16")
v = T.match_buffer(var_v, (kv_len, 12, 64), "float16")
kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1)
mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1)
tree_size = T.int32(is_size_var=True)
mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1)
output = T.match_buffer(var_output, (qo_len, 12, 64), "float16")
lse = T.match_buffer(var_lse, (qo_len, 12))
# with T.block("root"):
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(12, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 32), scope="shared")
S_local = T.alloc_buffer((32, 32), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 31) // 32):
L_kv_start: T.int32 = iterator_1 * 32
L_kv_base: T.int32 = kv_indptr[b_idx]
for lz_ly_fused_0 in range(4):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("KV_load"):
i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_base + L_kv_start + i
if L_kv_start + i < kv_chunk_len[0]:
freq = T.float32()
K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[cur_L, by, j + 32] * T.float16(-1.0), k[cur_L, by, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[cur_L, by, j])
V_smem[i, j] = v[cur_L, by, j]
else:
K_smem[i, j] = T.float16(0.0)
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64])
T.writes(S_local[0:32, 0:32])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 4):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
k_1 = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 4):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(32):
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i])
T.writes(S_smem[row, 0:32])
for j in range(32):
if row < 32:
row_: T.int32 = LH_start + row
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(32):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@T.prim_func
def chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle):
T.func_attr({"op_pattern": 4, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True)
A = T.match_buffer(var_A, (batch_size, vocab_size))
temperature = T.match_buffer(var_temperature, (batch_size,))
num_chunks = T.int64(is_size_var=True)
chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks))
chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks))
# with T.block("root"):
A_pad = T.alloc_buffer((batch_size, num_chunks, T.int64(4096)))
temp_max = T.alloc_buffer((batch_size, num_chunks))
temp_sum = T.alloc_buffer((batch_size, num_chunks))
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)):
with T.block("pad"):
v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2])
T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2])
T.writes(A_pad[v0, v1, v2])
A_pad[v0, v1, v2] = T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * T.int64(4096) + v2] / temperature[v0], A[v0, v1 * T.int64(4096) + v2]), T.float32(-340282346638528859811704183484516925440.0))
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)):
with T.block("max"):
v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2])
T.reads(A_pad[v0, v1, v2])
T.writes(temp_max[v0, v1])
with T.init():
temp_max[v0, v1] = T.float32(-340282346638528859811704183484516925440.0)
temp_max[v0, v1] = T.max(temp_max[v0, v1], A_pad[v0, v1, v2])
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)):
with T.block("sum_exp"):
v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2])
T.reads(temperature[v0], A_pad[v0, v1, v2], temp_max[v0, v1])
T.writes(temp_sum[v0, v1])
with T.init():
temp_sum[v0, v1] = T.float32(0.0)
temp_sum[v0, v1] = temp_sum[v0, v1] + T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A_pad[v0, v1, v2] - temp_max[v0, v1]), T.Cast("float32", A_pad[v0, v1, v2] == temp_max[v0, v1])), T.float32(0.0))
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(1)):
with T.block("log"):
v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2])
T.reads(temperature[v0], temp_sum[v0, v1], temp_max[v0, v1])
T.writes(chunked_sum[v0, v1], chunked_max[v0, v1])
chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum[v0, v1]), temp_sum[v0, v1])
chunked_max[v0, v1] = temp_max[v0, v1]
@T.prim_func
def compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
num_pages = T.int32()
pages = T.match_buffer(var_pages, (num_pages, 2, 12, 16, 64), "float16", offset_factor=1)
copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1)
total_copy_length = T.int32()
copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1)
with T.block("root"):
T.reads()
T.writes()
for bhd_o in T.thread_binding(batch_size * 3, thread="blockIdx.x"):
for bhd_i in T.thread_binding(256, thread="threadIdx.x"):
b: T.int32 = (bhd_o * 256 + bhd_i) // 768
h: T.int32 = (bhd_o * 256 + bhd_i) // 64 % 12
d: T.int32 = (bhd_o * 256 + bhd_i) % 64
if bhd_o * 256 + bhd_i < batch_size * 12 * 64:
for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]):
src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i]
dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i]
pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d]
pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d]
@T.prim_func
def copy_single_page(var_pages: T.handle, src_page_id: T.int64, tgt_page_id: T.int64, copy_length: T.int64):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
num_pages, page_size = T.int32(), T.int64()
pages = T.match_buffer(var_pages, (num_pages, 2, 12, page_size, 64), "float16", offset_factor=1)
# with T.block("root"):
for b in T.thread_binding(copy_length * T.int64(3), thread="blockIdx.x"):
for t in T.thread_binding(256, thread="threadIdx.x"):
with T.block("copy"):
vh = T.axis.spatial(12, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) // (copy_length * T.int64(64))))
vp = T.axis.spatial(copy_length, (b * T.int64(256) + T.Cast("int64", t)) % (copy_length * T.int64(64)) // T.int64(64))
vd = T.axis.spatial(64, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) % T.int64(64)))
T.where(b * T.int64(256) + T.Cast("int64", t) < copy_length * T.int64(12) * T.int64(64))
T.reads(pages[src_page_id, 0:2, vh, vp, vd])
T.writes(pages[tgt_page_id, 0:2, vh, vp, vd])
pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd]
pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd]
@T.prim_func(private=True)
def fused_NT_matmul10_add10(layer_norm25: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16"), gpt_neox_layers_0_attention_query_key_value_weight2: T.Buffer((T.int64(2304), T.int64(768)), "float16"), gpt_neox_layers_0_attention_query_key_value_bias2: T.Buffer((T.int64(2304),), "float16"), T_add_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(2304)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(2304)), "float16")
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(2304), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm25[v_i0, v_i1, v_k], gpt_neox_layers_0_attention_query_key_value_weight2[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + layer_norm25[v_i0, v_i1, v_k] * gpt_neox_layers_0_attention_query_key_value_weight2[v_i2, v_k]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(2304)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_attention_query_key_value_bias2[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_attention_query_key_value_bias2[v_ax2]
@T.prim_func(private=True)
def fused_NT_matmul11_add11_add14_add14(reshape51: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16"), gpt_neox_layers_0_attention_dense_weight2: T.Buffer((T.int64(768), T.int64(768)), "float16"), gpt_neox_layers_0_attention_dense_bias2: T.Buffer((T.int64(768),), "float16"), astype26: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16"), input_embed: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16"), T_add_intermediate_1_2: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")
T_add_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")
T_add_intermediate_1 = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(768), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(reshape51[v_i0, v_i1, v_k], gpt_neox_layers_0_attention_dense_weight2[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + reshape51[v_i0, v_i1, v_k] * gpt_neox_layers_0_attention_dense_weight2[v_i2, v_k]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_attention_dense_bias2[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_attention_dense_bias2[v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("T_add_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(astype26[v_ax0, v_ax1, v_ax2], T_add_intermediate[v_ax0, v_ax1, v_ax2])
T.writes(T_add_intermediate_1[v_ax0, v_ax1, v_ax2])
T_add_intermediate_1[v_ax0, v_ax1, v_ax2] = astype26[v_ax0, v_ax1, v_ax2] + T_add_intermediate[v_ax0, v_ax1, v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("T_add_2"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate_1[v_ax0, v_ax1, v_ax2], input_embed[v_ax0, v_ax1, v_ax2])
T.writes(T_add_intermediate_1_2[v_ax0, v_ax1, v_ax2])
T_add_intermediate_1_2[v_ax0, v_ax1, v_ax2] = T_add_intermediate_1[v_ax0, v_ax1, v_ax2] + input_embed[v_ax0, v_ax1, v_ax2]
@T.prim_func(private=True)
def fused_NT_matmul12_add12_gelu2_cast6(layer_norm26: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16"), gpt_neox_layers_0_mlp_dense_h_to_4h_weight2: T.Buffer((T.int64(3072), T.int64(768)), "float16"), gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: T.Buffer((T.int64(3072),), "float32"), compute_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(3072)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(3072)))
T_add_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(3072)))
T_multiply = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(3072)))
compute = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(3072)))
T_multiply_1 = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(3072)))
T_add = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(3072)))
T_multiply_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(3072)))
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(3072), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm26[v_i0, v_i1, v_k], gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + T.Cast("float32", layer_norm26[v_i0, v_i1, v_k]) * T.Cast("float32", gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[v_i2, v_k])
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(3072)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(3072)):
with T.block("T_multiply"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply[v_ax0, v_ax1, v_ax2])
T_multiply[v_ax0, v_ax1, v_ax2] = T_add_intermediate[v_ax0, v_ax1, v_ax2] * T.float32(0.70710678118654757)
for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(3072)):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_multiply[v_i0, v_i1, v_i2])
T.writes(compute[v_i0, v_i1, v_i2])
compute[v_i0, v_i1, v_i2] = T.erf(T_multiply[v_i0, v_i1, v_i2])
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(3072)):
with T.block("T_multiply_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(compute[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply_1[v_ax0, v_ax1, v_ax2])
T_multiply_1[v_ax0, v_ax1, v_ax2] = compute[v_ax0, v_ax1, v_ax2] * T.float32(0.5)
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(3072)):
with T.block("T_add_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_multiply_1[v_ax0, v_ax1, v_ax2])
T.writes(T_add[v_ax0, v_ax1, v_ax2])
T_add[v_ax0, v_ax1, v_ax2] = T.float32(0.5) + T_multiply_1[v_ax0, v_ax1, v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(3072)):
with T.block("T_multiply_2"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate[v_ax0, v_ax1, v_ax2], T_add[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply_intermediate[v_ax0, v_ax1, v_ax2])
T_multiply_intermediate[v_ax0, v_ax1, v_ax2] = T_add_intermediate[v_ax0, v_ax1, v_ax2] * T_add[v_ax0, v_ax1, v_ax2]
for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(3072)):
with T.block("compute_1"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_multiply_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float16", T_multiply_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul13_add13_cast7(astype25: T.Buffer((T.int64(1), T.int64(1), T.int64(3072)), "float16"), gpt_neox_layers_0_mlp_dense_4h_to_h_weight2: T.Buffer((T.int64(768), T.int64(3072)), "float16"), gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: T.Buffer((T.int64(768),), "float32"), compute_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(768)))
T_add_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(768)))
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(768), T.int64(3072)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(astype25[v_i0, v_i1, v_k], gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + T.Cast("float32", astype25[v_i0, v_i1, v_k]) * T.Cast("float32", gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[v_i2, v_k])
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[v_ax2]
for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_add_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float16", T_add_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul14_cast8(layer_norm49: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16"), p_embed_out_weight2: T.handle, p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
vocab_size = T.int64()
embed_out_weight2 = T.match_buffer(p_embed_out_weight2, (vocab_size, T.int64(768)), "float16")
compute_intermediate = T.match_buffer(p_output0, (T.int64(1), T.int64(1), vocab_size))
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), vocab_size), "float16")
for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), vocab_size, T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm49[v_i0, v_i1, v_k], embed_out_weight2[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + layer_norm49[v_i0, v_i1, v_k] * embed_out_weight2[v_i2, v_k]
for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), vocab_size):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(NT_matmul_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float32", NT_matmul_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul1_add1_add4_add4(p_reshape147: T.handle, gpt_neox_layers_0_attention_dense_weight4: T.Buffer((T.int64(768), T.int64(768)), "float16"), gpt_neox_layers_0_attention_dense_bias4: T.Buffer((T.int64(768),), "float16"), p_astype76: T.handle, p_input_embeds: T.handle, p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
reshape147 = T.match_buffer(p_reshape147, (batch_size, T.int64(1), T.int64(768)), "float16")
astype76 = T.match_buffer(p_astype76, (batch_size, T.int64(1), T.int64(768)), "float16")
input_embeds = T.match_buffer(p_input_embeds, (batch_size, T.int64(1), T.int64(768)), "float16")
T_add_intermediate_1_2 = T.match_buffer(p_output0, (batch_size, T.int64(1), T.int64(768)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(768)), "float16")
T_add_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(768)), "float16")
T_add_intermediate_1 = T.alloc_buffer((batch_size, T.int64(1), T.int64(768)), "float16")
for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(768), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(reshape147[v_i0, v_i1, v_k], gpt_neox_layers_0_attention_dense_weight4[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + reshape147[v_i0, v_i1, v_k] * gpt_neox_layers_0_attention_dense_weight4[v_i2, v_k]
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_attention_dense_bias4[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_attention_dense_bias4[v_ax2]
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("T_add_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(astype76[v_ax0, v_ax1, v_ax2], T_add_intermediate[v_ax0, v_ax1, v_ax2])
T.writes(T_add_intermediate_1[v_ax0, v_ax1, v_ax2])
T_add_intermediate_1[v_ax0, v_ax1, v_ax2] = astype76[v_ax0, v_ax1, v_ax2] + T_add_intermediate[v_ax0, v_ax1, v_ax2]
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("T_add_2"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate_1[v_ax0, v_ax1, v_ax2], input_embeds[v_ax0, v_ax1, v_ax2])
T.writes(T_add_intermediate_1_2[v_ax0, v_ax1, v_ax2])
T_add_intermediate_1_2[v_ax0, v_ax1, v_ax2] = T_add_intermediate_1[v_ax0, v_ax1, v_ax2] + input_embeds[v_ax0, v_ax1, v_ax2]
@T.prim_func(private=True)
def fused_NT_matmul2_add2_gelu_cast(p_layer_norm76: T.handle, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4: T.Buffer((T.int64(3072), T.int64(768)), "float16"), gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: T.Buffer((T.int64(3072),), "float32"), p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
layer_norm76 = T.match_buffer(p_layer_norm76, (batch_size, T.int64(1), T.int64(768)), "float16")
compute_intermediate = T.match_buffer(p_output0, (batch_size, T.int64(1), T.int64(3072)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(3072)))
T_add_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(3072)))
T_multiply = T.alloc_buffer((batch_size, T.int64(1), T.int64(3072)))
compute = T.alloc_buffer((batch_size, T.int64(1), T.int64(3072)))
T_multiply_1 = T.alloc_buffer((batch_size, T.int64(1), T.int64(3072)))
T_add = T.alloc_buffer((batch_size, T.int64(1), T.int64(3072)))
T_multiply_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(3072)))
for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(3072), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm76[v_i0, v_i1, v_k], gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + T.Cast("float32", layer_norm76[v_i0, v_i1, v_k]) * T.Cast("float32", gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_i2, v_k])
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(3072)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[v_ax2]
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(3072)):
with T.block("T_multiply"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply[v_ax0, v_ax1, v_ax2])
T_multiply[v_ax0, v_ax1, v_ax2] = T_add_intermediate[v_ax0, v_ax1, v_ax2] * T.float32(0.70710678118654757)
for i0, i1, i2 in T.grid(batch_size, T.int64(1), T.int64(3072)):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_multiply[v_i0, v_i1, v_i2])
T.writes(compute[v_i0, v_i1, v_i2])
compute[v_i0, v_i1, v_i2] = T.erf(T_multiply[v_i0, v_i1, v_i2])
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(3072)):
with T.block("T_multiply_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(compute[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply_1[v_ax0, v_ax1, v_ax2])
T_multiply_1[v_ax0, v_ax1, v_ax2] = compute[v_ax0, v_ax1, v_ax2] * T.float32(0.5)
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(3072)):
with T.block("T_add_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_multiply_1[v_ax0, v_ax1, v_ax2])
T.writes(T_add[v_ax0, v_ax1, v_ax2])
T_add[v_ax0, v_ax1, v_ax2] = T.float32(0.5) + T_multiply_1[v_ax0, v_ax1, v_ax2]
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(3072)):
with T.block("T_multiply_2"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate[v_ax0, v_ax1, v_ax2], T_add[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply_intermediate[v_ax0, v_ax1, v_ax2])
T_multiply_intermediate[v_ax0, v_ax1, v_ax2] = T_add_intermediate[v_ax0, v_ax1, v_ax2] * T_add[v_ax0, v_ax1, v_ax2]
for i0, i1, i2 in T.grid(batch_size, T.int64(1), T.int64(3072)):
with T.block("compute_1"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_multiply_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float16", T_multiply_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul3_add3_cast1(p_astype75: T.handle, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4: T.Buffer((T.int64(768), T.int64(3072)), "float16"), gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: T.Buffer((T.int64(768),), "float32"), p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
astype75 = T.match_buffer(p_astype75, (batch_size, T.int64(1), T.int64(3072)), "float16")
compute_intermediate = T.match_buffer(p_output0, (batch_size, T.int64(1), T.int64(768)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(768)))
T_add_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(768)))
for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(768), T.int64(3072)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(astype75[v_i0, v_i1, v_k], gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + T.Cast("float32", astype75[v_i0, v_i1, v_k]) * T.Cast("float32", gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_i2, v_k])
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[v_ax2]
for i0, i1, i2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_add_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float16", T_add_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul4_cast2(p_layer_norm99: T.handle, p_embed_out_weight4: T.handle, p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
layer_norm99 = T.match_buffer(p_layer_norm99, (batch_size, T.int64(1), T.int64(768)), "float16")
vocab_size = T.int64()
embed_out_weight4 = T.match_buffer(p_embed_out_weight4, (vocab_size, T.int64(768)), "float16")
compute_intermediate = T.match_buffer(p_output0, (batch_size, T.int64(1), vocab_size))
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((batch_size, T.int64(1), vocab_size), "float16")
for i0, i1, i2, k in T.grid(batch_size, T.int64(1), vocab_size, T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm99[v_i0, v_i1, v_k], embed_out_weight4[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + layer_norm99[v_i0, v_i1, v_k] * embed_out_weight4[v_i2, v_k]
for i0, i1, i2 in T.grid(batch_size, T.int64(1), vocab_size):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(NT_matmul_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float32", NT_matmul_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul5_add5(p_layer_norm50: T.handle, gpt_neox_layers_0_attention_query_key_value_weight3: T.Buffer((T.int64(2304), T.int64(768)), "float16"), gpt_neox_layers_0_attention_query_key_value_bias3: T.Buffer((T.int64(2304),), "float16"), p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
seq_len = T.int64()
layer_norm50 = T.match_buffer(p_layer_norm50, (T.int64(1), seq_len, T.int64(768)), "float16")
T_add_intermediate = T.match_buffer(p_output0, (T.int64(1), seq_len, T.int64(2304)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(2304)), "float16")
for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(2304), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm50[v_i0, v_i1, v_k], gpt_neox_layers_0_attention_query_key_value_weight3[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + layer_norm50[v_i0, v_i1, v_k] * gpt_neox_layers_0_attention_query_key_value_weight3[v_i2, v_k]
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(2304)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_attention_query_key_value_bias3[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_attention_query_key_value_bias3[v_ax2]
@T.prim_func(private=True)
def fused_NT_matmul6_add6_add9_add9(p_reshape99: T.handle, gpt_neox_layers_0_attention_dense_weight3: T.Buffer((T.int64(768), T.int64(768)), "float16"), gpt_neox_layers_0_attention_dense_bias3: T.Buffer((T.int64(768),), "float16"), p_astype51: T.handle, p_input_embeds: T.handle, p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
seq_len = T.int64()
reshape99 = T.match_buffer(p_reshape99, (T.int64(1), seq_len, T.int64(768)), "float16")
astype51 = T.match_buffer(p_astype51, (T.int64(1), seq_len, T.int64(768)), "float16")
input_embeds = T.match_buffer(p_input_embeds, (T.int64(1), seq_len, T.int64(768)), "float16")
T_add_intermediate_1_2 = T.match_buffer(p_output0, (T.int64(1), seq_len, T.int64(768)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(768)), "float16")
T_add_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(768)), "float16")
T_add_intermediate_1 = T.alloc_buffer((T.int64(1), seq_len, T.int64(768)), "float16")
for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(768), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(reshape99[v_i0, v_i1, v_k], gpt_neox_layers_0_attention_dense_weight3[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + reshape99[v_i0, v_i1, v_k] * gpt_neox_layers_0_attention_dense_weight3[v_i2, v_k]
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_attention_dense_bias3[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_attention_dense_bias3[v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("T_add_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(astype51[v_ax0, v_ax1, v_ax2], T_add_intermediate[v_ax0, v_ax1, v_ax2])
T.writes(T_add_intermediate_1[v_ax0, v_ax1, v_ax2])
T_add_intermediate_1[v_ax0, v_ax1, v_ax2] = astype51[v_ax0, v_ax1, v_ax2] + T_add_intermediate[v_ax0, v_ax1, v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("T_add_2"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate_1[v_ax0, v_ax1, v_ax2], input_embeds[v_ax0, v_ax1, v_ax2])
T.writes(T_add_intermediate_1_2[v_ax0, v_ax1, v_ax2])
T_add_intermediate_1_2[v_ax0, v_ax1, v_ax2] = T_add_intermediate_1[v_ax0, v_ax1, v_ax2] + input_embeds[v_ax0, v_ax1, v_ax2]
@T.prim_func(private=True)
def fused_NT_matmul7_add7_gelu1_cast3(p_layer_norm51: T.handle, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3: T.Buffer((T.int64(3072), T.int64(768)), "float16"), gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: T.Buffer((T.int64(3072),), "float32"), p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
seq_len = T.int64()
layer_norm51 = T.match_buffer(p_layer_norm51, (T.int64(1), seq_len, T.int64(768)), "float16")
compute_intermediate = T.match_buffer(p_output0, (T.int64(1), seq_len, T.int64(3072)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(3072)))
T_add_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(3072)))
T_multiply = T.alloc_buffer((T.int64(1), seq_len, T.int64(3072)))
compute = T.alloc_buffer((T.int64(1), seq_len, T.int64(3072)))
T_multiply_1 = T.alloc_buffer((T.int64(1), seq_len, T.int64(3072)))
T_add = T.alloc_buffer((T.int64(1), seq_len, T.int64(3072)))
T_multiply_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(3072)))
for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(3072), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm51[v_i0, v_i1, v_k], gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + T.Cast("float32", layer_norm51[v_i0, v_i1, v_k]) * T.Cast("float32", gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_i2, v_k])
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(3072)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(3072)):
with T.block("T_multiply"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply[v_ax0, v_ax1, v_ax2])
T_multiply[v_ax0, v_ax1, v_ax2] = T_add_intermediate[v_ax0, v_ax1, v_ax2] * T.float32(0.70710678118654757)
for i0, i1, i2 in T.grid(T.int64(1), seq_len, T.int64(3072)):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_multiply[v_i0, v_i1, v_i2])
T.writes(compute[v_i0, v_i1, v_i2])
compute[v_i0, v_i1, v_i2] = T.erf(T_multiply[v_i0, v_i1, v_i2])
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(3072)):
with T.block("T_multiply_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(compute[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply_1[v_ax0, v_ax1, v_ax2])
T_multiply_1[v_ax0, v_ax1, v_ax2] = compute[v_ax0, v_ax1, v_ax2] * T.float32(0.5)
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(3072)):
with T.block("T_add_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_multiply_1[v_ax0, v_ax1, v_ax2])
T.writes(T_add[v_ax0, v_ax1, v_ax2])
T_add[v_ax0, v_ax1, v_ax2] = T.float32(0.5) + T_multiply_1[v_ax0, v_ax1, v_ax2]
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(3072)):
with T.block("T_multiply_2"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_add_intermediate[v_ax0, v_ax1, v_ax2], T_add[v_ax0, v_ax1, v_ax2])
T.writes(T_multiply_intermediate[v_ax0, v_ax1, v_ax2])
T_multiply_intermediate[v_ax0, v_ax1, v_ax2] = T_add_intermediate[v_ax0, v_ax1, v_ax2] * T_add[v_ax0, v_ax1, v_ax2]
for i0, i1, i2 in T.grid(T.int64(1), seq_len, T.int64(3072)):
with T.block("compute_1"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_multiply_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float16", T_multiply_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul8_add8_cast4(p_astype50: T.handle, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3: T.Buffer((T.int64(768), T.int64(3072)), "float16"), gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: T.Buffer((T.int64(768),), "float32"), p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
seq_len = T.int64()
astype50 = T.match_buffer(p_astype50, (T.int64(1), seq_len, T.int64(3072)), "float16")
compute_intermediate = T.match_buffer(p_output0, (T.int64(1), seq_len, T.int64(768)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(768)))
T_add_intermediate = T.alloc_buffer((T.int64(1), seq_len, T.int64(768)))
for i0, i1, i2, k in T.grid(T.int64(1), seq_len, T.int64(768), T.int64(3072)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(astype50[v_i0, v_i1, v_k], gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + T.Cast("float32", astype50[v_i0, v_i1, v_k]) * T.Cast("float32", gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_i2, v_k])
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[v_ax2]
for i0, i1, i2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(T_add_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float16", T_add_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul9_cast5(p_take1: T.handle, p_embed_out_weight3: T.handle, p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
take1 = T.match_buffer(p_take1, (T.int64(1), batch_size, T.int64(768)), "float16")
vocab_size = T.int64()
embed_out_weight3 = T.match_buffer(p_embed_out_weight3, (vocab_size, T.int64(768)), "float16")
compute_intermediate = T.match_buffer(p_output0, (T.int64(1), batch_size, vocab_size))
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((T.int64(1), batch_size, vocab_size), "float16")
for i0, i1, i2, k in T.grid(T.int64(1), batch_size, vocab_size, T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(take1[v_i0, v_i1, v_k], embed_out_weight3[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + take1[v_i0, v_i1, v_k] * embed_out_weight3[v_i2, v_k]
for i0, i1, i2 in T.grid(T.int64(1), batch_size, vocab_size):
with T.block("compute"):
v_i0, v_i1, v_i2 = T.axis.remap("SSS", [i0, i1, i2])
T.reads(NT_matmul_intermediate[v_i0, v_i1, v_i2])
T.writes(compute_intermediate[v_i0, v_i1, v_i2])
compute_intermediate[v_i0, v_i1, v_i2] = T.Cast("float32", NT_matmul_intermediate[v_i0, v_i1, v_i2])
@T.prim_func(private=True)
def fused_NT_matmul_add(p_layer_norm75: T.handle, gpt_neox_layers_0_attention_query_key_value_weight4: T.Buffer((T.int64(2304), T.int64(768)), "float16"), gpt_neox_layers_0_attention_query_key_value_bias4: T.Buffer((T.int64(2304),), "float16"), p_output0: T.handle):
T.func_attr({"tir.noalias": T.bool(True)})
batch_size = T.int64()
layer_norm75 = T.match_buffer(p_layer_norm75, (batch_size, T.int64(1), T.int64(768)), "float16")
T_add_intermediate = T.match_buffer(p_output0, (batch_size, T.int64(1), T.int64(2304)), "float16")
# with T.block("root"):
NT_matmul_intermediate = T.alloc_buffer((batch_size, T.int64(1), T.int64(2304)), "float16")
for i0, i1, i2, k in T.grid(batch_size, T.int64(1), T.int64(2304), T.int64(768)):
with T.block("NT_matmul"):
v_i0, v_i1, v_i2, v_k = T.axis.remap("SSSR", [i0, i1, i2, k])
T.reads(layer_norm75[v_i0, v_i1, v_k], gpt_neox_layers_0_attention_query_key_value_weight4[v_i2, v_k])
T.writes(NT_matmul_intermediate[v_i0, v_i1, v_i2])
with T.init():
NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0.0)
NT_matmul_intermediate[v_i0, v_i1, v_i2] = NT_matmul_intermediate[v_i0, v_i1, v_i2] + layer_norm75[v_i0, v_i1, v_k] * gpt_neox_layers_0_attention_query_key_value_weight4[v_i2, v_k]
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(2304)):
with T.block("T_add"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], gpt_neox_layers_0_attention_query_key_value_bias4[v_ax2])
T.writes(T_add_intermediate[v_ax0, v_ax1, v_ax2])
T_add_intermediate[v_ax0, v_ax1, v_ax2] = NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] + gpt_neox_layers_0_attention_query_key_value_bias4[v_ax2]
@T.prim_func(private=True)
def fused_reshape10_reshape11(lv13: T.Buffer((T.int64(1), T.int64(12), T.int64(64)), "float16"), T_reshape_intermediate_1: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
T_reshape_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(12), T.int64(64)), "float16")
for ax0, ax1, ax2, ax3 in T.grid(T.int64(1), T.int64(1), T.int64(12), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap("SSSS", [ax0, ax1, ax2, ax3])
T.reads(lv13[T.int64(0), (v_ax3 // T.int64(64) + v_ax2) % T.int64(12), v_ax3 % T.int64(64)])
T.writes(T_reshape_intermediate[v_ax0, v_ax1, v_ax2, v_ax3])
T_reshape_intermediate[v_ax0, v_ax1, v_ax2, v_ax3] = lv13[T.int64(0), (v_ax3 // T.int64(64) + v_ax2) % T.int64(12), v_ax3 % T.int64(64)]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("T_reshape_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_reshape_intermediate[T.int64(0), T.int64(0), v_ax2 % T.int64(768) // T.int64(64), v_ax2 % T.int64(64)])
T.writes(T_reshape_intermediate_1[v_ax0, v_ax1, v_ax2])
T_reshape_intermediate_1[v_ax0, v_ax1, v_ax2] = T_reshape_intermediate[T.int64(0), T.int64(0), v_ax2 % T.int64(768) // T.int64(64), v_ax2 % T.int64(64)]
@T.prim_func(private=True)
def fused_reshape8_reshape9(add72: T.Buffer((T.int64(1), T.int64(1), T.int64(2304)), "float16"), T_reshape_intermediate_1: T.Buffer((T.int64(1), T.int64(36), T.int64(64)), "float16")):
T.func_attr({"tir.noalias": T.bool(True)})
# with T.block("root"):
T_reshape_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(36), T.int64(64)), "float16")
for ax0, ax1, ax2, ax3 in T.grid(T.int64(1), T.int64(1), T.int64(36), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap("SSSS", [ax0, ax1, ax2, ax3])
T.reads(add72[T.int64(0), T.int64(0), (v_ax2 * T.int64(64) + v_ax3) % T.int64(2304)])
T.writes(T_reshape_intermediate[v_ax0, v_ax1, v_ax2, v_ax3])
T_reshape_intermediate[v_ax0, v_ax1, v_ax2, v_ax3] = add72[T.int64(0), T.int64(0), (v_ax2 * T.int64(64) + v_ax3) % T.int64(2304)]
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(36), T.int64(64)):
with T.block("T_reshape_1"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(T_reshape_intermediate[T.int64(0), T.int64(0), (v_ax2 // T.int64(64) + v_ax1) % T.int64(36), v_ax2 % T.int64(64)])
T.writes(T_reshape_intermediate_1[v_ax0, v_ax1, v_ax2])
T_reshape_intermediate_1[v_ax0, v_ax1, v_ax2] = T_reshape_intermediate[T.int64(0), T.int64(0), (v_ax2 // T.int64(64) + v_ax1) % T.int64(36), v_ax2 % T.int64(64)]
@T.prim_func
def fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
seq_len = T.int32()
qkv = T.match_buffer(var_qkv, (seq_len, 36, 64), "float16")
position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1)
q = T.match_buffer(var_q, (seq_len, 12, 64), "float16")
k = T.match_buffer(var_k, (seq_len, 12, 64), "float16")
v = T.match_buffer(var_v, (seq_len, 12, 64), "float16")
# with T.block("root"):
for iters_0, iters_1, iters_2 in T.grid(seq_len, 36, 64):
with T.block("llama_fused_rope"):
s, h, d = T.axis.remap("SSS", [iters_0, iters_1, iters_2])
T.reads(position_map[s], qkv[s, h, d - 8:d - 8 + 17])
T.writes(q[s, h, d], k[s, h - 12, d], v[s, h - 24, d])
if h < 12:
freq = T.float32()
q[s, h, d] = T.if_then_else(apply_rope > 0 and d < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 8, qkv[s, h, d + 8] * T.float16(-1.0), qkv[s, h, d - 8]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 16) / T.float32(16.0))}), qkv[s, h, d])
else:
if h < 24:
freq = T.float32()
k[s, h - 12, d] = T.if_then_else(apply_rope > 0 and d < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 8, qkv[s, h, d + 8] * T.float16(-1.0), qkv[s, h, d - 8]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 16) / T.float32(16.0))}), qkv[s, h, d])
else:
v[s, h - 24, d] = qkv[s, h, d]
@T.prim_func
def gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
m, n = T.int32(is_size_var=True), T.int32(is_size_var=True)
src = T.match_buffer(var_src, (m, n))
batch_size = T.int32(is_size_var=True)
indices = T.match_buffer(var_indices, (batch_size,), "int32")
dst = T.match_buffer(var_dst, (batch_size, n))
# with T.block("root"):
for b, j in T.grid(batch_size, n):
with T.block("gather_2d"):
vb, vj = T.axis.remap("SS", [b, j])
T.reads(src[indices[vb], vj], indices[vb])
T.writes(dst[vb, vj])
dst[vb, vj] = src[indices[vb], vj]
@T.prim_func(private=True)
def index(var_layer_norm24: T.handle, index: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")):
T.func_attr({"op_pattern": 8, "target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
seq_len = T.int64()
layer_norm24 = T.match_buffer(var_layer_norm24, (T.int64(1), seq_len, T.int64(768)), "float16")
# with T.block("root"):
for i, _, k in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("index"):
v_i, v__, v_k = T.axis.remap("SSS", [i, _, k])
T.reads(layer_norm24[v_i, seq_len - T.int64(1), v_k])
T.writes(index[v_i, v__, v_k])
index[v_i, v__, v_k] = layer_norm24[v_i, seq_len - T.int64(1), v_k]
@T.prim_func(private=True)
def layer_norm(var_input_embeds: T.handle, gpt_neox_layers_0_input_layernorm_weight4: T.Buffer((T.int64(768),), "float16"), gpt_neox_layers_0_input_layernorm_bias4: T.Buffer((T.int64(768),), "float16"), var_T_layer_norm: T.handle):
T.func_attr({"op_pattern": 4, "tir.noalias": T.bool(True)})
batch_size = T.int64()
input_embeds = T.match_buffer(var_input_embeds, (batch_size, T.int64(1), T.int64(768)), "float16")
T_layer_norm = T.match_buffer(var_T_layer_norm, (batch_size, T.int64(1), T.int64(768)), "float16")
# with T.block("root"):
input_embeds_red_temp_v0 = T.alloc_buffer((batch_size, T.int64(1)))
input_embeds_red_temp_v1 = T.alloc_buffer((batch_size, T.int64(1)))
for ax0, ax1, k2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("input_embeds_red_temp"):
v_ax0, v_ax1, v_k2 = T.axis.remap("SSR", [ax0, ax1, k2])
T.reads(input_embeds[v_ax0, v_ax1, v_k2])
T.writes(input_embeds_red_temp_v0[v_ax0, v_ax1], input_embeds_red_temp_v1[v_ax0, v_ax1])
with T.init():
input_embeds_red_temp_v0[v_ax0, v_ax1] = T.float32(0.0)
input_embeds_red_temp_v1[v_ax0, v_ax1] = T.float32(0.0)
v_input_embeds_red_temp_v0: T.float32 = input_embeds_red_temp_v0[v_ax0, v_ax1] + T.Cast("float32", input_embeds[v_ax0, v_ax1, v_k2])
v_input_embeds_red_temp_v1: T.float32 = input_embeds_red_temp_v1[v_ax0, v_ax1] + T.Cast("float32", input_embeds[v_ax0, v_ax1, v_k2]) * T.Cast("float32", input_embeds[v_ax0, v_ax1, v_k2])
input_embeds_red_temp_v0[v_ax0, v_ax1] = v_input_embeds_red_temp_v0
input_embeds_red_temp_v1[v_ax0, v_ax1] = v_input_embeds_red_temp_v1
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("T_layer_norm"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(input_embeds[v_ax0, v_ax1, v_ax2], input_embeds_red_temp_v0[v_ax0, v_ax1], input_embeds_red_temp_v1[v_ax0, v_ax1], gpt_neox_layers_0_input_layernorm_weight4[v_ax2], gpt_neox_layers_0_input_layernorm_bias4[v_ax2])
T.writes(T_layer_norm[v_ax0, v_ax1, v_ax2])
T_layer_norm[v_ax0, v_ax1, v_ax2] = T.Cast("float16", (T.Cast("float32", input_embeds[v_ax0, v_ax1, v_ax2]) - input_embeds_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333)) * T.rsqrt(input_embeds_red_temp_v1[v_ax0, v_ax1] * T.float32(0.0013020833333333333) - input_embeds_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333) * (input_embeds_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight4[v_ax2] + gpt_neox_layers_0_input_layernorm_bias4[v_ax2]
@T.prim_func(private=True)
def layer_norm1(var_input_embeds: T.handle, gpt_neox_layers_0_input_layernorm_weight3: T.Buffer((T.int64(768),), "float16"), gpt_neox_layers_0_input_layernorm_bias3: T.Buffer((T.int64(768),), "float16"), var_T_layer_norm: T.handle):
T.func_attr({"op_pattern": 4, "tir.noalias": T.bool(True)})
seq_len = T.int64()
input_embeds = T.match_buffer(var_input_embeds, (T.int64(1), seq_len, T.int64(768)), "float16")
T_layer_norm = T.match_buffer(var_T_layer_norm, (T.int64(1), seq_len, T.int64(768)), "float16")
# with T.block("root"):
input_embeds_red_temp_v0 = T.alloc_buffer((T.int64(1), seq_len))
input_embeds_red_temp_v1 = T.alloc_buffer((T.int64(1), seq_len))
for ax0, ax1, k2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("input_embeds_red_temp"):
v_ax0, v_ax1, v_k2 = T.axis.remap("SSR", [ax0, ax1, k2])
T.reads(input_embeds[v_ax0, v_ax1, v_k2])
T.writes(input_embeds_red_temp_v0[v_ax0, v_ax1], input_embeds_red_temp_v1[v_ax0, v_ax1])
with T.init():
input_embeds_red_temp_v0[v_ax0, v_ax1] = T.float32(0.0)
input_embeds_red_temp_v1[v_ax0, v_ax1] = T.float32(0.0)
v_input_embeds_red_temp_v0: T.float32 = input_embeds_red_temp_v0[v_ax0, v_ax1] + T.Cast("float32", input_embeds[v_ax0, v_ax1, v_k2])
v_input_embeds_red_temp_v1: T.float32 = input_embeds_red_temp_v1[v_ax0, v_ax1] + T.Cast("float32", input_embeds[v_ax0, v_ax1, v_k2]) * T.Cast("float32", input_embeds[v_ax0, v_ax1, v_k2])
input_embeds_red_temp_v0[v_ax0, v_ax1] = v_input_embeds_red_temp_v0
input_embeds_red_temp_v1[v_ax0, v_ax1] = v_input_embeds_red_temp_v1
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("T_layer_norm"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(input_embeds[v_ax0, v_ax1, v_ax2], input_embeds_red_temp_v0[v_ax0, v_ax1], input_embeds_red_temp_v1[v_ax0, v_ax1], gpt_neox_layers_0_input_layernorm_weight3[v_ax2], gpt_neox_layers_0_input_layernorm_bias3[v_ax2])
T.writes(T_layer_norm[v_ax0, v_ax1, v_ax2])
T_layer_norm[v_ax0, v_ax1, v_ax2] = T.Cast("float16", (T.Cast("float32", input_embeds[v_ax0, v_ax1, v_ax2]) - input_embeds_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333)) * T.rsqrt(input_embeds_red_temp_v1[v_ax0, v_ax1] * T.float32(0.0013020833333333333) - input_embeds_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333) * (input_embeds_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight3[v_ax2] + gpt_neox_layers_0_input_layernorm_bias3[v_ax2]
@T.prim_func(private=True)
def layer_norm2(input_embed: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16"), gpt_neox_layers_0_input_layernorm_weight2: T.Buffer((T.int64(768),), "float16"), gpt_neox_layers_0_input_layernorm_bias2: T.Buffer((T.int64(768),), "float16"), T_layer_norm: T.Buffer((T.int64(1), T.int64(1), T.int64(768)), "float16")):
T.func_attr({"op_pattern": 4, "tir.noalias": T.bool(True)})
# with T.block("root"):
input_embed_red_temp_v0 = T.alloc_buffer((T.int64(1), T.int64(1)))
input_embed_red_temp_v1 = T.alloc_buffer((T.int64(1), T.int64(1)))
for ax0, ax1, k2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("input_embed_red_temp"):
v_ax0, v_ax1, v_k2 = T.axis.remap("SSR", [ax0, ax1, k2])
T.reads(input_embed[v_ax0, v_ax1, v_k2])
T.writes(input_embed_red_temp_v0[v_ax0, v_ax1], input_embed_red_temp_v1[v_ax0, v_ax1])
with T.init():
input_embed_red_temp_v0[v_ax0, v_ax1] = T.float32(0.0)
input_embed_red_temp_v1[v_ax0, v_ax1] = T.float32(0.0)
v_input_embed_red_temp_v0: T.float32 = input_embed_red_temp_v0[v_ax0, v_ax1] + T.Cast("float32", input_embed[v_ax0, v_ax1, v_k2])
v_input_embed_red_temp_v1: T.float32 = input_embed_red_temp_v1[v_ax0, v_ax1] + T.Cast("float32", input_embed[v_ax0, v_ax1, v_k2]) * T.Cast("float32", input_embed[v_ax0, v_ax1, v_k2])
input_embed_red_temp_v0[v_ax0, v_ax1] = v_input_embed_red_temp_v0
input_embed_red_temp_v1[v_ax0, v_ax1] = v_input_embed_red_temp_v1
for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(768)):
with T.block("T_layer_norm"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(input_embed[v_ax0, v_ax1, v_ax2], input_embed_red_temp_v0[v_ax0, v_ax1], input_embed_red_temp_v1[v_ax0, v_ax1], gpt_neox_layers_0_input_layernorm_weight2[v_ax2], gpt_neox_layers_0_input_layernorm_bias2[v_ax2])
T.writes(T_layer_norm[v_ax0, v_ax1, v_ax2])
T_layer_norm[v_ax0, v_ax1, v_ax2] = T.Cast("float16", (T.Cast("float32", input_embed[v_ax0, v_ax1, v_ax2]) - input_embed_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333)) * T.rsqrt(input_embed_red_temp_v1[v_ax0, v_ax1] * T.float32(0.0013020833333333333) - input_embed_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333) * (input_embed_red_temp_v0[v_ax0, v_ax1] * T.float32(0.0013020833333333333)) + T.float32(1.0000000000000001e-05))) * gpt_neox_layers_0_input_layernorm_weight2[v_ax2] + gpt_neox_layers_0_input_layernorm_bias2[v_ax2]
@T.prim_func
def merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True)
V = T.match_buffer(v, (N, H, D), "float16")
S = T.match_buffer(s, (N, H))
V_other = T.match_buffer(v_other, (N, H, D), "float16")
S_other = T.match_buffer(s_other, (N, H))
# with T.block("root"):
for bx in T.thread_binding(N, thread="blockIdx.x"):
for by in T.thread_binding(1, thread="blockIdx.y"):
for ty in T.thread_binding(12, thread="threadIdx.y"):
for tx in T.thread_binding(16, thread="threadIdx.x"):
with T.block("merge"):
T.reads(S[bx, ty + by * 12], S_other[bx, ty + by * 12], V[bx, ty + by * 12, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 12, tx * 4:tx * 4 + 4])
T.writes(V[bx, ty + by * 12, tx * 4:tx * 4 + 4], S[bx, ty + by * 12])
s_val = T.alloc_buffer((1,), scope="local")
s_other_val = T.alloc_buffer((1,), scope="local")
s_max = T.alloc_buffer((1,), scope="local")
scale = T.alloc_buffer((1,), scope="local")
other_scale = T.alloc_buffer((1,), scope="local")
v_vec = T.alloc_buffer((4,), "float16", scope="local")
v_other_vec = T.alloc_buffer((4,), "float16", scope="local")
s_val[0] = S[bx, ty + by * 12]
s_other_val[0] = S_other[bx, ty + by * 12]
s_max[0] = T.max(s_val[0], s_other_val[0])
s_val[0] = T.exp2(s_val[0] - s_max[0])
s_other_val[0] = T.exp2(s_other_val[0] - s_max[0])
scale[0] = s_val[0] / (s_val[0] + s_other_val[0])
other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0])
for vec in T.vectorized(4):
v_vec[vec] = V[bx, ty + by * 12, tx * 4 + vec]
for vec in T.vectorized(4):
v_other_vec[vec] = V_other[bx, ty + by * 12, tx * 4 + vec]
for vec in range(4):
v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0])
for vec in T.vectorized(4):
V[bx, ty + by * 12, tx * 4 + vec] = v_vec[vec]
S[bx, ty + by * 12] = T.log2(s_val[0] + s_other_val[0]) + s_max[0]
@T.prim_func(private=True)
def reshape(var_add216: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
batch_size = T.int64()
add216 = T.match_buffer(var_add216, (batch_size, T.int64(1), T.int64(2304)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (batch_size, T.int64(1), T.int64(36), T.int64(64)), "float16")
# with T.block("root"):
for ax0, ax1, ax2, ax3 in T.grid(batch_size, T.int64(1), T.int64(36), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap("SSSS", [ax0, ax1, ax2, ax3])
T.reads(add216[((v_ax2 * T.int64(64) + v_ax3) // T.int64(2304) + v_ax0 + v_ax1) % batch_size, T.int64(0), (v_ax2 * T.int64(64) + v_ax3) % T.int64(2304)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2, v_ax3])
T_reshape[v_ax0, v_ax1, v_ax2, v_ax3] = add216[((v_ax2 * T.int64(64) + v_ax3) // T.int64(2304) + v_ax0 + v_ax1) % batch_size, T.int64(0), (v_ax2 * T.int64(64) + v_ax3) % T.int64(2304)]
@T.prim_func(private=True)
def reshape1(var_reshape144: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
batch_size = T.int64()
reshape144 = T.match_buffer(var_reshape144, (batch_size, T.int64(1), T.int64(36), T.int64(64)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (batch_size, T.int64(36), T.int64(64)), "float16")
# with T.block("root"):
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(36), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(reshape144[((v_ax2 // T.int64(64) + v_ax1) // T.int64(36) + v_ax0) % batch_size, T.int64(0), (v_ax2 // T.int64(64) + v_ax1) % T.int64(36), v_ax2 % T.int64(64)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2])
T_reshape[v_ax0, v_ax1, v_ax2] = reshape144[((v_ax2 // T.int64(64) + v_ax1) // T.int64(36) + v_ax0) % batch_size, T.int64(0), (v_ax2 // T.int64(64) + v_ax1) % T.int64(36), v_ax2 % T.int64(64)]
@T.prim_func(private=True)
def reshape2(var_lv37: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
batch_size = T.int64()
lv37 = T.match_buffer(var_lv37, (batch_size, T.int64(12), T.int64(64)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (batch_size, T.int64(1), T.int64(12), T.int64(64)), "float16")
# with T.block("root"):
for ax0, ax1, ax2, ax3 in T.grid(batch_size, T.int64(1), T.int64(12), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap("SSSS", [ax0, ax1, ax2, ax3])
T.reads(lv37[((v_ax3 // T.int64(64) + v_ax2) // T.int64(12) + v_ax0 + v_ax1) % batch_size, (v_ax3 // T.int64(64) + v_ax2) % T.int64(12), v_ax3 % T.int64(64)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2, v_ax3])
T_reshape[v_ax0, v_ax1, v_ax2, v_ax3] = lv37[((v_ax3 // T.int64(64) + v_ax2) // T.int64(12) + v_ax0 + v_ax1) % batch_size, (v_ax3 // T.int64(64) + v_ax2) % T.int64(12), v_ax3 % T.int64(64)]
@T.prim_func(private=True)
def reshape3(var_reshape146: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
batch_size = T.int64()
reshape146 = T.match_buffer(var_reshape146, (batch_size, T.int64(1), T.int64(12), T.int64(64)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (batch_size, T.int64(1), T.int64(768)), "float16")
# with T.block("root"):
for ax0, ax1, ax2 in T.grid(batch_size, T.int64(1), T.int64(768)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(reshape146[(v_ax2 // T.int64(768) + v_ax0 + v_ax1) % batch_size, T.int64(0), v_ax2 % T.int64(768) // T.int64(64), v_ax2 % T.int64(64)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2])
T_reshape[v_ax0, v_ax1, v_ax2] = reshape146[(v_ax2 // T.int64(768) + v_ax0 + v_ax1) % batch_size, T.int64(0), v_ax2 % T.int64(768) // T.int64(64), v_ax2 % T.int64(64)]
@T.prim_func(private=True)
def reshape4(var_add144: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
seq_len = T.int64()
add144 = T.match_buffer(var_add144, (T.int64(1), seq_len, T.int64(2304)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (T.int64(1), seq_len, T.int64(36), T.int64(64)), "float16")
# with T.block("root"):
for ax0, ax1, ax2, ax3 in T.grid(T.int64(1), seq_len, T.int64(36), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap("SSSS", [ax0, ax1, ax2, ax3])
T.reads(add144[T.int64(0), ((v_ax2 * T.int64(64) + v_ax3) // T.int64(2304) + v_ax0 * seq_len + v_ax1) % seq_len, (v_ax2 * T.int64(64) + v_ax3) % T.int64(2304)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2, v_ax3])
T_reshape[v_ax0, v_ax1, v_ax2, v_ax3] = add144[T.int64(0), ((v_ax2 * T.int64(64) + v_ax3) // T.int64(2304) + v_ax0 * seq_len + v_ax1) % seq_len, (v_ax2 * T.int64(64) + v_ax3) % T.int64(2304)]
@T.prim_func(private=True)
def reshape5(var_reshape96: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
seq_len = T.int64()
reshape96 = T.match_buffer(var_reshape96, (T.int64(1), seq_len, T.int64(36), T.int64(64)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (seq_len, T.int64(36), T.int64(64)), "float16")
# with T.block("root"):
for ax0, ax1, ax2 in T.grid(seq_len, T.int64(36), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(reshape96[T.int64(0), ((v_ax2 // T.int64(64) + v_ax1) // T.int64(36) + v_ax0) % seq_len, (v_ax2 // T.int64(64) + v_ax1) % T.int64(36), v_ax2 % T.int64(64)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2])
T_reshape[v_ax0, v_ax1, v_ax2] = reshape96[T.int64(0), ((v_ax2 // T.int64(64) + v_ax1) // T.int64(36) + v_ax0) % seq_len, (v_ax2 // T.int64(64) + v_ax1) % T.int64(36), v_ax2 % T.int64(64)]
@T.prim_func(private=True)
def reshape6(var_lv25: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
seq_len = T.int64()
lv25 = T.match_buffer(var_lv25, (seq_len, T.int64(12), T.int64(64)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (T.int64(1), seq_len, T.int64(12), T.int64(64)), "float16")
# with T.block("root"):
for ax0, ax1, ax2, ax3 in T.grid(T.int64(1), seq_len, T.int64(12), T.int64(64)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap("SSSS", [ax0, ax1, ax2, ax3])
T.reads(lv25[((v_ax3 // T.int64(64) + v_ax2) // T.int64(12) + v_ax0 * seq_len + v_ax1) % seq_len, (v_ax3 // T.int64(64) + v_ax2) % T.int64(12), v_ax3 % T.int64(64)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2, v_ax3])
T_reshape[v_ax0, v_ax1, v_ax2, v_ax3] = lv25[((v_ax3 // T.int64(64) + v_ax2) // T.int64(12) + v_ax0 * seq_len + v_ax1) % seq_len, (v_ax3 // T.int64(64) + v_ax2) % T.int64(12), v_ax3 % T.int64(64)]
@T.prim_func(private=True)
def reshape7(var_reshape98: T.handle, var_T_reshape: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
seq_len = T.int64()
reshape98 = T.match_buffer(var_reshape98, (T.int64(1), seq_len, T.int64(12), T.int64(64)), "float16")
T_reshape = T.match_buffer(var_T_reshape, (T.int64(1), seq_len, T.int64(768)), "float16")
# with T.block("root"):
for ax0, ax1, ax2 in T.grid(T.int64(1), seq_len, T.int64(768)):
with T.block("T_reshape"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(reshape98[T.int64(0), (v_ax2 // T.int64(768) + v_ax0 * seq_len + v_ax1) % seq_len, v_ax2 % T.int64(768) // T.int64(64), v_ax2 % T.int64(64)])
T.writes(T_reshape[v_ax0, v_ax1, v_ax2])
T_reshape[v_ax0, v_ax1, v_ax2] = reshape98[T.int64(0), (v_ax2 // T.int64(768) + v_ax0 * seq_len + v_ax1) % seq_len, v_ax2 % T.int64(768) // T.int64(64), v_ax2 % T.int64(64)]
@T.prim_func
def scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True)
src = T.match_buffer(var_src, (batch_size, n))
indices = T.match_buffer(var_indices, (batch_size,), "int32")
m = T.int32(is_size_var=True)
dst = T.match_buffer(var_dst, (m, n))
# with T.block("root"):
for b, j in T.grid(batch_size, n):
with T.block("scatter_2d"):
vb, vj = T.axis.remap("SS", [b, j])
T.reads(src[vb, vj], indices[vb])
T.writes(dst[indices[vb], vj])
dst[indices[vb], vj] = src[vb, vj]
@T.prim_func
def softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True)
A = T.match_buffer(var_A, (batch_size, vocab_size))
temperature = T.match_buffer(var_temperature, (batch_size,))
num_chunks = T.int64(is_size_var=True)
chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks))
chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks))
softmax = T.match_buffer(var_softmax, (batch_size, vocab_size))
# with T.block("root"):
temp_max_shared = T.alloc_buffer((batch_size,), scope="shared")
temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared")
for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"):
for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"):
for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}):
with T.block("max"):
v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks)
v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1)
T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks)
T.reads(chunked_max[v0, v1])
T.writes(temp_max_shared[v0])
with T.init():
temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0)
temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1])
for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"):
for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}):
with T.block("sum_exp"):
v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks)
v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1)
T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks)
T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0])
T.writes(temp_sum_shared[v0])
with T.init():
temp_sum_shared[v0] = T.float32(0.0)
temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1])
for l2_0 in T.serial(T.int64(16), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}):
for l2_1 in T.thread_binding(T.int64(8), thread="threadIdx.y"):
for l2_2 in T.thread_binding(T.int64(32), thread="threadIdx.x"):
with T.block("log_pad"):
v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks)
v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks)
v2 = T.axis.spatial(T.int64(4096), l2_0 * T.int64(256) + l2_1 * T.int64(32) + l2_2)
T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2], temp_sum_shared[v0], temp_max_shared[v0])
T.writes(softmax[v0, v1 * T.int64(4096) + v2])
if v1 * T.int64(4096) + v2 < vocab_size:
softmax[v0, v1 * T.int64(4096) + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * T.int64(4096) + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * T.int64(4096) + v2] == temp_max_shared[v0]) / temp_sum_shared[v0])
@T.prim_func(private=True)
def take(var_layer_norm74: T.handle, var_logit_positions: T.handle, var_T_take: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
seq_len = T.int64()
layer_norm74 = T.match_buffer(var_layer_norm74, (T.int64(1), seq_len, T.int64(768)), "float16")
batch_size = T.int64()
logit_positions = T.match_buffer(var_logit_positions, (batch_size,), "int32")
T_take = T.match_buffer(var_T_take, (T.int64(1), batch_size, T.int64(768)), "float16")
# with T.block("root"):
for ax0, ax1, ax2 in T.grid(T.int64(1), batch_size, T.int64(768)):
with T.block("T_take"):
v_ax0, v_ax1, v_ax2 = T.axis.remap("SSS", [ax0, ax1, ax2])
T.reads(layer_norm74[v_ax0, logit_positions[v_ax1], v_ax2], logit_positions[v_ax1])
T.writes(T_take[v_ax0, v_ax1, v_ax2])
T_take[v_ax0, v_ax1, v_ax2] = layer_norm74[v_ax0, logit_positions[v_ax1], v_ax2]
@T.prim_func(private=True)
def take1(var_gpt_neox_embed_in_weight: T.handle, var_input_ids: T.handle, var_T_take: T.handle):
T.func_attr({"op_pattern": 8, "tir.noalias": T.bool(True)})
vocab_size = T.int64()
gpt_neox_embed_in_weight = T.match_buffer(var_gpt_neox_embed_in_weight, (vocab_size, T.int64(768)), "float16")
seq_len = T.int64()
input_ids = T.match_buffer(var_input_ids, (seq_len,), "int32")
T_take = T.match_buffer(var_T_take, (seq_len, T.int64(768)), "float16")
# with T.block("root"):
for ax0, ax1 in T.grid(seq_len, T.int64(768)):
with T.block("T_take"):
v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
T.reads(gpt_neox_embed_in_weight[input_ids[v_ax0], v_ax1], input_ids[v_ax0])
T.writes(T_take[v_ax0, v_ax1])
T_take[v_ax0, v_ax1] = gpt_neox_embed_in_weight[input_ids[v_ax0], v_ax1]
@T.prim_func
def tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int64):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
num_pages, page_size = T.int64(), T.int64(is_size_var=True)
pages = T.match_buffer(var_pages, (num_pages, 2, 12, page_size, 64), "float16", offset_factor=1)
seqlen = T.int64(is_size_var=True)
position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1)
k_data = T.match_buffer(var_k_data, (12, seqlen, 12, 64), "float16")
v_data = T.match_buffer(var_v_data, (12, seqlen, 12, 64), "float16")
# with T.block("root"):
for p, h, d in T.grid(seqlen, 12, 64):
with T.block("copy0"):
vp, vh, vd = T.axis.remap("SSS", [p, h, d])
T.reads(position_map[vp], pages[T.Cast("int64", position_map[vp]) // page_size, 0:2, vh, T.Cast("int64", position_map[vp]) % page_size, vd])
T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd])
position: T.int32 = position_map[vp]
k_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 0, vh, T.Cast("int64", position) % page_size, vd]
v_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 1, vh, T.Cast("int64", position) % page_size, vd]
@T.prim_func
def tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
num_pages = T.int64()
pages = T.match_buffer(var_pages, (num_pages, 2, 12, 16, 64), "float16", offset_factor=1)
ntoken = T.int64(is_size_var=True)
k_data = T.match_buffer(var_k_data, (ntoken, 12, 64), "float16")
v_data = T.match_buffer(var_v_data, (ntoken, 12, 64), "float16")
position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1)
# with T.block("root"):
for global_pos, h, f in T.grid(ntoken, 12, 64):
if position_map[global_pos] != -1:
with T.block("k_transpose_append"):
vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f])
T.reads(position_map[vgpos], k_data[vgpos, vh, vf])
T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf])
position: T.int32 = position_map[vgpos]
pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf]
with T.block("v_transpose_append"):
vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f])
T.reads(position_map[vgpos], v_data[vgpos, vh, vf])
T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf])
position: T.int32 = position_map[vgpos]
pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf]
@T.prim_func
def tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
total_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (total_len, 12, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(var_pages, (max_num_pages, 2, 12, 32, 64), "float16")
page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (total_len, 12, 64), "float16")
lse = T.match_buffer(var_lse, (total_len, 12))
tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1)
total_tree_order_len = T.int32(is_size_var=True)
tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1)
# with T.block("root"):
assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention."
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(12, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 32), scope="shared")
S_local = T.alloc_buffer((32, 32), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
cur_page_indptr_begin: T.int32 = page_indptr[b_idx]
cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0)
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 31) // 32):
L_kv_start: T.int32 = iterator_1 * 32
for lz_ly_fused_0 in range(4):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("K_load"):
i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32]
page_offset: T.int32 = seq_offset % 32
K_smem[i, j] = pages[page_no, 0, by, page_offset, j]
else:
K_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_ly_fused_0 in range(4):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("V_load"):
i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32]
page_offset: T.int32 = seq_offset % 32
V_smem[i, j] = pages[page_no, 1, by, page_offset, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64])
T.writes(S_local[0:32, 0:32])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 4):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
k = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 4):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(32):
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i])
T.writes(S_smem[row, 0:32])
for j in range(32):
if row < 32:
row_: T.int32 = LH_start + row
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(32):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k = T.axis.reduce(32, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@R.function
def alloc_embedding_tensor() -> R.Tensor((2048, 768), dtype="float16"):
R.func_attr({"relax.memory_plan_dynamic_func_output": True})
gv: R.Tensor((2048, 768), dtype="float16") = R.builtin.alloc_tensor(R.shape([2048, 768]), R.dtype("float16"), R.prim_value(0), R.str("global"))
return gv
@R.function
def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 768), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor(("vocab_size", 768), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object):
batch_size = T.int64()
vocab_size = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_layers_0_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[6]
gpt_neox_layers_0_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[9]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[10]
gpt_neox_layers_0_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[12]
gpt_neox_layers_1_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[13]
gpt_neox_layers_1_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[14]
gpt_neox_layers_1_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[15]
gpt_neox_layers_1_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[16]
gpt_neox_layers_1_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[17]
gpt_neox_layers_1_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[19]
gpt_neox_layers_1_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[21]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[22]
gpt_neox_layers_1_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[23]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[24]
gpt_neox_layers_2_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[25]
gpt_neox_layers_2_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[26]
gpt_neox_layers_2_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[27]
gpt_neox_layers_2_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[28]
gpt_neox_layers_2_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[29]
gpt_neox_layers_2_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[30]
gpt_neox_layers_2_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[31]
gpt_neox_layers_2_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[32]
gpt_neox_layers_2_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[33]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[34]
gpt_neox_layers_2_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[35]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[36]
gpt_neox_layers_3_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[37]
gpt_neox_layers_3_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[38]
gpt_neox_layers_3_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[39]
gpt_neox_layers_3_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[40]
gpt_neox_layers_3_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[41]
gpt_neox_layers_3_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[42]
gpt_neox_layers_3_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[43]
gpt_neox_layers_3_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[44]
gpt_neox_layers_3_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[45]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[46]
gpt_neox_layers_3_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[47]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[48]
gpt_neox_layers_4_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[49]
gpt_neox_layers_4_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[50]
gpt_neox_layers_4_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[51]
gpt_neox_layers_4_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[52]
gpt_neox_layers_4_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[53]
gpt_neox_layers_4_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[54]
gpt_neox_layers_4_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[55]
gpt_neox_layers_4_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[56]
gpt_neox_layers_4_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[57]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[58]
gpt_neox_layers_4_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[59]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[60]
gpt_neox_layers_5_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[61]
gpt_neox_layers_5_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[62]
gpt_neox_layers_5_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[63]
gpt_neox_layers_5_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[64]
gpt_neox_layers_5_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[65]
gpt_neox_layers_5_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[66]
gpt_neox_layers_5_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[67]
gpt_neox_layers_5_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[68]
gpt_neox_layers_5_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[69]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[70]
gpt_neox_layers_5_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[71]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[72]
gpt_neox_layers_6_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[73]
gpt_neox_layers_6_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[74]
gpt_neox_layers_6_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[75]
gpt_neox_layers_6_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[76]
gpt_neox_layers_6_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[77]
gpt_neox_layers_6_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[78]
gpt_neox_layers_6_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[79]
gpt_neox_layers_6_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[80]
gpt_neox_layers_6_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[81]
gpt_neox_layers_6_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[82]
gpt_neox_layers_6_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[83]
gpt_neox_layers_6_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[84]
gpt_neox_layers_7_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[85]
gpt_neox_layers_7_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[86]
gpt_neox_layers_7_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[87]
gpt_neox_layers_7_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[88]
gpt_neox_layers_7_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[89]
gpt_neox_layers_7_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[90]
gpt_neox_layers_7_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[91]
gpt_neox_layers_7_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[92]
gpt_neox_layers_7_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[93]
gpt_neox_layers_7_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[94]
gpt_neox_layers_7_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[95]
gpt_neox_layers_7_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[96]
gpt_neox_layers_8_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[97]
gpt_neox_layers_8_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[98]
gpt_neox_layers_8_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[99]
gpt_neox_layers_8_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[100]
gpt_neox_layers_8_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[101]
gpt_neox_layers_8_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[102]
gpt_neox_layers_8_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[103]
gpt_neox_layers_8_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[104]
gpt_neox_layers_8_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[105]
gpt_neox_layers_8_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[106]
gpt_neox_layers_8_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[107]
gpt_neox_layers_8_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[108]
gpt_neox_layers_9_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[109]
gpt_neox_layers_9_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[110]
gpt_neox_layers_9_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[111]
gpt_neox_layers_9_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[112]
gpt_neox_layers_9_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[113]
gpt_neox_layers_9_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[114]
gpt_neox_layers_9_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[115]
gpt_neox_layers_9_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[116]
gpt_neox_layers_9_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[117]
gpt_neox_layers_9_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[118]
gpt_neox_layers_9_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[119]
gpt_neox_layers_9_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[120]
gpt_neox_layers_10_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[121]
gpt_neox_layers_10_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[122]
gpt_neox_layers_10_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[123]
gpt_neox_layers_10_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[124]
gpt_neox_layers_10_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[125]
gpt_neox_layers_10_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[126]
gpt_neox_layers_10_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[127]
gpt_neox_layers_10_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[128]
gpt_neox_layers_10_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[129]
gpt_neox_layers_10_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[130]
gpt_neox_layers_10_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[131]
gpt_neox_layers_10_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[132]
gpt_neox_layers_11_input_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[133]
gpt_neox_layers_11_input_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[134]
gpt_neox_layers_11_post_attention_layernorm_weight4: R.Tensor((768,), dtype="float16") = packed_params[135]
gpt_neox_layers_11_post_attention_layernorm_bias4: R.Tensor((768,), dtype="float16") = packed_params[136]
gpt_neox_layers_11_attention_query_key_value_weight4: R.Tensor((2304, 768), dtype="float16") = packed_params[137]
gpt_neox_layers_11_attention_query_key_value_bias4: R.Tensor((2304,), dtype="float16") = packed_params[138]
gpt_neox_layers_11_attention_dense_weight4: R.Tensor((768, 768), dtype="float16") = packed_params[139]
gpt_neox_layers_11_attention_dense_bias4: R.Tensor((768,), dtype="float16") = packed_params[140]
gpt_neox_layers_11_mlp_dense_h_to_4h_weight4: R.Tensor((3072, 768), dtype="float16") = packed_params[141]
gpt_neox_layers_11_mlp_dense_h_to_4h_bias4: R.Tensor((3072,), dtype="float32") = packed_params[142]
gpt_neox_layers_11_mlp_dense_4h_to_h_weight4: R.Tensor((768, 3072), dtype="float16") = packed_params[143]
gpt_neox_layers_11_mlp_dense_4h_to_h_bias4: R.Tensor((768,), dtype="float32") = packed_params[144]
gpt_neox_final_layer_norm_weight4: R.Tensor((768,), dtype="float16") = packed_params[145]
gpt_neox_final_layer_norm_bias4: R.Tensor((768,), dtype="float16") = packed_params[146]
embed_out_weight4: R.Tensor((vocab_size, 768), dtype="float16") = packed_params[147]
layer_norm75 = R.call_tir(cls.layer_norm, (input_embeds, gpt_neox_layers_0_input_layernorm_weight4, gpt_neox_layers_0_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv = R.call_tir(cls.fused_NT_matmul_add, (layer_norm75, gpt_neox_layers_0_attention_query_key_value_weight4, gpt_neox_layers_0_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape144 = R.call_tir(cls.reshape, (lv,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape145 = R.call_tir(cls.reshape1, (reshape144,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv37 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape145), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape146 = R.call_tir(cls.reshape2, (lv37,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape147 = R.call_tir(cls.reshape3, (reshape146,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm76 = R.call_tir(cls.layer_norm, (input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight4, gpt_neox_layers_0_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv1 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm76, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv2 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv1, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv3 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape147, gpt_neox_layers_0_attention_dense_weight4, gpt_neox_layers_0_attention_dense_bias4, lv2, input_embeds), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm77 = R.call_tir(cls.layer_norm, (lv3, gpt_neox_layers_1_input_layernorm_weight4, gpt_neox_layers_1_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv4 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm77, gpt_neox_layers_1_attention_query_key_value_weight4, gpt_neox_layers_1_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape148 = R.call_tir(cls.reshape, (lv4,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape149 = R.call_tir(cls.reshape1, (reshape148,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv38 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape149), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape150 = R.call_tir(cls.reshape2, (lv38,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape151 = R.call_tir(cls.reshape3, (reshape150,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm78 = R.call_tir(cls.layer_norm, (lv3, gpt_neox_layers_1_post_attention_layernorm_weight4, gpt_neox_layers_1_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv5 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm78, gpt_neox_layers_1_mlp_dense_h_to_4h_weight4, gpt_neox_layers_1_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv6 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv5, gpt_neox_layers_1_mlp_dense_4h_to_h_weight4, gpt_neox_layers_1_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv7 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape151, gpt_neox_layers_1_attention_dense_weight4, gpt_neox_layers_1_attention_dense_bias4, lv6, lv3), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm79 = R.call_tir(cls.layer_norm, (lv7, gpt_neox_layers_2_input_layernorm_weight4, gpt_neox_layers_2_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv8 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm79, gpt_neox_layers_2_attention_query_key_value_weight4, gpt_neox_layers_2_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape152 = R.call_tir(cls.reshape, (lv8,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape153 = R.call_tir(cls.reshape1, (reshape152,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv39 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape153), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape154 = R.call_tir(cls.reshape2, (lv39,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape155 = R.call_tir(cls.reshape3, (reshape154,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm80 = R.call_tir(cls.layer_norm, (lv7, gpt_neox_layers_2_post_attention_layernorm_weight4, gpt_neox_layers_2_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv9 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm80, gpt_neox_layers_2_mlp_dense_h_to_4h_weight4, gpt_neox_layers_2_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv10 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv9, gpt_neox_layers_2_mlp_dense_4h_to_h_weight4, gpt_neox_layers_2_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv11 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape155, gpt_neox_layers_2_attention_dense_weight4, gpt_neox_layers_2_attention_dense_bias4, lv10, lv7), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm81 = R.call_tir(cls.layer_norm, (lv11, gpt_neox_layers_3_input_layernorm_weight4, gpt_neox_layers_3_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv12 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm81, gpt_neox_layers_3_attention_query_key_value_weight4, gpt_neox_layers_3_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape156 = R.call_tir(cls.reshape, (lv12,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape157 = R.call_tir(cls.reshape1, (reshape156,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv40 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape157), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape158 = R.call_tir(cls.reshape2, (lv40,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape159 = R.call_tir(cls.reshape3, (reshape158,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm82 = R.call_tir(cls.layer_norm, (lv11, gpt_neox_layers_3_post_attention_layernorm_weight4, gpt_neox_layers_3_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv13 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm82, gpt_neox_layers_3_mlp_dense_h_to_4h_weight4, gpt_neox_layers_3_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv14 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv13, gpt_neox_layers_3_mlp_dense_4h_to_h_weight4, gpt_neox_layers_3_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv15 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape159, gpt_neox_layers_3_attention_dense_weight4, gpt_neox_layers_3_attention_dense_bias4, lv14, lv11), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm83 = R.call_tir(cls.layer_norm, (lv15, gpt_neox_layers_4_input_layernorm_weight4, gpt_neox_layers_4_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv16 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm83, gpt_neox_layers_4_attention_query_key_value_weight4, gpt_neox_layers_4_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape160 = R.call_tir(cls.reshape, (lv16,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape161 = R.call_tir(cls.reshape1, (reshape160,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv41 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape161), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape162 = R.call_tir(cls.reshape2, (lv41,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape163 = R.call_tir(cls.reshape3, (reshape162,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm84 = R.call_tir(cls.layer_norm, (lv15, gpt_neox_layers_4_post_attention_layernorm_weight4, gpt_neox_layers_4_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv17 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm84, gpt_neox_layers_4_mlp_dense_h_to_4h_weight4, gpt_neox_layers_4_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv18 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv17, gpt_neox_layers_4_mlp_dense_4h_to_h_weight4, gpt_neox_layers_4_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv19 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape163, gpt_neox_layers_4_attention_dense_weight4, gpt_neox_layers_4_attention_dense_bias4, lv18, lv15), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm85 = R.call_tir(cls.layer_norm, (lv19, gpt_neox_layers_5_input_layernorm_weight4, gpt_neox_layers_5_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv20 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm85, gpt_neox_layers_5_attention_query_key_value_weight4, gpt_neox_layers_5_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape164 = R.call_tir(cls.reshape, (lv20,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape165 = R.call_tir(cls.reshape1, (reshape164,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv42 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape165), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape166 = R.call_tir(cls.reshape2, (lv42,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape167 = R.call_tir(cls.reshape3, (reshape166,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm86 = R.call_tir(cls.layer_norm, (lv19, gpt_neox_layers_5_post_attention_layernorm_weight4, gpt_neox_layers_5_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv21 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm86, gpt_neox_layers_5_mlp_dense_h_to_4h_weight4, gpt_neox_layers_5_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv22 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv21, gpt_neox_layers_5_mlp_dense_4h_to_h_weight4, gpt_neox_layers_5_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv23 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape167, gpt_neox_layers_5_attention_dense_weight4, gpt_neox_layers_5_attention_dense_bias4, lv22, lv19), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm87 = R.call_tir(cls.layer_norm, (lv23, gpt_neox_layers_6_input_layernorm_weight4, gpt_neox_layers_6_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv24 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm87, gpt_neox_layers_6_attention_query_key_value_weight4, gpt_neox_layers_6_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape168 = R.call_tir(cls.reshape, (lv24,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape169 = R.call_tir(cls.reshape1, (reshape168,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv43 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape169), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape170 = R.call_tir(cls.reshape2, (lv43,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape171 = R.call_tir(cls.reshape3, (reshape170,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm88 = R.call_tir(cls.layer_norm, (lv23, gpt_neox_layers_6_post_attention_layernorm_weight4, gpt_neox_layers_6_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv25 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm88, gpt_neox_layers_6_mlp_dense_h_to_4h_weight4, gpt_neox_layers_6_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv26 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv25, gpt_neox_layers_6_mlp_dense_4h_to_h_weight4, gpt_neox_layers_6_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv27 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape171, gpt_neox_layers_6_attention_dense_weight4, gpt_neox_layers_6_attention_dense_bias4, lv26, lv23), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm89 = R.call_tir(cls.layer_norm, (lv27, gpt_neox_layers_7_input_layernorm_weight4, gpt_neox_layers_7_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv28 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm89, gpt_neox_layers_7_attention_query_key_value_weight4, gpt_neox_layers_7_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape172 = R.call_tir(cls.reshape, (lv28,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape173 = R.call_tir(cls.reshape1, (reshape172,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv44 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape173), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape174 = R.call_tir(cls.reshape2, (lv44,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape175 = R.call_tir(cls.reshape3, (reshape174,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm90 = R.call_tir(cls.layer_norm, (lv27, gpt_neox_layers_7_post_attention_layernorm_weight4, gpt_neox_layers_7_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv29 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm90, gpt_neox_layers_7_mlp_dense_h_to_4h_weight4, gpt_neox_layers_7_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv30 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv29, gpt_neox_layers_7_mlp_dense_4h_to_h_weight4, gpt_neox_layers_7_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv31 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape175, gpt_neox_layers_7_attention_dense_weight4, gpt_neox_layers_7_attention_dense_bias4, lv30, lv27), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm91 = R.call_tir(cls.layer_norm, (lv31, gpt_neox_layers_8_input_layernorm_weight4, gpt_neox_layers_8_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv32 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm91, gpt_neox_layers_8_attention_query_key_value_weight4, gpt_neox_layers_8_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape176 = R.call_tir(cls.reshape, (lv32,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape177 = R.call_tir(cls.reshape1, (reshape176,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv45 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape177), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape178 = R.call_tir(cls.reshape2, (lv45,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape179 = R.call_tir(cls.reshape3, (reshape178,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm92 = R.call_tir(cls.layer_norm, (lv31, gpt_neox_layers_8_post_attention_layernorm_weight4, gpt_neox_layers_8_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv33 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm92, gpt_neox_layers_8_mlp_dense_h_to_4h_weight4, gpt_neox_layers_8_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv34 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv33, gpt_neox_layers_8_mlp_dense_4h_to_h_weight4, gpt_neox_layers_8_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv35 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape179, gpt_neox_layers_8_attention_dense_weight4, gpt_neox_layers_8_attention_dense_bias4, lv34, lv31), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm93 = R.call_tir(cls.layer_norm, (lv35, gpt_neox_layers_9_input_layernorm_weight4, gpt_neox_layers_9_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv36 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm93, gpt_neox_layers_9_attention_query_key_value_weight4, gpt_neox_layers_9_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape180 = R.call_tir(cls.reshape, (lv36,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape181 = R.call_tir(cls.reshape1, (reshape180,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv46 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape181), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape182 = R.call_tir(cls.reshape2, (lv46,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape183 = R.call_tir(cls.reshape3, (reshape182,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm94 = R.call_tir(cls.layer_norm, (lv35, gpt_neox_layers_9_post_attention_layernorm_weight4, gpt_neox_layers_9_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv37_1 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm94, gpt_neox_layers_9_mlp_dense_h_to_4h_weight4, gpt_neox_layers_9_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv38_1 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv37_1, gpt_neox_layers_9_mlp_dense_4h_to_h_weight4, gpt_neox_layers_9_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv39_1 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape183, gpt_neox_layers_9_attention_dense_weight4, gpt_neox_layers_9_attention_dense_bias4, lv38_1, lv35), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm95 = R.call_tir(cls.layer_norm, (lv39_1, gpt_neox_layers_10_input_layernorm_weight4, gpt_neox_layers_10_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv40_1 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm95, gpt_neox_layers_10_attention_query_key_value_weight4, gpt_neox_layers_10_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape184 = R.call_tir(cls.reshape, (lv40_1,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape185 = R.call_tir(cls.reshape1, (reshape184,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv47 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape185), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape186 = R.call_tir(cls.reshape2, (lv47,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape187 = R.call_tir(cls.reshape3, (reshape186,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm96 = R.call_tir(cls.layer_norm, (lv39_1, gpt_neox_layers_10_post_attention_layernorm_weight4, gpt_neox_layers_10_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv41_1 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm96, gpt_neox_layers_10_mlp_dense_h_to_4h_weight4, gpt_neox_layers_10_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv42_1 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv41_1, gpt_neox_layers_10_mlp_dense_4h_to_h_weight4, gpt_neox_layers_10_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv43_1 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape187, gpt_neox_layers_10_attention_dense_weight4, gpt_neox_layers_10_attention_dense_bias4, lv42_1, lv39_1), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm97 = R.call_tir(cls.layer_norm, (lv43_1, gpt_neox_layers_11_input_layernorm_weight4, gpt_neox_layers_11_input_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv44_1 = R.call_tir(cls.fused_NT_matmul_add, (layer_norm97, gpt_neox_layers_11_attention_query_key_value_weight4, gpt_neox_layers_11_attention_query_key_value_bias4), out_sinfo=R.Tensor((batch_size, 1, 2304), dtype="float16"))
reshape188 = R.call_tir(cls.reshape, (lv44_1,), out_sinfo=R.Tensor((batch_size, 1, 36, 64), dtype="float16"))
reshape189 = R.call_tir(cls.reshape1, (reshape188,), out_sinfo=R.Tensor((batch_size, 36, 64), dtype="float16"))
lv48 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape189), out_sinfo=R.Tensor((batch_size, 12, 64), dtype="float16"))
reshape190 = R.call_tir(cls.reshape2, (lv48,), out_sinfo=R.Tensor((batch_size, 1, 12, 64), dtype="float16"))
reshape191 = R.call_tir(cls.reshape3, (reshape190,), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm98 = R.call_tir(cls.layer_norm, (lv43_1, gpt_neox_layers_11_post_attention_layernorm_weight4, gpt_neox_layers_11_post_attention_layernorm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv45_1 = R.call_tir(cls.fused_NT_matmul2_add2_gelu_cast, (layer_norm98, gpt_neox_layers_11_mlp_dense_h_to_4h_weight4, gpt_neox_layers_11_mlp_dense_h_to_4h_bias4), out_sinfo=R.Tensor((batch_size, 1, 3072), dtype="float16"))
lv46_1 = R.call_tir(cls.fused_NT_matmul3_add3_cast1, (lv45_1, gpt_neox_layers_11_mlp_dense_4h_to_h_weight4, gpt_neox_layers_11_mlp_dense_4h_to_h_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv47_1 = R.call_tir(cls.fused_NT_matmul1_add1_add4_add4, (reshape191, gpt_neox_layers_11_attention_dense_weight4, gpt_neox_layers_11_attention_dense_bias4, lv46_1, lv43_1), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
layer_norm99 = R.call_tir(cls.layer_norm, (lv47_1, gpt_neox_final_layer_norm_weight4, gpt_neox_final_layer_norm_bias4), out_sinfo=R.Tensor((batch_size, 1, 768), dtype="float16"))
lv48_1 = R.call_tir(cls.fused_NT_matmul4_cast2, (layer_norm99, embed_out_weight4), out_sinfo=R.Tensor((batch_size, 1, vocab_size), dtype="float32"))
gv4: R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype="float32"), R.Object) = lv48_1, paged_kv_cache
R.output(gv4)
return gv4
@R.function
def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 768), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor(("vocab_size", 768), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object):
batch_size = T.int64()
vocab_size = T.int64()
seq_len = T.int64()
R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_layers_0_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[6]
gpt_neox_layers_0_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[9]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[10]
gpt_neox_layers_0_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[12]
gpt_neox_layers_1_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[13]
gpt_neox_layers_1_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[14]
gpt_neox_layers_1_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[15]
gpt_neox_layers_1_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[16]
gpt_neox_layers_1_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[17]
gpt_neox_layers_1_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[19]
gpt_neox_layers_1_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[21]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[22]
gpt_neox_layers_1_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[23]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[24]
gpt_neox_layers_2_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[25]
gpt_neox_layers_2_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[26]
gpt_neox_layers_2_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[27]
gpt_neox_layers_2_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[28]
gpt_neox_layers_2_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[29]
gpt_neox_layers_2_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[30]
gpt_neox_layers_2_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[31]
gpt_neox_layers_2_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[32]
gpt_neox_layers_2_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[33]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[34]
gpt_neox_layers_2_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[35]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[36]
gpt_neox_layers_3_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[37]
gpt_neox_layers_3_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[38]
gpt_neox_layers_3_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[39]
gpt_neox_layers_3_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[40]
gpt_neox_layers_3_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[41]
gpt_neox_layers_3_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[42]
gpt_neox_layers_3_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[43]
gpt_neox_layers_3_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[44]
gpt_neox_layers_3_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[45]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[46]
gpt_neox_layers_3_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[47]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[48]
gpt_neox_layers_4_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[49]
gpt_neox_layers_4_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[50]
gpt_neox_layers_4_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[51]
gpt_neox_layers_4_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[52]
gpt_neox_layers_4_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[53]
gpt_neox_layers_4_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[54]
gpt_neox_layers_4_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[55]
gpt_neox_layers_4_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[56]
gpt_neox_layers_4_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[57]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[58]
gpt_neox_layers_4_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[59]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[60]
gpt_neox_layers_5_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[61]
gpt_neox_layers_5_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[62]
gpt_neox_layers_5_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[63]
gpt_neox_layers_5_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[64]
gpt_neox_layers_5_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[65]
gpt_neox_layers_5_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[66]
gpt_neox_layers_5_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[67]
gpt_neox_layers_5_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[68]
gpt_neox_layers_5_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[69]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[70]
gpt_neox_layers_5_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[71]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[72]
gpt_neox_layers_6_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[73]
gpt_neox_layers_6_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[74]
gpt_neox_layers_6_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[75]
gpt_neox_layers_6_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[76]
gpt_neox_layers_6_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[77]
gpt_neox_layers_6_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[78]
gpt_neox_layers_6_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[79]
gpt_neox_layers_6_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[80]
gpt_neox_layers_6_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[81]
gpt_neox_layers_6_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[82]
gpt_neox_layers_6_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[83]
gpt_neox_layers_6_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[84]
gpt_neox_layers_7_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[85]
gpt_neox_layers_7_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[86]
gpt_neox_layers_7_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[87]
gpt_neox_layers_7_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[88]
gpt_neox_layers_7_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[89]
gpt_neox_layers_7_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[90]
gpt_neox_layers_7_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[91]
gpt_neox_layers_7_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[92]
gpt_neox_layers_7_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[93]
gpt_neox_layers_7_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[94]
gpt_neox_layers_7_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[95]
gpt_neox_layers_7_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[96]
gpt_neox_layers_8_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[97]
gpt_neox_layers_8_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[98]
gpt_neox_layers_8_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[99]
gpt_neox_layers_8_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[100]
gpt_neox_layers_8_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[101]
gpt_neox_layers_8_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[102]
gpt_neox_layers_8_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[103]
gpt_neox_layers_8_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[104]
gpt_neox_layers_8_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[105]
gpt_neox_layers_8_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[106]
gpt_neox_layers_8_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[107]
gpt_neox_layers_8_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[108]
gpt_neox_layers_9_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[109]
gpt_neox_layers_9_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[110]
gpt_neox_layers_9_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[111]
gpt_neox_layers_9_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[112]
gpt_neox_layers_9_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[113]
gpt_neox_layers_9_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[114]
gpt_neox_layers_9_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[115]
gpt_neox_layers_9_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[116]
gpt_neox_layers_9_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[117]
gpt_neox_layers_9_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[118]
gpt_neox_layers_9_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[119]
gpt_neox_layers_9_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[120]
gpt_neox_layers_10_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[121]
gpt_neox_layers_10_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[122]
gpt_neox_layers_10_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[123]
gpt_neox_layers_10_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[124]
gpt_neox_layers_10_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[125]
gpt_neox_layers_10_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[126]
gpt_neox_layers_10_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[127]
gpt_neox_layers_10_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[128]
gpt_neox_layers_10_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[129]
gpt_neox_layers_10_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[130]
gpt_neox_layers_10_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[131]
gpt_neox_layers_10_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[132]
gpt_neox_layers_11_input_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[133]
gpt_neox_layers_11_input_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[134]
gpt_neox_layers_11_post_attention_layernorm_weight3: R.Tensor((768,), dtype="float16") = packed_params[135]
gpt_neox_layers_11_post_attention_layernorm_bias3: R.Tensor((768,), dtype="float16") = packed_params[136]
gpt_neox_layers_11_attention_query_key_value_weight3: R.Tensor((2304, 768), dtype="float16") = packed_params[137]
gpt_neox_layers_11_attention_query_key_value_bias3: R.Tensor((2304,), dtype="float16") = packed_params[138]
gpt_neox_layers_11_attention_dense_weight3: R.Tensor((768, 768), dtype="float16") = packed_params[139]
gpt_neox_layers_11_attention_dense_bias3: R.Tensor((768,), dtype="float16") = packed_params[140]
gpt_neox_layers_11_mlp_dense_h_to_4h_weight3: R.Tensor((3072, 768), dtype="float16") = packed_params[141]
gpt_neox_layers_11_mlp_dense_h_to_4h_bias3: R.Tensor((3072,), dtype="float32") = packed_params[142]
gpt_neox_layers_11_mlp_dense_4h_to_h_weight3: R.Tensor((768, 3072), dtype="float16") = packed_params[143]
gpt_neox_layers_11_mlp_dense_4h_to_h_bias3: R.Tensor((768,), dtype="float32") = packed_params[144]
gpt_neox_final_layer_norm_weight3: R.Tensor((768,), dtype="float16") = packed_params[145]
gpt_neox_final_layer_norm_bias3: R.Tensor((768,), dtype="float16") = packed_params[146]
embed_out_weight3: R.Tensor((vocab_size, 768), dtype="float16") = packed_params[147]
layer_norm50 = R.call_tir(cls.layer_norm1, (input_embeds, gpt_neox_layers_0_input_layernorm_weight3, gpt_neox_layers_0_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv49 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm50, gpt_neox_layers_0_attention_query_key_value_weight3, gpt_neox_layers_0_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape96 = R.call_tir(cls.reshape4, (lv49,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape97 = R.call_tir(cls.reshape5, (reshape96,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv25 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape97), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape98 = R.call_tir(cls.reshape6, (lv25,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape99 = R.call_tir(cls.reshape7, (reshape98,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm51 = R.call_tir(cls.layer_norm1, (input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight3, gpt_neox_layers_0_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv50 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm51, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv51 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv50, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv52 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape99, gpt_neox_layers_0_attention_dense_weight3, gpt_neox_layers_0_attention_dense_bias3, lv51, input_embeds), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm52 = R.call_tir(cls.layer_norm1, (lv52, gpt_neox_layers_1_input_layernorm_weight3, gpt_neox_layers_1_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv53 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm52, gpt_neox_layers_1_attention_query_key_value_weight3, gpt_neox_layers_1_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape100 = R.call_tir(cls.reshape4, (lv53,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape101 = R.call_tir(cls.reshape5, (reshape100,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv26 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape101), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape102 = R.call_tir(cls.reshape6, (lv26,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape103 = R.call_tir(cls.reshape7, (reshape102,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm53 = R.call_tir(cls.layer_norm1, (lv52, gpt_neox_layers_1_post_attention_layernorm_weight3, gpt_neox_layers_1_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv54 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm53, gpt_neox_layers_1_mlp_dense_h_to_4h_weight3, gpt_neox_layers_1_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv55 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv54, gpt_neox_layers_1_mlp_dense_4h_to_h_weight3, gpt_neox_layers_1_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv56 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape103, gpt_neox_layers_1_attention_dense_weight3, gpt_neox_layers_1_attention_dense_bias3, lv55, lv52), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm54 = R.call_tir(cls.layer_norm1, (lv56, gpt_neox_layers_2_input_layernorm_weight3, gpt_neox_layers_2_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv57 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm54, gpt_neox_layers_2_attention_query_key_value_weight3, gpt_neox_layers_2_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape104 = R.call_tir(cls.reshape4, (lv57,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape105 = R.call_tir(cls.reshape5, (reshape104,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv27 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape105), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape106 = R.call_tir(cls.reshape6, (lv27,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape107 = R.call_tir(cls.reshape7, (reshape106,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm55 = R.call_tir(cls.layer_norm1, (lv56, gpt_neox_layers_2_post_attention_layernorm_weight3, gpt_neox_layers_2_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv58 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm55, gpt_neox_layers_2_mlp_dense_h_to_4h_weight3, gpt_neox_layers_2_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv59 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv58, gpt_neox_layers_2_mlp_dense_4h_to_h_weight3, gpt_neox_layers_2_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv60 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape107, gpt_neox_layers_2_attention_dense_weight3, gpt_neox_layers_2_attention_dense_bias3, lv59, lv56), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm56 = R.call_tir(cls.layer_norm1, (lv60, gpt_neox_layers_3_input_layernorm_weight3, gpt_neox_layers_3_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv61 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm56, gpt_neox_layers_3_attention_query_key_value_weight3, gpt_neox_layers_3_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape108 = R.call_tir(cls.reshape4, (lv61,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape109 = R.call_tir(cls.reshape5, (reshape108,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv28 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape109), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape110 = R.call_tir(cls.reshape6, (lv28,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape111 = R.call_tir(cls.reshape7, (reshape110,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm57 = R.call_tir(cls.layer_norm1, (lv60, gpt_neox_layers_3_post_attention_layernorm_weight3, gpt_neox_layers_3_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv62 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm57, gpt_neox_layers_3_mlp_dense_h_to_4h_weight3, gpt_neox_layers_3_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv63 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv62, gpt_neox_layers_3_mlp_dense_4h_to_h_weight3, gpt_neox_layers_3_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv64 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape111, gpt_neox_layers_3_attention_dense_weight3, gpt_neox_layers_3_attention_dense_bias3, lv63, lv60), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm58 = R.call_tir(cls.layer_norm1, (lv64, gpt_neox_layers_4_input_layernorm_weight3, gpt_neox_layers_4_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv65 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm58, gpt_neox_layers_4_attention_query_key_value_weight3, gpt_neox_layers_4_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape112 = R.call_tir(cls.reshape4, (lv65,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape113 = R.call_tir(cls.reshape5, (reshape112,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv29 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape113), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape114 = R.call_tir(cls.reshape6, (lv29,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape115 = R.call_tir(cls.reshape7, (reshape114,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm59 = R.call_tir(cls.layer_norm1, (lv64, gpt_neox_layers_4_post_attention_layernorm_weight3, gpt_neox_layers_4_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv66 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm59, gpt_neox_layers_4_mlp_dense_h_to_4h_weight3, gpt_neox_layers_4_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv67 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv66, gpt_neox_layers_4_mlp_dense_4h_to_h_weight3, gpt_neox_layers_4_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv68 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape115, gpt_neox_layers_4_attention_dense_weight3, gpt_neox_layers_4_attention_dense_bias3, lv67, lv64), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm60 = R.call_tir(cls.layer_norm1, (lv68, gpt_neox_layers_5_input_layernorm_weight3, gpt_neox_layers_5_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv69 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm60, gpt_neox_layers_5_attention_query_key_value_weight3, gpt_neox_layers_5_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape116 = R.call_tir(cls.reshape4, (lv69,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape117 = R.call_tir(cls.reshape5, (reshape116,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv30 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape117), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape118 = R.call_tir(cls.reshape6, (lv30,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape119 = R.call_tir(cls.reshape7, (reshape118,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm61 = R.call_tir(cls.layer_norm1, (lv68, gpt_neox_layers_5_post_attention_layernorm_weight3, gpt_neox_layers_5_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv70 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm61, gpt_neox_layers_5_mlp_dense_h_to_4h_weight3, gpt_neox_layers_5_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv71 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv70, gpt_neox_layers_5_mlp_dense_4h_to_h_weight3, gpt_neox_layers_5_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv72 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape119, gpt_neox_layers_5_attention_dense_weight3, gpt_neox_layers_5_attention_dense_bias3, lv71, lv68), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm62 = R.call_tir(cls.layer_norm1, (lv72, gpt_neox_layers_6_input_layernorm_weight3, gpt_neox_layers_6_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv73 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm62, gpt_neox_layers_6_attention_query_key_value_weight3, gpt_neox_layers_6_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape120 = R.call_tir(cls.reshape4, (lv73,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape121 = R.call_tir(cls.reshape5, (reshape120,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv31 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape121), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape122 = R.call_tir(cls.reshape6, (lv31,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape123 = R.call_tir(cls.reshape7, (reshape122,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm63 = R.call_tir(cls.layer_norm1, (lv72, gpt_neox_layers_6_post_attention_layernorm_weight3, gpt_neox_layers_6_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv74 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm63, gpt_neox_layers_6_mlp_dense_h_to_4h_weight3, gpt_neox_layers_6_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv75 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv74, gpt_neox_layers_6_mlp_dense_4h_to_h_weight3, gpt_neox_layers_6_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv76 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape123, gpt_neox_layers_6_attention_dense_weight3, gpt_neox_layers_6_attention_dense_bias3, lv75, lv72), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm64 = R.call_tir(cls.layer_norm1, (lv76, gpt_neox_layers_7_input_layernorm_weight3, gpt_neox_layers_7_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv77 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm64, gpt_neox_layers_7_attention_query_key_value_weight3, gpt_neox_layers_7_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape124 = R.call_tir(cls.reshape4, (lv77,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape125 = R.call_tir(cls.reshape5, (reshape124,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv32 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape125), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape126 = R.call_tir(cls.reshape6, (lv32,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape127 = R.call_tir(cls.reshape7, (reshape126,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm65 = R.call_tir(cls.layer_norm1, (lv76, gpt_neox_layers_7_post_attention_layernorm_weight3, gpt_neox_layers_7_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv78 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm65, gpt_neox_layers_7_mlp_dense_h_to_4h_weight3, gpt_neox_layers_7_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv79 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv78, gpt_neox_layers_7_mlp_dense_4h_to_h_weight3, gpt_neox_layers_7_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv80 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape127, gpt_neox_layers_7_attention_dense_weight3, gpt_neox_layers_7_attention_dense_bias3, lv79, lv76), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm66 = R.call_tir(cls.layer_norm1, (lv80, gpt_neox_layers_8_input_layernorm_weight3, gpt_neox_layers_8_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv81 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm66, gpt_neox_layers_8_attention_query_key_value_weight3, gpt_neox_layers_8_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape128 = R.call_tir(cls.reshape4, (lv81,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape129 = R.call_tir(cls.reshape5, (reshape128,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv33 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape129), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape130 = R.call_tir(cls.reshape6, (lv33,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape131 = R.call_tir(cls.reshape7, (reshape130,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm67 = R.call_tir(cls.layer_norm1, (lv80, gpt_neox_layers_8_post_attention_layernorm_weight3, gpt_neox_layers_8_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv82 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm67, gpt_neox_layers_8_mlp_dense_h_to_4h_weight3, gpt_neox_layers_8_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv83 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv82, gpt_neox_layers_8_mlp_dense_4h_to_h_weight3, gpt_neox_layers_8_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv84 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape131, gpt_neox_layers_8_attention_dense_weight3, gpt_neox_layers_8_attention_dense_bias3, lv83, lv80), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm68 = R.call_tir(cls.layer_norm1, (lv84, gpt_neox_layers_9_input_layernorm_weight3, gpt_neox_layers_9_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv85 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm68, gpt_neox_layers_9_attention_query_key_value_weight3, gpt_neox_layers_9_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape132 = R.call_tir(cls.reshape4, (lv85,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape133 = R.call_tir(cls.reshape5, (reshape132,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv34 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape133), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape134 = R.call_tir(cls.reshape6, (lv34,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape135 = R.call_tir(cls.reshape7, (reshape134,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm69 = R.call_tir(cls.layer_norm1, (lv84, gpt_neox_layers_9_post_attention_layernorm_weight3, gpt_neox_layers_9_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv86 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm69, gpt_neox_layers_9_mlp_dense_h_to_4h_weight3, gpt_neox_layers_9_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv87 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv86, gpt_neox_layers_9_mlp_dense_4h_to_h_weight3, gpt_neox_layers_9_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv88 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape135, gpt_neox_layers_9_attention_dense_weight3, gpt_neox_layers_9_attention_dense_bias3, lv87, lv84), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm70 = R.call_tir(cls.layer_norm1, (lv88, gpt_neox_layers_10_input_layernorm_weight3, gpt_neox_layers_10_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv89 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm70, gpt_neox_layers_10_attention_query_key_value_weight3, gpt_neox_layers_10_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape136 = R.call_tir(cls.reshape4, (lv89,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape137 = R.call_tir(cls.reshape5, (reshape136,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv35 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape137), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape138 = R.call_tir(cls.reshape6, (lv35,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape139 = R.call_tir(cls.reshape7, (reshape138,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm71 = R.call_tir(cls.layer_norm1, (lv88, gpt_neox_layers_10_post_attention_layernorm_weight3, gpt_neox_layers_10_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv90 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm71, gpt_neox_layers_10_mlp_dense_h_to_4h_weight3, gpt_neox_layers_10_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv91 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv90, gpt_neox_layers_10_mlp_dense_4h_to_h_weight3, gpt_neox_layers_10_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv92 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape139, gpt_neox_layers_10_attention_dense_weight3, gpt_neox_layers_10_attention_dense_bias3, lv91, lv88), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm72 = R.call_tir(cls.layer_norm1, (lv92, gpt_neox_layers_11_input_layernorm_weight3, gpt_neox_layers_11_input_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv93 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm72, gpt_neox_layers_11_attention_query_key_value_weight3, gpt_neox_layers_11_attention_query_key_value_bias3), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape140 = R.call_tir(cls.reshape4, (lv93,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape141 = R.call_tir(cls.reshape5, (reshape140,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv36 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape141), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape142 = R.call_tir(cls.reshape6, (lv36,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape143 = R.call_tir(cls.reshape7, (reshape142,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm73 = R.call_tir(cls.layer_norm1, (lv92, gpt_neox_layers_11_post_attention_layernorm_weight3, gpt_neox_layers_11_post_attention_layernorm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv94 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm73, gpt_neox_layers_11_mlp_dense_h_to_4h_weight3, gpt_neox_layers_11_mlp_dense_h_to_4h_bias3), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv95 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv94, gpt_neox_layers_11_mlp_dense_4h_to_h_weight3, gpt_neox_layers_11_mlp_dense_4h_to_h_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv96 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape143, gpt_neox_layers_11_attention_dense_weight3, gpt_neox_layers_11_attention_dense_bias3, lv95, lv92), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm74 = R.call_tir(cls.layer_norm1, (lv96, gpt_neox_final_layer_norm_weight3, gpt_neox_final_layer_norm_bias3), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
take1 = R.call_tir(cls.take, (layer_norm74, logit_positions), out_sinfo=R.Tensor((1, batch_size, 768), dtype="float16"))
lv97 = R.call_tir(cls.fused_NT_matmul9_cast5, (take1, embed_out_weight3), out_sinfo=R.Tensor((1, batch_size, vocab_size), dtype="float32"))
gv3: R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype="float32"), R.Object) = lv97, paged_kv_cache
R.output(gv3)
return gv3
@R.function
def batch_verify(input_embeds: R.Tensor((1, "seq_len", 768), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor(("vocab_size", 768), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object):
seq_len = T.int64()
vocab_size = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_layers_0_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[6]
gpt_neox_layers_0_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[9]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[10]
gpt_neox_layers_0_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[12]
gpt_neox_layers_1_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[13]
gpt_neox_layers_1_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[14]
gpt_neox_layers_1_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[15]
gpt_neox_layers_1_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[16]
gpt_neox_layers_1_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[17]
gpt_neox_layers_1_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[19]
gpt_neox_layers_1_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[21]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[22]
gpt_neox_layers_1_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[23]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[24]
gpt_neox_layers_2_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[25]
gpt_neox_layers_2_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[26]
gpt_neox_layers_2_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[27]
gpt_neox_layers_2_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[28]
gpt_neox_layers_2_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[29]
gpt_neox_layers_2_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[30]
gpt_neox_layers_2_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[31]
gpt_neox_layers_2_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[32]
gpt_neox_layers_2_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[33]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[34]
gpt_neox_layers_2_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[35]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[36]
gpt_neox_layers_3_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[37]
gpt_neox_layers_3_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[38]
gpt_neox_layers_3_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[39]
gpt_neox_layers_3_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[40]
gpt_neox_layers_3_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[41]
gpt_neox_layers_3_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[42]
gpt_neox_layers_3_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[43]
gpt_neox_layers_3_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[44]
gpt_neox_layers_3_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[45]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[46]
gpt_neox_layers_3_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[47]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[48]
gpt_neox_layers_4_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[49]
gpt_neox_layers_4_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[50]
gpt_neox_layers_4_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[51]
gpt_neox_layers_4_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[52]
gpt_neox_layers_4_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[53]
gpt_neox_layers_4_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[54]
gpt_neox_layers_4_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[55]
gpt_neox_layers_4_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[56]
gpt_neox_layers_4_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[57]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[58]
gpt_neox_layers_4_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[59]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[60]
gpt_neox_layers_5_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[61]
gpt_neox_layers_5_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[62]
gpt_neox_layers_5_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[63]
gpt_neox_layers_5_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[64]
gpt_neox_layers_5_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[65]
gpt_neox_layers_5_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[66]
gpt_neox_layers_5_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[67]
gpt_neox_layers_5_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[68]
gpt_neox_layers_5_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[69]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[70]
gpt_neox_layers_5_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[71]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[72]
gpt_neox_layers_6_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[73]
gpt_neox_layers_6_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[74]
gpt_neox_layers_6_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[75]
gpt_neox_layers_6_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[76]
gpt_neox_layers_6_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[77]
gpt_neox_layers_6_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[78]
gpt_neox_layers_6_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[79]
gpt_neox_layers_6_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[80]
gpt_neox_layers_6_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[81]
gpt_neox_layers_6_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[82]
gpt_neox_layers_6_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[83]
gpt_neox_layers_6_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[84]
gpt_neox_layers_7_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[85]
gpt_neox_layers_7_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[86]
gpt_neox_layers_7_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[87]
gpt_neox_layers_7_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[88]
gpt_neox_layers_7_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[89]
gpt_neox_layers_7_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[90]
gpt_neox_layers_7_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[91]
gpt_neox_layers_7_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[92]
gpt_neox_layers_7_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[93]
gpt_neox_layers_7_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[94]
gpt_neox_layers_7_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[95]
gpt_neox_layers_7_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[96]
gpt_neox_layers_8_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[97]
gpt_neox_layers_8_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[98]
gpt_neox_layers_8_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[99]
gpt_neox_layers_8_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[100]
gpt_neox_layers_8_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[101]
gpt_neox_layers_8_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[102]
gpt_neox_layers_8_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[103]
gpt_neox_layers_8_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[104]
gpt_neox_layers_8_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[105]
gpt_neox_layers_8_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[106]
gpt_neox_layers_8_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[107]
gpt_neox_layers_8_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[108]
gpt_neox_layers_9_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[109]
gpt_neox_layers_9_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[110]
gpt_neox_layers_9_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[111]
gpt_neox_layers_9_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[112]
gpt_neox_layers_9_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[113]
gpt_neox_layers_9_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[114]
gpt_neox_layers_9_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[115]
gpt_neox_layers_9_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[116]
gpt_neox_layers_9_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[117]
gpt_neox_layers_9_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[118]
gpt_neox_layers_9_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[119]
gpt_neox_layers_9_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[120]
gpt_neox_layers_10_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[121]
gpt_neox_layers_10_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[122]
gpt_neox_layers_10_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[123]
gpt_neox_layers_10_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[124]
gpt_neox_layers_10_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[125]
gpt_neox_layers_10_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[126]
gpt_neox_layers_10_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[127]
gpt_neox_layers_10_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[128]
gpt_neox_layers_10_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[129]
gpt_neox_layers_10_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[130]
gpt_neox_layers_10_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[131]
gpt_neox_layers_10_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[132]
gpt_neox_layers_11_input_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[133]
gpt_neox_layers_11_input_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[134]
gpt_neox_layers_11_post_attention_layernorm_weight5: R.Tensor((768,), dtype="float16") = packed_params[135]
gpt_neox_layers_11_post_attention_layernorm_bias5: R.Tensor((768,), dtype="float16") = packed_params[136]
gpt_neox_layers_11_attention_query_key_value_weight5: R.Tensor((2304, 768), dtype="float16") = packed_params[137]
gpt_neox_layers_11_attention_query_key_value_bias5: R.Tensor((2304,), dtype="float16") = packed_params[138]
gpt_neox_layers_11_attention_dense_weight5: R.Tensor((768, 768), dtype="float16") = packed_params[139]
gpt_neox_layers_11_attention_dense_bias5: R.Tensor((768,), dtype="float16") = packed_params[140]
gpt_neox_layers_11_mlp_dense_h_to_4h_weight5: R.Tensor((3072, 768), dtype="float16") = packed_params[141]
gpt_neox_layers_11_mlp_dense_h_to_4h_bias5: R.Tensor((3072,), dtype="float32") = packed_params[142]
gpt_neox_layers_11_mlp_dense_4h_to_h_weight5: R.Tensor((768, 3072), dtype="float16") = packed_params[143]
gpt_neox_layers_11_mlp_dense_4h_to_h_bias5: R.Tensor((768,), dtype="float32") = packed_params[144]
gpt_neox_final_layer_norm_weight5: R.Tensor((768,), dtype="float16") = packed_params[145]
gpt_neox_final_layer_norm_bias5: R.Tensor((768,), dtype="float16") = packed_params[146]
embed_out_weight5: R.Tensor((vocab_size, 768), dtype="float16") = packed_params[147]
layer_norm100 = R.call_tir(cls.layer_norm1, (input_embeds, gpt_neox_layers_0_input_layernorm_weight5, gpt_neox_layers_0_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv98 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm100, gpt_neox_layers_0_attention_query_key_value_weight5, gpt_neox_layers_0_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape192 = R.call_tir(cls.reshape4, (lv98,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape193 = R.call_tir(cls.reshape5, (reshape192,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv49 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape193), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape194 = R.call_tir(cls.reshape6, (lv49,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape195 = R.call_tir(cls.reshape7, (reshape194,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm101 = R.call_tir(cls.layer_norm1, (input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight5, gpt_neox_layers_0_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv99 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm101, gpt_neox_layers_0_mlp_dense_h_to_4h_weight5, gpt_neox_layers_0_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv100 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv99, gpt_neox_layers_0_mlp_dense_4h_to_h_weight5, gpt_neox_layers_0_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv101 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape195, gpt_neox_layers_0_attention_dense_weight5, gpt_neox_layers_0_attention_dense_bias5, lv100, input_embeds), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm102 = R.call_tir(cls.layer_norm1, (lv101, gpt_neox_layers_1_input_layernorm_weight5, gpt_neox_layers_1_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv102 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm102, gpt_neox_layers_1_attention_query_key_value_weight5, gpt_neox_layers_1_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape196 = R.call_tir(cls.reshape4, (lv102,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape197 = R.call_tir(cls.reshape5, (reshape196,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv50 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape197), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape198 = R.call_tir(cls.reshape6, (lv50,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape199 = R.call_tir(cls.reshape7, (reshape198,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm103 = R.call_tir(cls.layer_norm1, (lv101, gpt_neox_layers_1_post_attention_layernorm_weight5, gpt_neox_layers_1_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv103 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm103, gpt_neox_layers_1_mlp_dense_h_to_4h_weight5, gpt_neox_layers_1_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv104 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv103, gpt_neox_layers_1_mlp_dense_4h_to_h_weight5, gpt_neox_layers_1_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv105 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape199, gpt_neox_layers_1_attention_dense_weight5, gpt_neox_layers_1_attention_dense_bias5, lv104, lv101), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm104 = R.call_tir(cls.layer_norm1, (lv105, gpt_neox_layers_2_input_layernorm_weight5, gpt_neox_layers_2_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv106 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm104, gpt_neox_layers_2_attention_query_key_value_weight5, gpt_neox_layers_2_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape200 = R.call_tir(cls.reshape4, (lv106,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape201 = R.call_tir(cls.reshape5, (reshape200,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv51 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape201), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape202 = R.call_tir(cls.reshape6, (lv51,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape203 = R.call_tir(cls.reshape7, (reshape202,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm105 = R.call_tir(cls.layer_norm1, (lv105, gpt_neox_layers_2_post_attention_layernorm_weight5, gpt_neox_layers_2_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv107 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm105, gpt_neox_layers_2_mlp_dense_h_to_4h_weight5, gpt_neox_layers_2_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv108 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv107, gpt_neox_layers_2_mlp_dense_4h_to_h_weight5, gpt_neox_layers_2_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv109 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape203, gpt_neox_layers_2_attention_dense_weight5, gpt_neox_layers_2_attention_dense_bias5, lv108, lv105), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm106 = R.call_tir(cls.layer_norm1, (lv109, gpt_neox_layers_3_input_layernorm_weight5, gpt_neox_layers_3_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv110 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm106, gpt_neox_layers_3_attention_query_key_value_weight5, gpt_neox_layers_3_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape204 = R.call_tir(cls.reshape4, (lv110,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape205 = R.call_tir(cls.reshape5, (reshape204,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv52 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape205), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape206 = R.call_tir(cls.reshape6, (lv52,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape207 = R.call_tir(cls.reshape7, (reshape206,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm107 = R.call_tir(cls.layer_norm1, (lv109, gpt_neox_layers_3_post_attention_layernorm_weight5, gpt_neox_layers_3_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv111 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm107, gpt_neox_layers_3_mlp_dense_h_to_4h_weight5, gpt_neox_layers_3_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv112 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv111, gpt_neox_layers_3_mlp_dense_4h_to_h_weight5, gpt_neox_layers_3_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv113 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape207, gpt_neox_layers_3_attention_dense_weight5, gpt_neox_layers_3_attention_dense_bias5, lv112, lv109), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm108 = R.call_tir(cls.layer_norm1, (lv113, gpt_neox_layers_4_input_layernorm_weight5, gpt_neox_layers_4_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv114 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm108, gpt_neox_layers_4_attention_query_key_value_weight5, gpt_neox_layers_4_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape208 = R.call_tir(cls.reshape4, (lv114,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape209 = R.call_tir(cls.reshape5, (reshape208,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv53 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape209), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape210 = R.call_tir(cls.reshape6, (lv53,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape211 = R.call_tir(cls.reshape7, (reshape210,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm109 = R.call_tir(cls.layer_norm1, (lv113, gpt_neox_layers_4_post_attention_layernorm_weight5, gpt_neox_layers_4_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv115 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm109, gpt_neox_layers_4_mlp_dense_h_to_4h_weight5, gpt_neox_layers_4_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv116 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv115, gpt_neox_layers_4_mlp_dense_4h_to_h_weight5, gpt_neox_layers_4_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv117 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape211, gpt_neox_layers_4_attention_dense_weight5, gpt_neox_layers_4_attention_dense_bias5, lv116, lv113), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm110 = R.call_tir(cls.layer_norm1, (lv117, gpt_neox_layers_5_input_layernorm_weight5, gpt_neox_layers_5_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv118 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm110, gpt_neox_layers_5_attention_query_key_value_weight5, gpt_neox_layers_5_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape212 = R.call_tir(cls.reshape4, (lv118,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape213 = R.call_tir(cls.reshape5, (reshape212,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv54 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape213), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape214 = R.call_tir(cls.reshape6, (lv54,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape215 = R.call_tir(cls.reshape7, (reshape214,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm111 = R.call_tir(cls.layer_norm1, (lv117, gpt_neox_layers_5_post_attention_layernorm_weight5, gpt_neox_layers_5_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv119 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm111, gpt_neox_layers_5_mlp_dense_h_to_4h_weight5, gpt_neox_layers_5_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv120 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv119, gpt_neox_layers_5_mlp_dense_4h_to_h_weight5, gpt_neox_layers_5_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv121 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape215, gpt_neox_layers_5_attention_dense_weight5, gpt_neox_layers_5_attention_dense_bias5, lv120, lv117), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm112 = R.call_tir(cls.layer_norm1, (lv121, gpt_neox_layers_6_input_layernorm_weight5, gpt_neox_layers_6_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv122 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm112, gpt_neox_layers_6_attention_query_key_value_weight5, gpt_neox_layers_6_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape216 = R.call_tir(cls.reshape4, (lv122,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape217 = R.call_tir(cls.reshape5, (reshape216,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv55 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape217), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape218 = R.call_tir(cls.reshape6, (lv55,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape219 = R.call_tir(cls.reshape7, (reshape218,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm113 = R.call_tir(cls.layer_norm1, (lv121, gpt_neox_layers_6_post_attention_layernorm_weight5, gpt_neox_layers_6_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv123 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm113, gpt_neox_layers_6_mlp_dense_h_to_4h_weight5, gpt_neox_layers_6_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv124 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv123, gpt_neox_layers_6_mlp_dense_4h_to_h_weight5, gpt_neox_layers_6_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv125 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape219, gpt_neox_layers_6_attention_dense_weight5, gpt_neox_layers_6_attention_dense_bias5, lv124, lv121), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm114 = R.call_tir(cls.layer_norm1, (lv125, gpt_neox_layers_7_input_layernorm_weight5, gpt_neox_layers_7_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv126 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm114, gpt_neox_layers_7_attention_query_key_value_weight5, gpt_neox_layers_7_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape220 = R.call_tir(cls.reshape4, (lv126,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape221 = R.call_tir(cls.reshape5, (reshape220,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv56 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape221), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape222 = R.call_tir(cls.reshape6, (lv56,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape223 = R.call_tir(cls.reshape7, (reshape222,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm115 = R.call_tir(cls.layer_norm1, (lv125, gpt_neox_layers_7_post_attention_layernorm_weight5, gpt_neox_layers_7_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv127 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm115, gpt_neox_layers_7_mlp_dense_h_to_4h_weight5, gpt_neox_layers_7_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv128 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv127, gpt_neox_layers_7_mlp_dense_4h_to_h_weight5, gpt_neox_layers_7_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv129 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape223, gpt_neox_layers_7_attention_dense_weight5, gpt_neox_layers_7_attention_dense_bias5, lv128, lv125), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm116 = R.call_tir(cls.layer_norm1, (lv129, gpt_neox_layers_8_input_layernorm_weight5, gpt_neox_layers_8_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv130 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm116, gpt_neox_layers_8_attention_query_key_value_weight5, gpt_neox_layers_8_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape224 = R.call_tir(cls.reshape4, (lv130,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape225 = R.call_tir(cls.reshape5, (reshape224,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv57 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape225), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape226 = R.call_tir(cls.reshape6, (lv57,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape227 = R.call_tir(cls.reshape7, (reshape226,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm117 = R.call_tir(cls.layer_norm1, (lv129, gpt_neox_layers_8_post_attention_layernorm_weight5, gpt_neox_layers_8_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv131 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm117, gpt_neox_layers_8_mlp_dense_h_to_4h_weight5, gpt_neox_layers_8_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv132 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv131, gpt_neox_layers_8_mlp_dense_4h_to_h_weight5, gpt_neox_layers_8_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv133 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape227, gpt_neox_layers_8_attention_dense_weight5, gpt_neox_layers_8_attention_dense_bias5, lv132, lv129), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm118 = R.call_tir(cls.layer_norm1, (lv133, gpt_neox_layers_9_input_layernorm_weight5, gpt_neox_layers_9_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv134 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm118, gpt_neox_layers_9_attention_query_key_value_weight5, gpt_neox_layers_9_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape228 = R.call_tir(cls.reshape4, (lv134,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape229 = R.call_tir(cls.reshape5, (reshape228,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv58 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape229), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape230 = R.call_tir(cls.reshape6, (lv58,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape231 = R.call_tir(cls.reshape7, (reshape230,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm119 = R.call_tir(cls.layer_norm1, (lv133, gpt_neox_layers_9_post_attention_layernorm_weight5, gpt_neox_layers_9_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv135 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm119, gpt_neox_layers_9_mlp_dense_h_to_4h_weight5, gpt_neox_layers_9_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv136 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv135, gpt_neox_layers_9_mlp_dense_4h_to_h_weight5, gpt_neox_layers_9_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv137 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape231, gpt_neox_layers_9_attention_dense_weight5, gpt_neox_layers_9_attention_dense_bias5, lv136, lv133), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm120 = R.call_tir(cls.layer_norm1, (lv137, gpt_neox_layers_10_input_layernorm_weight5, gpt_neox_layers_10_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv138 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm120, gpt_neox_layers_10_attention_query_key_value_weight5, gpt_neox_layers_10_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape232 = R.call_tir(cls.reshape4, (lv138,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape233 = R.call_tir(cls.reshape5, (reshape232,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv59 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape233), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape234 = R.call_tir(cls.reshape6, (lv59,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape235 = R.call_tir(cls.reshape7, (reshape234,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm121 = R.call_tir(cls.layer_norm1, (lv137, gpt_neox_layers_10_post_attention_layernorm_weight5, gpt_neox_layers_10_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv139 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm121, gpt_neox_layers_10_mlp_dense_h_to_4h_weight5, gpt_neox_layers_10_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv140 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv139, gpt_neox_layers_10_mlp_dense_4h_to_h_weight5, gpt_neox_layers_10_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv141 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape235, gpt_neox_layers_10_attention_dense_weight5, gpt_neox_layers_10_attention_dense_bias5, lv140, lv137), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm122 = R.call_tir(cls.layer_norm1, (lv141, gpt_neox_layers_11_input_layernorm_weight5, gpt_neox_layers_11_input_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv142 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm122, gpt_neox_layers_11_attention_query_key_value_weight5, gpt_neox_layers_11_attention_query_key_value_bias5), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape236 = R.call_tir(cls.reshape4, (lv142,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape237 = R.call_tir(cls.reshape5, (reshape236,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv60 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape237), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape238 = R.call_tir(cls.reshape6, (lv60,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape239 = R.call_tir(cls.reshape7, (reshape238,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm123 = R.call_tir(cls.layer_norm1, (lv141, gpt_neox_layers_11_post_attention_layernorm_weight5, gpt_neox_layers_11_post_attention_layernorm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv143 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm123, gpt_neox_layers_11_mlp_dense_h_to_4h_weight5, gpt_neox_layers_11_mlp_dense_h_to_4h_bias5), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv144 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv143, gpt_neox_layers_11_mlp_dense_4h_to_h_weight5, gpt_neox_layers_11_mlp_dense_4h_to_h_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv145 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape239, gpt_neox_layers_11_attention_dense_weight5, gpt_neox_layers_11_attention_dense_bias5, lv144, lv141), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm124 = R.call_tir(cls.layer_norm1, (lv145, gpt_neox_final_layer_norm_weight5, gpt_neox_final_layer_norm_bias5), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv146 = R.call_tir(cls.fused_NT_matmul9_cast5, (layer_norm124, embed_out_weight5), out_sinfo=R.Tensor((1, seq_len, vocab_size), dtype="float32"))
gv5: R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype="float32"), R.Object) = lv146, paged_kv_cache
R.output(gv5)
return gv5
@R.function
def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object:
max_batch_size = T.int64()
max_total_seq_len = T.int64()
prefill_chunk_size = T.int64()
page_size = T.int64()
support_sliding_window = T.int64()
R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
paged_kv_cache: R.Object = R.call_pure_packed("vm.builtin.paged_attention_kv_cache_create_reduced", R.shape([max_batch_size, max_total_seq_len, prefill_chunk_size, page_size, support_sliding_window]), R.shape([0, 12]), R.prim_value(12), R.prim_value(12), R.prim_value(64), R.prim_value(1), R.prim_value(1), R.prim_value(10000), R.const(0.0, "float16"), cls.tir_kv_cache_transpose_append, cls.batch_prefill_paged_kv, cls.batch_decode_paged_kv, cls.batch_prefill_paged_kv_sliding_window, cls.batch_decode_paged_kv_sliding_window, cls.batch_prefill_ragged_kv, cls.merge_state_inplace, cls.fused_rope, cls.copy_single_page, cls.tir_kv_cache_debug_get_kv, cls.compact_kv_copy, cls.batch_tree_attn, cls.tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,))
return paged_kv_cache
@R.function
def decode(input_embed: R.Tensor((1, 1, 768), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor(("vocab_size", 768), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object):
vocab_size = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_layers_0_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[6]
gpt_neox_layers_0_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[9]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[10]
gpt_neox_layers_0_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[12]
gpt_neox_layers_1_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[13]
gpt_neox_layers_1_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[14]
gpt_neox_layers_1_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[15]
gpt_neox_layers_1_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[16]
gpt_neox_layers_1_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[17]
gpt_neox_layers_1_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[19]
gpt_neox_layers_1_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[21]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[22]
gpt_neox_layers_1_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[23]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[24]
gpt_neox_layers_2_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[25]
gpt_neox_layers_2_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[26]
gpt_neox_layers_2_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[27]
gpt_neox_layers_2_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[28]
gpt_neox_layers_2_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[29]
gpt_neox_layers_2_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[30]
gpt_neox_layers_2_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[31]
gpt_neox_layers_2_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[32]
gpt_neox_layers_2_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[33]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[34]
gpt_neox_layers_2_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[35]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[36]
gpt_neox_layers_3_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[37]
gpt_neox_layers_3_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[38]
gpt_neox_layers_3_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[39]
gpt_neox_layers_3_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[40]
gpt_neox_layers_3_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[41]
gpt_neox_layers_3_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[42]
gpt_neox_layers_3_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[43]
gpt_neox_layers_3_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[44]
gpt_neox_layers_3_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[45]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[46]
gpt_neox_layers_3_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[47]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[48]
gpt_neox_layers_4_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[49]
gpt_neox_layers_4_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[50]
gpt_neox_layers_4_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[51]
gpt_neox_layers_4_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[52]
gpt_neox_layers_4_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[53]
gpt_neox_layers_4_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[54]
gpt_neox_layers_4_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[55]
gpt_neox_layers_4_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[56]
gpt_neox_layers_4_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[57]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[58]
gpt_neox_layers_4_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[59]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[60]
gpt_neox_layers_5_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[61]
gpt_neox_layers_5_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[62]
gpt_neox_layers_5_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[63]
gpt_neox_layers_5_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[64]
gpt_neox_layers_5_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[65]
gpt_neox_layers_5_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[66]
gpt_neox_layers_5_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[67]
gpt_neox_layers_5_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[68]
gpt_neox_layers_5_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[69]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[70]
gpt_neox_layers_5_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[71]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[72]
gpt_neox_layers_6_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[73]
gpt_neox_layers_6_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[74]
gpt_neox_layers_6_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[75]
gpt_neox_layers_6_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[76]
gpt_neox_layers_6_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[77]
gpt_neox_layers_6_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[78]
gpt_neox_layers_6_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[79]
gpt_neox_layers_6_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[80]
gpt_neox_layers_6_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[81]
gpt_neox_layers_6_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[82]
gpt_neox_layers_6_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[83]
gpt_neox_layers_6_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[84]
gpt_neox_layers_7_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[85]
gpt_neox_layers_7_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[86]
gpt_neox_layers_7_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[87]
gpt_neox_layers_7_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[88]
gpt_neox_layers_7_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[89]
gpt_neox_layers_7_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[90]
gpt_neox_layers_7_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[91]
gpt_neox_layers_7_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[92]
gpt_neox_layers_7_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[93]
gpt_neox_layers_7_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[94]
gpt_neox_layers_7_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[95]
gpt_neox_layers_7_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[96]
gpt_neox_layers_8_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[97]
gpt_neox_layers_8_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[98]
gpt_neox_layers_8_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[99]
gpt_neox_layers_8_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[100]
gpt_neox_layers_8_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[101]
gpt_neox_layers_8_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[102]
gpt_neox_layers_8_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[103]
gpt_neox_layers_8_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[104]
gpt_neox_layers_8_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[105]
gpt_neox_layers_8_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[106]
gpt_neox_layers_8_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[107]
gpt_neox_layers_8_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[108]
gpt_neox_layers_9_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[109]
gpt_neox_layers_9_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[110]
gpt_neox_layers_9_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[111]
gpt_neox_layers_9_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[112]
gpt_neox_layers_9_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[113]
gpt_neox_layers_9_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[114]
gpt_neox_layers_9_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[115]
gpt_neox_layers_9_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[116]
gpt_neox_layers_9_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[117]
gpt_neox_layers_9_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[118]
gpt_neox_layers_9_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[119]
gpt_neox_layers_9_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[120]
gpt_neox_layers_10_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[121]
gpt_neox_layers_10_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[122]
gpt_neox_layers_10_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[123]
gpt_neox_layers_10_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[124]
gpt_neox_layers_10_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[125]
gpt_neox_layers_10_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[126]
gpt_neox_layers_10_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[127]
gpt_neox_layers_10_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[128]
gpt_neox_layers_10_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[129]
gpt_neox_layers_10_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[130]
gpt_neox_layers_10_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[131]
gpt_neox_layers_10_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[132]
gpt_neox_layers_11_input_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[133]
gpt_neox_layers_11_input_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[134]
gpt_neox_layers_11_post_attention_layernorm_weight2: R.Tensor((768,), dtype="float16") = packed_params[135]
gpt_neox_layers_11_post_attention_layernorm_bias2: R.Tensor((768,), dtype="float16") = packed_params[136]
gpt_neox_layers_11_attention_query_key_value_weight2: R.Tensor((2304, 768), dtype="float16") = packed_params[137]
gpt_neox_layers_11_attention_query_key_value_bias2: R.Tensor((2304,), dtype="float16") = packed_params[138]
gpt_neox_layers_11_attention_dense_weight2: R.Tensor((768, 768), dtype="float16") = packed_params[139]
gpt_neox_layers_11_attention_dense_bias2: R.Tensor((768,), dtype="float16") = packed_params[140]
gpt_neox_layers_11_mlp_dense_h_to_4h_weight2: R.Tensor((3072, 768), dtype="float16") = packed_params[141]
gpt_neox_layers_11_mlp_dense_h_to_4h_bias2: R.Tensor((3072,), dtype="float32") = packed_params[142]
gpt_neox_layers_11_mlp_dense_4h_to_h_weight2: R.Tensor((768, 3072), dtype="float16") = packed_params[143]
gpt_neox_layers_11_mlp_dense_4h_to_h_bias2: R.Tensor((768,), dtype="float32") = packed_params[144]
gpt_neox_final_layer_norm_weight2: R.Tensor((768,), dtype="float16") = packed_params[145]
gpt_neox_final_layer_norm_bias2: R.Tensor((768,), dtype="float16") = packed_params[146]
embed_out_weight2: R.Tensor((vocab_size, 768), dtype="float16") = packed_params[147]
layer_norm25 = R.call_tir(cls.layer_norm2, (input_embed, gpt_neox_layers_0_input_layernorm_weight2, gpt_neox_layers_0_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv147 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm25, gpt_neox_layers_0_attention_query_key_value_weight2, gpt_neox_layers_0_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv148 = R.call_tir(cls.fused_reshape8_reshape9, (lv147,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv13 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), lv148), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv149 = R.call_tir(cls.fused_reshape10_reshape11, (lv13,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm26 = R.call_tir(cls.layer_norm2, (input_embed, gpt_neox_layers_0_post_attention_layernorm_weight2, gpt_neox_layers_0_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv150 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm26, gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, gpt_neox_layers_0_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv151 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv150, gpt_neox_layers_0_mlp_dense_4h_to_h_weight2, gpt_neox_layers_0_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv152 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv149, gpt_neox_layers_0_attention_dense_weight2, gpt_neox_layers_0_attention_dense_bias2, lv151, input_embed), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm27 = R.call_tir(cls.layer_norm2, (lv152, gpt_neox_layers_1_input_layernorm_weight2, gpt_neox_layers_1_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv153 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm27, gpt_neox_layers_1_attention_query_key_value_weight2, gpt_neox_layers_1_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv154 = R.call_tir(cls.fused_reshape8_reshape9, (lv153,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv14 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), lv154), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv155 = R.call_tir(cls.fused_reshape10_reshape11, (lv14,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm28 = R.call_tir(cls.layer_norm2, (lv152, gpt_neox_layers_1_post_attention_layernorm_weight2, gpt_neox_layers_1_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv156 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm28, gpt_neox_layers_1_mlp_dense_h_to_4h_weight2, gpt_neox_layers_1_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv157 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv156, gpt_neox_layers_1_mlp_dense_4h_to_h_weight2, gpt_neox_layers_1_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv158 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv155, gpt_neox_layers_1_attention_dense_weight2, gpt_neox_layers_1_attention_dense_bias2, lv157, lv152), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm29 = R.call_tir(cls.layer_norm2, (lv158, gpt_neox_layers_2_input_layernorm_weight2, gpt_neox_layers_2_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv159 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm29, gpt_neox_layers_2_attention_query_key_value_weight2, gpt_neox_layers_2_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv160 = R.call_tir(cls.fused_reshape8_reshape9, (lv159,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv15 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), lv160), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv161 = R.call_tir(cls.fused_reshape10_reshape11, (lv15,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm30 = R.call_tir(cls.layer_norm2, (lv158, gpt_neox_layers_2_post_attention_layernorm_weight2, gpt_neox_layers_2_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv162 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm30, gpt_neox_layers_2_mlp_dense_h_to_4h_weight2, gpt_neox_layers_2_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv163 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv162, gpt_neox_layers_2_mlp_dense_4h_to_h_weight2, gpt_neox_layers_2_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv164 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv161, gpt_neox_layers_2_attention_dense_weight2, gpt_neox_layers_2_attention_dense_bias2, lv163, lv158), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm31 = R.call_tir(cls.layer_norm2, (lv164, gpt_neox_layers_3_input_layernorm_weight2, gpt_neox_layers_3_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv165 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm31, gpt_neox_layers_3_attention_query_key_value_weight2, gpt_neox_layers_3_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv166 = R.call_tir(cls.fused_reshape8_reshape9, (lv165,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv16 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), lv166), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv167 = R.call_tir(cls.fused_reshape10_reshape11, (lv16,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm32 = R.call_tir(cls.layer_norm2, (lv164, gpt_neox_layers_3_post_attention_layernorm_weight2, gpt_neox_layers_3_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv168 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm32, gpt_neox_layers_3_mlp_dense_h_to_4h_weight2, gpt_neox_layers_3_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv169 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv168, gpt_neox_layers_3_mlp_dense_4h_to_h_weight2, gpt_neox_layers_3_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv170 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv167, gpt_neox_layers_3_attention_dense_weight2, gpt_neox_layers_3_attention_dense_bias2, lv169, lv164), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm33 = R.call_tir(cls.layer_norm2, (lv170, gpt_neox_layers_4_input_layernorm_weight2, gpt_neox_layers_4_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv171 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm33, gpt_neox_layers_4_attention_query_key_value_weight2, gpt_neox_layers_4_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv172 = R.call_tir(cls.fused_reshape8_reshape9, (lv171,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv17 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), lv172), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv173 = R.call_tir(cls.fused_reshape10_reshape11, (lv17,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm34 = R.call_tir(cls.layer_norm2, (lv170, gpt_neox_layers_4_post_attention_layernorm_weight2, gpt_neox_layers_4_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv174 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm34, gpt_neox_layers_4_mlp_dense_h_to_4h_weight2, gpt_neox_layers_4_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv175 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv174, gpt_neox_layers_4_mlp_dense_4h_to_h_weight2, gpt_neox_layers_4_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv176 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv173, gpt_neox_layers_4_attention_dense_weight2, gpt_neox_layers_4_attention_dense_bias2, lv175, lv170), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm35 = R.call_tir(cls.layer_norm2, (lv176, gpt_neox_layers_5_input_layernorm_weight2, gpt_neox_layers_5_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv177 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm35, gpt_neox_layers_5_attention_query_key_value_weight2, gpt_neox_layers_5_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv178 = R.call_tir(cls.fused_reshape8_reshape9, (lv177,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv18 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), lv178), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv179 = R.call_tir(cls.fused_reshape10_reshape11, (lv18,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm36 = R.call_tir(cls.layer_norm2, (lv176, gpt_neox_layers_5_post_attention_layernorm_weight2, gpt_neox_layers_5_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv180 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm36, gpt_neox_layers_5_mlp_dense_h_to_4h_weight2, gpt_neox_layers_5_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv181 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv180, gpt_neox_layers_5_mlp_dense_4h_to_h_weight2, gpt_neox_layers_5_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv182 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv179, gpt_neox_layers_5_attention_dense_weight2, gpt_neox_layers_5_attention_dense_bias2, lv181, lv176), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm37 = R.call_tir(cls.layer_norm2, (lv182, gpt_neox_layers_6_input_layernorm_weight2, gpt_neox_layers_6_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv183 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm37, gpt_neox_layers_6_attention_query_key_value_weight2, gpt_neox_layers_6_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv184 = R.call_tir(cls.fused_reshape8_reshape9, (lv183,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv19 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), lv184), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv185 = R.call_tir(cls.fused_reshape10_reshape11, (lv19,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm38 = R.call_tir(cls.layer_norm2, (lv182, gpt_neox_layers_6_post_attention_layernorm_weight2, gpt_neox_layers_6_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv186 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm38, gpt_neox_layers_6_mlp_dense_h_to_4h_weight2, gpt_neox_layers_6_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv187 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv186, gpt_neox_layers_6_mlp_dense_4h_to_h_weight2, gpt_neox_layers_6_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv188 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv185, gpt_neox_layers_6_attention_dense_weight2, gpt_neox_layers_6_attention_dense_bias2, lv187, lv182), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm39 = R.call_tir(cls.layer_norm2, (lv188, gpt_neox_layers_7_input_layernorm_weight2, gpt_neox_layers_7_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv189 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm39, gpt_neox_layers_7_attention_query_key_value_weight2, gpt_neox_layers_7_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv190 = R.call_tir(cls.fused_reshape8_reshape9, (lv189,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv20 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), lv190), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv191 = R.call_tir(cls.fused_reshape10_reshape11, (lv20,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm40 = R.call_tir(cls.layer_norm2, (lv188, gpt_neox_layers_7_post_attention_layernorm_weight2, gpt_neox_layers_7_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv192 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm40, gpt_neox_layers_7_mlp_dense_h_to_4h_weight2, gpt_neox_layers_7_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv193 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv192, gpt_neox_layers_7_mlp_dense_4h_to_h_weight2, gpt_neox_layers_7_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv194 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv191, gpt_neox_layers_7_attention_dense_weight2, gpt_neox_layers_7_attention_dense_bias2, lv193, lv188), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm41 = R.call_tir(cls.layer_norm2, (lv194, gpt_neox_layers_8_input_layernorm_weight2, gpt_neox_layers_8_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv195 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm41, gpt_neox_layers_8_attention_query_key_value_weight2, gpt_neox_layers_8_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv196 = R.call_tir(cls.fused_reshape8_reshape9, (lv195,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv21 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), lv196), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv197 = R.call_tir(cls.fused_reshape10_reshape11, (lv21,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm42 = R.call_tir(cls.layer_norm2, (lv194, gpt_neox_layers_8_post_attention_layernorm_weight2, gpt_neox_layers_8_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv198 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm42, gpt_neox_layers_8_mlp_dense_h_to_4h_weight2, gpt_neox_layers_8_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv199 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv198, gpt_neox_layers_8_mlp_dense_4h_to_h_weight2, gpt_neox_layers_8_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv200 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv197, gpt_neox_layers_8_attention_dense_weight2, gpt_neox_layers_8_attention_dense_bias2, lv199, lv194), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm43 = R.call_tir(cls.layer_norm2, (lv200, gpt_neox_layers_9_input_layernorm_weight2, gpt_neox_layers_9_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv201 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm43, gpt_neox_layers_9_attention_query_key_value_weight2, gpt_neox_layers_9_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv202 = R.call_tir(cls.fused_reshape8_reshape9, (lv201,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv22 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), lv202), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv203 = R.call_tir(cls.fused_reshape10_reshape11, (lv22,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm44 = R.call_tir(cls.layer_norm2, (lv200, gpt_neox_layers_9_post_attention_layernorm_weight2, gpt_neox_layers_9_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv204 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm44, gpt_neox_layers_9_mlp_dense_h_to_4h_weight2, gpt_neox_layers_9_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv205 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv204, gpt_neox_layers_9_mlp_dense_4h_to_h_weight2, gpt_neox_layers_9_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv206 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv203, gpt_neox_layers_9_attention_dense_weight2, gpt_neox_layers_9_attention_dense_bias2, lv205, lv200), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm45 = R.call_tir(cls.layer_norm2, (lv206, gpt_neox_layers_10_input_layernorm_weight2, gpt_neox_layers_10_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv207 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm45, gpt_neox_layers_10_attention_query_key_value_weight2, gpt_neox_layers_10_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv208 = R.call_tir(cls.fused_reshape8_reshape9, (lv207,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv23 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), lv208), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv209 = R.call_tir(cls.fused_reshape10_reshape11, (lv23,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm46 = R.call_tir(cls.layer_norm2, (lv206, gpt_neox_layers_10_post_attention_layernorm_weight2, gpt_neox_layers_10_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv210 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm46, gpt_neox_layers_10_mlp_dense_h_to_4h_weight2, gpt_neox_layers_10_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv211 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv210, gpt_neox_layers_10_mlp_dense_4h_to_h_weight2, gpt_neox_layers_10_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv212 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv209, gpt_neox_layers_10_attention_dense_weight2, gpt_neox_layers_10_attention_dense_bias2, lv211, lv206), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm47 = R.call_tir(cls.layer_norm2, (lv212, gpt_neox_layers_11_input_layernorm_weight2, gpt_neox_layers_11_input_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv213 = R.call_tir(cls.fused_NT_matmul10_add10, (layer_norm47, gpt_neox_layers_11_attention_query_key_value_weight2, gpt_neox_layers_11_attention_query_key_value_bias2), out_sinfo=R.Tensor((1, 1, 2304), dtype="float16"))
lv214 = R.call_tir(cls.fused_reshape8_reshape9, (lv213,), out_sinfo=R.Tensor((1, 36, 64), dtype="float16"))
lv24 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), lv214), out_sinfo=R.Tensor((1, 12, 64), dtype="float16"))
lv215 = R.call_tir(cls.fused_reshape10_reshape11, (lv24,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm48 = R.call_tir(cls.layer_norm2, (lv212, gpt_neox_layers_11_post_attention_layernorm_weight2, gpt_neox_layers_11_post_attention_layernorm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv216 = R.call_tir(cls.fused_NT_matmul12_add12_gelu2_cast6, (layer_norm48, gpt_neox_layers_11_mlp_dense_h_to_4h_weight2, gpt_neox_layers_11_mlp_dense_h_to_4h_bias2), out_sinfo=R.Tensor((1, 1, 3072), dtype="float16"))
lv217 = R.call_tir(cls.fused_NT_matmul13_add13_cast7, (lv216, gpt_neox_layers_11_mlp_dense_4h_to_h_weight2, gpt_neox_layers_11_mlp_dense_4h_to_h_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv218 = R.call_tir(cls.fused_NT_matmul11_add11_add14_add14, (lv215, gpt_neox_layers_11_attention_dense_weight2, gpt_neox_layers_11_attention_dense_bias2, lv217, lv212), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
layer_norm49 = R.call_tir(cls.layer_norm2, (lv218, gpt_neox_final_layer_norm_weight2, gpt_neox_final_layer_norm_bias2), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv219 = R.call_tir(cls.fused_NT_matmul14_cast8, (layer_norm49, embed_out_weight2), out_sinfo=R.Tensor((1, 1, vocab_size), dtype="float32"))
gv2: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = lv219, paged_kv_cache
R.output(gv2)
return gv2
@R.function
def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor(("vocab_size", 768), dtype="float16"))) -> R.Tensor(("seq_len", 768), dtype="float16"):
seq_len = T.int64()
vocab_size = T.int64()
R.func_attr({"num_input": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_embed_in_weight: R.Tensor((vocab_size, 768), dtype="float16") = packed_params[0]
gv = R.call_tir(cls.take1, (gpt_neox_embed_in_weight, input_ids), out_sinfo=R.Tensor((seq_len, 768), dtype="float16"))
R.output(gv)
return gv
@R.function
def prefill(input_embed: R.Tensor((1, "seq_len", 768), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((2304, 768), dtype="float16"), R.Tensor((2304,), dtype="float16"), R.Tensor((768, 768), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor((3072, 768), dtype="float16"), R.Tensor((3072,), dtype="float32"), R.Tensor((768, 3072), dtype="float16"), R.Tensor((768,), dtype="float32"), R.Tensor((768,), dtype="float16"), R.Tensor((768,), dtype="float16"), R.Tensor(("vocab_size", 768), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object):
vocab_size = T.int64()
seq_len = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_layers_0_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[6]
gpt_neox_layers_0_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[9]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[10]
gpt_neox_layers_0_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[12]
gpt_neox_layers_1_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[13]
gpt_neox_layers_1_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[14]
gpt_neox_layers_1_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[15]
gpt_neox_layers_1_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[16]
gpt_neox_layers_1_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[17]
gpt_neox_layers_1_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[19]
gpt_neox_layers_1_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[21]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[22]
gpt_neox_layers_1_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[23]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[24]
gpt_neox_layers_2_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[25]
gpt_neox_layers_2_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[26]
gpt_neox_layers_2_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[27]
gpt_neox_layers_2_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[28]
gpt_neox_layers_2_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[29]
gpt_neox_layers_2_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[30]
gpt_neox_layers_2_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[31]
gpt_neox_layers_2_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[32]
gpt_neox_layers_2_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[33]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[34]
gpt_neox_layers_2_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[35]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[36]
gpt_neox_layers_3_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[37]
gpt_neox_layers_3_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[38]
gpt_neox_layers_3_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[39]
gpt_neox_layers_3_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[40]
gpt_neox_layers_3_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[41]
gpt_neox_layers_3_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[42]
gpt_neox_layers_3_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[43]
gpt_neox_layers_3_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[44]
gpt_neox_layers_3_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[45]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[46]
gpt_neox_layers_3_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[47]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[48]
gpt_neox_layers_4_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[49]
gpt_neox_layers_4_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[50]
gpt_neox_layers_4_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[51]
gpt_neox_layers_4_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[52]
gpt_neox_layers_4_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[53]
gpt_neox_layers_4_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[54]
gpt_neox_layers_4_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[55]
gpt_neox_layers_4_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[56]
gpt_neox_layers_4_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[57]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[58]
gpt_neox_layers_4_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[59]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[60]
gpt_neox_layers_5_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[61]
gpt_neox_layers_5_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[62]
gpt_neox_layers_5_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[63]
gpt_neox_layers_5_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[64]
gpt_neox_layers_5_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[65]
gpt_neox_layers_5_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[66]
gpt_neox_layers_5_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[67]
gpt_neox_layers_5_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[68]
gpt_neox_layers_5_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[69]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[70]
gpt_neox_layers_5_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[71]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[72]
gpt_neox_layers_6_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[73]
gpt_neox_layers_6_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[74]
gpt_neox_layers_6_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[75]
gpt_neox_layers_6_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[76]
gpt_neox_layers_6_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[77]
gpt_neox_layers_6_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[78]
gpt_neox_layers_6_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[79]
gpt_neox_layers_6_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[80]
gpt_neox_layers_6_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[81]
gpt_neox_layers_6_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[82]
gpt_neox_layers_6_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[83]
gpt_neox_layers_6_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[84]
gpt_neox_layers_7_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[85]
gpt_neox_layers_7_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[86]
gpt_neox_layers_7_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[87]
gpt_neox_layers_7_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[88]
gpt_neox_layers_7_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[89]
gpt_neox_layers_7_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[90]
gpt_neox_layers_7_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[91]
gpt_neox_layers_7_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[92]
gpt_neox_layers_7_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[93]
gpt_neox_layers_7_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[94]
gpt_neox_layers_7_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[95]
gpt_neox_layers_7_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[96]
gpt_neox_layers_8_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[97]
gpt_neox_layers_8_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[98]
gpt_neox_layers_8_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[99]
gpt_neox_layers_8_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[100]
gpt_neox_layers_8_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[101]
gpt_neox_layers_8_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[102]
gpt_neox_layers_8_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[103]
gpt_neox_layers_8_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[104]
gpt_neox_layers_8_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[105]
gpt_neox_layers_8_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[106]
gpt_neox_layers_8_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[107]
gpt_neox_layers_8_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[108]
gpt_neox_layers_9_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[109]
gpt_neox_layers_9_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[110]
gpt_neox_layers_9_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[111]
gpt_neox_layers_9_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[112]
gpt_neox_layers_9_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[113]
gpt_neox_layers_9_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[114]
gpt_neox_layers_9_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[115]
gpt_neox_layers_9_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[116]
gpt_neox_layers_9_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[117]
gpt_neox_layers_9_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[118]
gpt_neox_layers_9_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[119]
gpt_neox_layers_9_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[120]
gpt_neox_layers_10_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[121]
gpt_neox_layers_10_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[122]
gpt_neox_layers_10_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[123]
gpt_neox_layers_10_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[124]
gpt_neox_layers_10_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[125]
gpt_neox_layers_10_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[126]
gpt_neox_layers_10_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[127]
gpt_neox_layers_10_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[128]
gpt_neox_layers_10_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[129]
gpt_neox_layers_10_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[130]
gpt_neox_layers_10_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[131]
gpt_neox_layers_10_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[132]
gpt_neox_layers_11_input_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[133]
gpt_neox_layers_11_input_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[134]
gpt_neox_layers_11_post_attention_layernorm_weight1: R.Tensor((768,), dtype="float16") = packed_params[135]
gpt_neox_layers_11_post_attention_layernorm_bias1: R.Tensor((768,), dtype="float16") = packed_params[136]
gpt_neox_layers_11_attention_query_key_value_weight1: R.Tensor((2304, 768), dtype="float16") = packed_params[137]
gpt_neox_layers_11_attention_query_key_value_bias1: R.Tensor((2304,), dtype="float16") = packed_params[138]
gpt_neox_layers_11_attention_dense_weight1: R.Tensor((768, 768), dtype="float16") = packed_params[139]
gpt_neox_layers_11_attention_dense_bias1: R.Tensor((768,), dtype="float16") = packed_params[140]
gpt_neox_layers_11_mlp_dense_h_to_4h_weight1: R.Tensor((3072, 768), dtype="float16") = packed_params[141]
gpt_neox_layers_11_mlp_dense_h_to_4h_bias1: R.Tensor((3072,), dtype="float32") = packed_params[142]
gpt_neox_layers_11_mlp_dense_4h_to_h_weight1: R.Tensor((768, 3072), dtype="float16") = packed_params[143]
gpt_neox_layers_11_mlp_dense_4h_to_h_bias1: R.Tensor((768,), dtype="float32") = packed_params[144]
gpt_neox_final_layer_norm_weight1: R.Tensor((768,), dtype="float16") = packed_params[145]
gpt_neox_final_layer_norm_bias1: R.Tensor((768,), dtype="float16") = packed_params[146]
embed_out_weight1: R.Tensor((vocab_size, 768), dtype="float16") = packed_params[147]
layer_norm = R.call_tir(cls.layer_norm1, (input_embed, gpt_neox_layers_0_input_layernorm_weight1, gpt_neox_layers_0_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv220 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm, gpt_neox_layers_0_attention_query_key_value_weight1, gpt_neox_layers_0_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape = R.call_tir(cls.reshape4, (lv220,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape1 = R.call_tir(cls.reshape5, (reshape,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape2 = R.call_tir(cls.reshape6, (lv,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape3 = R.call_tir(cls.reshape7, (reshape2,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm1 = R.call_tir(cls.layer_norm1, (input_embed, gpt_neox_layers_0_post_attention_layernorm_weight1, gpt_neox_layers_0_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv221 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm1, gpt_neox_layers_0_mlp_dense_h_to_4h_weight1, gpt_neox_layers_0_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv222 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv221, gpt_neox_layers_0_mlp_dense_4h_to_h_weight1, gpt_neox_layers_0_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv223 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape3, gpt_neox_layers_0_attention_dense_weight1, gpt_neox_layers_0_attention_dense_bias1, lv222, input_embed), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm2 = R.call_tir(cls.layer_norm1, (lv223, gpt_neox_layers_1_input_layernorm_weight1, gpt_neox_layers_1_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv224 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm2, gpt_neox_layers_1_attention_query_key_value_weight1, gpt_neox_layers_1_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape4 = R.call_tir(cls.reshape4, (lv224,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape5 = R.call_tir(cls.reshape5, (reshape4,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv1 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape6 = R.call_tir(cls.reshape6, (lv1,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape7 = R.call_tir(cls.reshape7, (reshape6,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm3 = R.call_tir(cls.layer_norm1, (lv223, gpt_neox_layers_1_post_attention_layernorm_weight1, gpt_neox_layers_1_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv225 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm3, gpt_neox_layers_1_mlp_dense_h_to_4h_weight1, gpt_neox_layers_1_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv226 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv225, gpt_neox_layers_1_mlp_dense_4h_to_h_weight1, gpt_neox_layers_1_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv227 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape7, gpt_neox_layers_1_attention_dense_weight1, gpt_neox_layers_1_attention_dense_bias1, lv226, lv223), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm4 = R.call_tir(cls.layer_norm1, (lv227, gpt_neox_layers_2_input_layernorm_weight1, gpt_neox_layers_2_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv228 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm4, gpt_neox_layers_2_attention_query_key_value_weight1, gpt_neox_layers_2_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape8 = R.call_tir(cls.reshape4, (lv228,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape9 = R.call_tir(cls.reshape5, (reshape8,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv2 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape10 = R.call_tir(cls.reshape6, (lv2,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape11 = R.call_tir(cls.reshape7, (reshape10,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm5 = R.call_tir(cls.layer_norm1, (lv227, gpt_neox_layers_2_post_attention_layernorm_weight1, gpt_neox_layers_2_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv229 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm5, gpt_neox_layers_2_mlp_dense_h_to_4h_weight1, gpt_neox_layers_2_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv230 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv229, gpt_neox_layers_2_mlp_dense_4h_to_h_weight1, gpt_neox_layers_2_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv231 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape11, gpt_neox_layers_2_attention_dense_weight1, gpt_neox_layers_2_attention_dense_bias1, lv230, lv227), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm6 = R.call_tir(cls.layer_norm1, (lv231, gpt_neox_layers_3_input_layernorm_weight1, gpt_neox_layers_3_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv232 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm6, gpt_neox_layers_3_attention_query_key_value_weight1, gpt_neox_layers_3_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape12 = R.call_tir(cls.reshape4, (lv232,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape13 = R.call_tir(cls.reshape5, (reshape12,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv3 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape14 = R.call_tir(cls.reshape6, (lv3,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape15 = R.call_tir(cls.reshape7, (reshape14,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm7 = R.call_tir(cls.layer_norm1, (lv231, gpt_neox_layers_3_post_attention_layernorm_weight1, gpt_neox_layers_3_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv233 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm7, gpt_neox_layers_3_mlp_dense_h_to_4h_weight1, gpt_neox_layers_3_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv234 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv233, gpt_neox_layers_3_mlp_dense_4h_to_h_weight1, gpt_neox_layers_3_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv235 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape15, gpt_neox_layers_3_attention_dense_weight1, gpt_neox_layers_3_attention_dense_bias1, lv234, lv231), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm8 = R.call_tir(cls.layer_norm1, (lv235, gpt_neox_layers_4_input_layernorm_weight1, gpt_neox_layers_4_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv236 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm8, gpt_neox_layers_4_attention_query_key_value_weight1, gpt_neox_layers_4_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape16 = R.call_tir(cls.reshape4, (lv236,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape17 = R.call_tir(cls.reshape5, (reshape16,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv4 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape18 = R.call_tir(cls.reshape6, (lv4,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape19 = R.call_tir(cls.reshape7, (reshape18,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm9 = R.call_tir(cls.layer_norm1, (lv235, gpt_neox_layers_4_post_attention_layernorm_weight1, gpt_neox_layers_4_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv237 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm9, gpt_neox_layers_4_mlp_dense_h_to_4h_weight1, gpt_neox_layers_4_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv238 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv237, gpt_neox_layers_4_mlp_dense_4h_to_h_weight1, gpt_neox_layers_4_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv239 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape19, gpt_neox_layers_4_attention_dense_weight1, gpt_neox_layers_4_attention_dense_bias1, lv238, lv235), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm10 = R.call_tir(cls.layer_norm1, (lv239, gpt_neox_layers_5_input_layernorm_weight1, gpt_neox_layers_5_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv240 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm10, gpt_neox_layers_5_attention_query_key_value_weight1, gpt_neox_layers_5_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape20 = R.call_tir(cls.reshape4, (lv240,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape21 = R.call_tir(cls.reshape5, (reshape20,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv5 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape22 = R.call_tir(cls.reshape6, (lv5,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape23 = R.call_tir(cls.reshape7, (reshape22,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm11 = R.call_tir(cls.layer_norm1, (lv239, gpt_neox_layers_5_post_attention_layernorm_weight1, gpt_neox_layers_5_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv241 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm11, gpt_neox_layers_5_mlp_dense_h_to_4h_weight1, gpt_neox_layers_5_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv242 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv241, gpt_neox_layers_5_mlp_dense_4h_to_h_weight1, gpt_neox_layers_5_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv243 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape23, gpt_neox_layers_5_attention_dense_weight1, gpt_neox_layers_5_attention_dense_bias1, lv242, lv239), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm12 = R.call_tir(cls.layer_norm1, (lv243, gpt_neox_layers_6_input_layernorm_weight1, gpt_neox_layers_6_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv244 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm12, gpt_neox_layers_6_attention_query_key_value_weight1, gpt_neox_layers_6_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape24 = R.call_tir(cls.reshape4, (lv244,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape25 = R.call_tir(cls.reshape5, (reshape24,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv6 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape25), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape26 = R.call_tir(cls.reshape6, (lv6,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape27 = R.call_tir(cls.reshape7, (reshape26,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm13 = R.call_tir(cls.layer_norm1, (lv243, gpt_neox_layers_6_post_attention_layernorm_weight1, gpt_neox_layers_6_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv245 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm13, gpt_neox_layers_6_mlp_dense_h_to_4h_weight1, gpt_neox_layers_6_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv246 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv245, gpt_neox_layers_6_mlp_dense_4h_to_h_weight1, gpt_neox_layers_6_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv247 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape27, gpt_neox_layers_6_attention_dense_weight1, gpt_neox_layers_6_attention_dense_bias1, lv246, lv243), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm14 = R.call_tir(cls.layer_norm1, (lv247, gpt_neox_layers_7_input_layernorm_weight1, gpt_neox_layers_7_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv248 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm14, gpt_neox_layers_7_attention_query_key_value_weight1, gpt_neox_layers_7_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape28 = R.call_tir(cls.reshape4, (lv248,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape29 = R.call_tir(cls.reshape5, (reshape28,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv7 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape29), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape30 = R.call_tir(cls.reshape6, (lv7,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape31 = R.call_tir(cls.reshape7, (reshape30,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm15 = R.call_tir(cls.layer_norm1, (lv247, gpt_neox_layers_7_post_attention_layernorm_weight1, gpt_neox_layers_7_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv249 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm15, gpt_neox_layers_7_mlp_dense_h_to_4h_weight1, gpt_neox_layers_7_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv250 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv249, gpt_neox_layers_7_mlp_dense_4h_to_h_weight1, gpt_neox_layers_7_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv251 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape31, gpt_neox_layers_7_attention_dense_weight1, gpt_neox_layers_7_attention_dense_bias1, lv250, lv247), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm16 = R.call_tir(cls.layer_norm1, (lv251, gpt_neox_layers_8_input_layernorm_weight1, gpt_neox_layers_8_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv252 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm16, gpt_neox_layers_8_attention_query_key_value_weight1, gpt_neox_layers_8_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape32 = R.call_tir(cls.reshape4, (lv252,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape33 = R.call_tir(cls.reshape5, (reshape32,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv8 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape33), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape34 = R.call_tir(cls.reshape6, (lv8,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape35 = R.call_tir(cls.reshape7, (reshape34,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm17 = R.call_tir(cls.layer_norm1, (lv251, gpt_neox_layers_8_post_attention_layernorm_weight1, gpt_neox_layers_8_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv253 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm17, gpt_neox_layers_8_mlp_dense_h_to_4h_weight1, gpt_neox_layers_8_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv254 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv253, gpt_neox_layers_8_mlp_dense_4h_to_h_weight1, gpt_neox_layers_8_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv255 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape35, gpt_neox_layers_8_attention_dense_weight1, gpt_neox_layers_8_attention_dense_bias1, lv254, lv251), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm18 = R.call_tir(cls.layer_norm1, (lv255, gpt_neox_layers_9_input_layernorm_weight1, gpt_neox_layers_9_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv256 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm18, gpt_neox_layers_9_attention_query_key_value_weight1, gpt_neox_layers_9_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape36 = R.call_tir(cls.reshape4, (lv256,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape37 = R.call_tir(cls.reshape5, (reshape36,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv9 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape37), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape38 = R.call_tir(cls.reshape6, (lv9,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape39 = R.call_tir(cls.reshape7, (reshape38,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm19 = R.call_tir(cls.layer_norm1, (lv255, gpt_neox_layers_9_post_attention_layernorm_weight1, gpt_neox_layers_9_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv257 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm19, gpt_neox_layers_9_mlp_dense_h_to_4h_weight1, gpt_neox_layers_9_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv258 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv257, gpt_neox_layers_9_mlp_dense_4h_to_h_weight1, gpt_neox_layers_9_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv259 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape39, gpt_neox_layers_9_attention_dense_weight1, gpt_neox_layers_9_attention_dense_bias1, lv258, lv255), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm20 = R.call_tir(cls.layer_norm1, (lv259, gpt_neox_layers_10_input_layernorm_weight1, gpt_neox_layers_10_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv260 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm20, gpt_neox_layers_10_attention_query_key_value_weight1, gpt_neox_layers_10_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape40 = R.call_tir(cls.reshape4, (lv260,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape41 = R.call_tir(cls.reshape5, (reshape40,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv10 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape41), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape42 = R.call_tir(cls.reshape6, (lv10,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape43 = R.call_tir(cls.reshape7, (reshape42,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm21 = R.call_tir(cls.layer_norm1, (lv259, gpt_neox_layers_10_post_attention_layernorm_weight1, gpt_neox_layers_10_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv261 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm21, gpt_neox_layers_10_mlp_dense_h_to_4h_weight1, gpt_neox_layers_10_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv262 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv261, gpt_neox_layers_10_mlp_dense_4h_to_h_weight1, gpt_neox_layers_10_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv263 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape43, gpt_neox_layers_10_attention_dense_weight1, gpt_neox_layers_10_attention_dense_bias1, lv262, lv259), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm22 = R.call_tir(cls.layer_norm1, (lv263, gpt_neox_layers_11_input_layernorm_weight1, gpt_neox_layers_11_input_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv264 = R.call_tir(cls.fused_NT_matmul5_add5, (layer_norm22, gpt_neox_layers_11_attention_query_key_value_weight1, gpt_neox_layers_11_attention_query_key_value_bias1), out_sinfo=R.Tensor((1, seq_len, 2304), dtype="float16"))
reshape44 = R.call_tir(cls.reshape4, (lv264,), out_sinfo=R.Tensor((1, seq_len, 36, 64), dtype="float16"))
reshape45 = R.call_tir(cls.reshape5, (reshape44,), out_sinfo=R.Tensor((seq_len, 36, 64), dtype="float16"))
lv11 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape45), out_sinfo=R.Tensor((seq_len, 12, 64), dtype="float16"))
reshape46 = R.call_tir(cls.reshape6, (lv11,), out_sinfo=R.Tensor((1, seq_len, 12, 64), dtype="float16"))
reshape47 = R.call_tir(cls.reshape7, (reshape46,), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm23 = R.call_tir(cls.layer_norm1, (lv263, gpt_neox_layers_11_post_attention_layernorm_weight1, gpt_neox_layers_11_post_attention_layernorm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv265 = R.call_tir(cls.fused_NT_matmul7_add7_gelu1_cast3, (layer_norm23, gpt_neox_layers_11_mlp_dense_h_to_4h_weight1, gpt_neox_layers_11_mlp_dense_h_to_4h_bias1), out_sinfo=R.Tensor((1, seq_len, 3072), dtype="float16"))
lv266 = R.call_tir(cls.fused_NT_matmul8_add8_cast4, (lv265, gpt_neox_layers_11_mlp_dense_4h_to_h_weight1, gpt_neox_layers_11_mlp_dense_4h_to_h_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv267 = R.call_tir(cls.fused_NT_matmul6_add6_add9_add9, (reshape47, gpt_neox_layers_11_attention_dense_weight1, gpt_neox_layers_11_attention_dense_bias1, lv266, lv263), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
layer_norm24 = R.call_tir(cls.layer_norm1, (lv267, gpt_neox_final_layer_norm_weight1, gpt_neox_final_layer_norm_bias1), out_sinfo=R.Tensor((1, seq_len, 768), dtype="float16"))
lv12 = R.call_tir(cls.index, (layer_norm24,), out_sinfo=R.Tensor((1, 1, 768), dtype="float16"))
lv268 = R.call_tir(cls.fused_NT_matmul14_cast8, (lv12, embed_out_weight1), out_sinfo=R.Tensor((1, 1, vocab_size), dtype="float32"))
gv1: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = lv268, paged_kv_cache
R.output(gv1)
return gv1
@R.function
def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"):
batch_size = T.int64(is_size_var=True)
vocab_size = T.int64(is_size_var=True)
R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", logits, R.shape([batch_size, vocab_size]), sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),))
lv1 = R.call_tir(cls.chunk_lse, (lv, temperature), out_sinfo=[R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32"), R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32")])
lv2: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[0]
lv3: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[1]
lv4 = R.call_tir(cls.softmax_with_chunked_sum, (lv, temperature, lv2, lv3), out_sinfo=R.Tensor((batch_size, vocab_size), dtype="float32"))
gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", lv4, R.shape([batch_size, 1, vocab_size]), sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),))
R.output(gv)
return gv |